AInstEmitSimdHelper.cs 47 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510
  1. using ChocolArm64.Decoder;
  2. using ChocolArm64.State;
  3. using ChocolArm64.Translation;
  4. using System;
  5. using System.Reflection;
  6. using System.Reflection.Emit;
  7. using System.Runtime.Intrinsics;
  8. using System.Runtime.Intrinsics.X86;
  9. namespace ChocolArm64.Instruction
  10. {
  11. static class AInstEmitSimdHelper
  12. {
  13. public static readonly Type[] IntTypesPerSizeLog2 = new Type[]
  14. {
  15. typeof(sbyte),
  16. typeof(short),
  17. typeof(int),
  18. typeof(long)
  19. };
  20. public static readonly Type[] UIntTypesPerSizeLog2 = new Type[]
  21. {
  22. typeof(byte),
  23. typeof(ushort),
  24. typeof(uint),
  25. typeof(ulong)
  26. };
  27. public static readonly Type[] VectorIntTypesPerSizeLog2 = new Type[]
  28. {
  29. typeof(Vector128<sbyte>),
  30. typeof(Vector128<short>),
  31. typeof(Vector128<int>),
  32. typeof(Vector128<long>)
  33. };
  34. public static readonly Type[] VectorUIntTypesPerSizeLog2 = new Type[]
  35. {
  36. typeof(Vector128<byte>),
  37. typeof(Vector128<ushort>),
  38. typeof(Vector128<uint>),
  39. typeof(Vector128<ulong>)
  40. };
  41. [Flags]
  42. public enum OperFlags
  43. {
  44. Rd = 1 << 0,
  45. Rn = 1 << 1,
  46. Rm = 1 << 2,
  47. Ra = 1 << 3,
  48. RnRm = Rn | Rm,
  49. RdRn = Rd | Rn,
  50. RaRnRm = Ra | Rn | Rm,
  51. RdRnRm = Rd | Rn | Rm
  52. }
  53. public static int GetImmShl(AOpCodeSimdShImm Op)
  54. {
  55. return Op.Imm - (8 << Op.Size);
  56. }
  57. public static int GetImmShr(AOpCodeSimdShImm Op)
  58. {
  59. return (8 << (Op.Size + 1)) - Op.Imm;
  60. }
  61. public static void EmitSse2Op(AILEmitterCtx Context, string Name)
  62. {
  63. EmitSseOp(Context, Name, typeof(Sse2));
  64. }
  65. public static void EmitSse41Op(AILEmitterCtx Context, string Name)
  66. {
  67. EmitSseOp(Context, Name, typeof(Sse41));
  68. }
  69. public static void EmitSse42Op(AILEmitterCtx Context, string Name)
  70. {
  71. EmitSseOp(Context, Name, typeof(Sse42));
  72. }
  73. private static void EmitSseOp(AILEmitterCtx Context, string Name, Type Type)
  74. {
  75. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  76. EmitLdvecWithSignedCast(Context, Op.Rn, Op.Size);
  77. Type BaseType = VectorIntTypesPerSizeLog2[Op.Size];
  78. if (Op is AOpCodeSimdReg BinOp)
  79. {
  80. EmitLdvecWithSignedCast(Context, BinOp.Rm, Op.Size);
  81. Context.EmitCall(Type.GetMethod(Name, new Type[] { BaseType, BaseType }));
  82. }
  83. else
  84. {
  85. Context.EmitCall(Type.GetMethod(Name, new Type[] { BaseType }));
  86. }
  87. EmitStvecWithSignedCast(Context, Op.Rd, Op.Size);
  88. if (Op.RegisterSize == ARegisterSize.SIMD64)
  89. {
  90. EmitVectorZeroUpper(Context, Op.Rd);
  91. }
  92. }
  93. public static void EmitLdvecWithSignedCast(AILEmitterCtx Context, int Reg, int Size)
  94. {
  95. Context.EmitLdvec(Reg);
  96. switch (Size)
  97. {
  98. case 0: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToSByte)); break;
  99. case 1: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToInt16)); break;
  100. case 2: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToInt32)); break;
  101. case 3: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToInt64)); break;
  102. default: throw new ArgumentOutOfRangeException(nameof(Size));
  103. }
  104. }
  105. public static void EmitLdvecWithCastToDouble(AILEmitterCtx Context, int Reg)
  106. {
  107. Context.EmitLdvec(Reg);
  108. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToDouble));
  109. }
  110. public static void EmitStvecWithCastFromDouble(AILEmitterCtx Context, int Reg)
  111. {
  112. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorDoubleToSingle));
  113. Context.EmitStvec(Reg);
  114. }
  115. public static void EmitLdvecWithUnsignedCast(AILEmitterCtx Context, int Reg, int Size)
  116. {
  117. Context.EmitLdvec(Reg);
  118. switch (Size)
  119. {
  120. case 0: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToByte)); break;
  121. case 1: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToUInt16)); break;
  122. case 2: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToUInt32)); break;
  123. case 3: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToUInt64)); break;
  124. default: throw new ArgumentOutOfRangeException(nameof(Size));
  125. }
  126. }
  127. public static void EmitStvecWithSignedCast(AILEmitterCtx Context, int Reg, int Size)
  128. {
  129. switch (Size)
  130. {
  131. case 0: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSByteToSingle)); break;
  132. case 1: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInt16ToSingle)); break;
  133. case 2: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInt32ToSingle)); break;
  134. case 3: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInt64ToSingle)); break;
  135. default: throw new ArgumentOutOfRangeException(nameof(Size));
  136. }
  137. Context.EmitStvec(Reg);
  138. }
  139. public static void EmitStvecWithUnsignedCast(AILEmitterCtx Context, int Reg, int Size)
  140. {
  141. switch (Size)
  142. {
  143. case 0: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorByteToSingle)); break;
  144. case 1: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorUInt16ToSingle)); break;
  145. case 2: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorUInt32ToSingle)); break;
  146. case 3: AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorUInt64ToSingle)); break;
  147. default: throw new ArgumentOutOfRangeException(nameof(Size));
  148. }
  149. Context.EmitStvec(Reg);
  150. }
  151. public static void EmitScalarSseOrSse2OpF(AILEmitterCtx Context, string Name)
  152. {
  153. EmitSseOrSse2OpF(Context, Name, true);
  154. }
  155. public static void EmitVectorSseOrSse2OpF(AILEmitterCtx Context, string Name)
  156. {
  157. EmitSseOrSse2OpF(Context, Name, false);
  158. }
  159. public static void EmitSseOrSse2OpF(AILEmitterCtx Context, string Name, bool Scalar)
  160. {
  161. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  162. int SizeF = Op.Size & 1;
  163. void Ldvec(int Reg)
  164. {
  165. Context.EmitLdvec(Reg);
  166. if (SizeF == 1)
  167. {
  168. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleToDouble));
  169. }
  170. }
  171. Ldvec(Op.Rn);
  172. Type Type;
  173. Type BaseType;
  174. if (SizeF == 0)
  175. {
  176. Type = typeof(Sse);
  177. BaseType = typeof(Vector128<float>);
  178. }
  179. else /* if (SizeF == 1) */
  180. {
  181. Type = typeof(Sse2);
  182. BaseType = typeof(Vector128<double>);
  183. }
  184. if (Op is AOpCodeSimdReg BinOp)
  185. {
  186. Ldvec(BinOp.Rm);
  187. Context.EmitCall(Type.GetMethod(Name, new Type[] { BaseType, BaseType }));
  188. }
  189. else
  190. {
  191. Context.EmitCall(Type.GetMethod(Name, new Type[] { BaseType }));
  192. }
  193. if (SizeF == 1)
  194. {
  195. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorDoubleToSingle));
  196. }
  197. Context.EmitStvec(Op.Rd);
  198. if (Scalar)
  199. {
  200. if (SizeF == 0)
  201. {
  202. EmitVectorZero32_128(Context, Op.Rd);
  203. }
  204. else /* if (SizeF == 1) */
  205. {
  206. EmitVectorZeroUpper(Context, Op.Rd);
  207. }
  208. }
  209. else if (Op.RegisterSize == ARegisterSize.SIMD64)
  210. {
  211. EmitVectorZeroUpper(Context, Op.Rd);
  212. }
  213. }
  214. public static void EmitUnaryMathCall(AILEmitterCtx Context, string Name)
  215. {
  216. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  217. int SizeF = Op.Size & 1;
  218. MethodInfo MthdInfo;
  219. if (SizeF == 0)
  220. {
  221. MthdInfo = typeof(MathF).GetMethod(Name, new Type[] { typeof(float) });
  222. }
  223. else /* if (SizeF == 1) */
  224. {
  225. MthdInfo = typeof(Math).GetMethod(Name, new Type[] { typeof(double) });
  226. }
  227. Context.EmitCall(MthdInfo);
  228. }
  229. public static void EmitBinaryMathCall(AILEmitterCtx Context, string Name)
  230. {
  231. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  232. int SizeF = Op.Size & 1;
  233. MethodInfo MthdInfo;
  234. if (SizeF == 0)
  235. {
  236. MthdInfo = typeof(MathF).GetMethod(Name, new Type[] { typeof(float), typeof(float) });
  237. }
  238. else /* if (SizeF == 1) */
  239. {
  240. MthdInfo = typeof(Math).GetMethod(Name, new Type[] { typeof(double), typeof(double) });
  241. }
  242. Context.EmitCall(MthdInfo);
  243. }
  244. public static void EmitRoundMathCall(AILEmitterCtx Context, MidpointRounding RoundMode)
  245. {
  246. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  247. int SizeF = Op.Size & 1;
  248. Context.EmitLdc_I4((int)RoundMode);
  249. MethodInfo MthdInfo;
  250. Type[] Types = new Type[] { null, typeof(MidpointRounding) };
  251. Types[0] = SizeF == 0
  252. ? typeof(float)
  253. : typeof(double);
  254. if (SizeF == 0)
  255. {
  256. MthdInfo = typeof(MathF).GetMethod(nameof(MathF.Round), Types);
  257. }
  258. else /* if (SizeF == 1) */
  259. {
  260. MthdInfo = typeof(Math).GetMethod(nameof(Math.Round), Types);
  261. }
  262. Context.EmitCall(MthdInfo);
  263. }
  264. public static void EmitUnarySoftFloatCall(AILEmitterCtx Context, string Name)
  265. {
  266. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  267. int SizeF = Op.Size & 1;
  268. MethodInfo MthdInfo;
  269. if (SizeF == 0)
  270. {
  271. MthdInfo = typeof(ASoftFloat).GetMethod(Name, new Type[] { typeof(float) });
  272. }
  273. else /* if (SizeF == 1) */
  274. {
  275. MthdInfo = typeof(ASoftFloat).GetMethod(Name, new Type[] { typeof(double) });
  276. }
  277. Context.EmitCall(MthdInfo);
  278. }
  279. public static void EmitBinarySoftFloatCall(AILEmitterCtx Context, string Name)
  280. {
  281. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  282. int SizeF = Op.Size & 1;
  283. MethodInfo MthdInfo;
  284. if (SizeF == 0)
  285. {
  286. MthdInfo = typeof(ASoftFloat).GetMethod(Name, new Type[] { typeof(float), typeof(float) });
  287. }
  288. else /* if (SizeF == 1) */
  289. {
  290. MthdInfo = typeof(ASoftFloat).GetMethod(Name, new Type[] { typeof(double), typeof(double) });
  291. }
  292. Context.EmitCall(MthdInfo);
  293. }
  294. public static void EmitScalarBinaryOpByElemF(AILEmitterCtx Context, Action Emit)
  295. {
  296. AOpCodeSimdRegElemF Op = (AOpCodeSimdRegElemF)Context.CurrOp;
  297. EmitScalarOpByElemF(Context, Emit, Op.Index, Ternary: false);
  298. }
  299. public static void EmitScalarTernaryOpByElemF(AILEmitterCtx Context, Action Emit)
  300. {
  301. AOpCodeSimdRegElemF Op = (AOpCodeSimdRegElemF)Context.CurrOp;
  302. EmitScalarOpByElemF(Context, Emit, Op.Index, Ternary: true);
  303. }
  304. public static void EmitScalarOpByElemF(AILEmitterCtx Context, Action Emit, int Elem, bool Ternary)
  305. {
  306. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  307. int SizeF = Op.Size & 1;
  308. if (Ternary)
  309. {
  310. EmitVectorExtractF(Context, Op.Rd, 0, SizeF);
  311. }
  312. EmitVectorExtractF(Context, Op.Rn, 0, SizeF);
  313. EmitVectorExtractF(Context, Op.Rm, Elem, SizeF);
  314. Emit();
  315. EmitScalarSetF(Context, Op.Rd, SizeF);
  316. }
  317. public static void EmitScalarUnaryOpSx(AILEmitterCtx Context, Action Emit)
  318. {
  319. EmitScalarOp(Context, Emit, OperFlags.Rn, true);
  320. }
  321. public static void EmitScalarBinaryOpSx(AILEmitterCtx Context, Action Emit)
  322. {
  323. EmitScalarOp(Context, Emit, OperFlags.RnRm, true);
  324. }
  325. public static void EmitScalarUnaryOpZx(AILEmitterCtx Context, Action Emit)
  326. {
  327. EmitScalarOp(Context, Emit, OperFlags.Rn, false);
  328. }
  329. public static void EmitScalarBinaryOpZx(AILEmitterCtx Context, Action Emit)
  330. {
  331. EmitScalarOp(Context, Emit, OperFlags.RnRm, false);
  332. }
  333. public static void EmitScalarTernaryOpZx(AILEmitterCtx Context, Action Emit)
  334. {
  335. EmitScalarOp(Context, Emit, OperFlags.RdRnRm, false);
  336. }
  337. public static void EmitScalarOp(AILEmitterCtx Context, Action Emit, OperFlags Opers, bool Signed)
  338. {
  339. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  340. bool Rd = (Opers & OperFlags.Rd) != 0;
  341. bool Rn = (Opers & OperFlags.Rn) != 0;
  342. bool Rm = (Opers & OperFlags.Rm) != 0;
  343. if (Rd)
  344. {
  345. EmitVectorExtract(Context, Op.Rd, 0, Op.Size, Signed);
  346. }
  347. if (Rn)
  348. {
  349. EmitVectorExtract(Context, Op.Rn, 0, Op.Size, Signed);
  350. }
  351. if (Rm)
  352. {
  353. EmitVectorExtract(Context, ((AOpCodeSimdReg)Op).Rm, 0, Op.Size, Signed);
  354. }
  355. Emit();
  356. EmitScalarSet(Context, Op.Rd, Op.Size);
  357. }
  358. public static void EmitScalarUnaryOpF(AILEmitterCtx Context, Action Emit)
  359. {
  360. EmitScalarOpF(Context, Emit, OperFlags.Rn);
  361. }
  362. public static void EmitScalarBinaryOpF(AILEmitterCtx Context, Action Emit)
  363. {
  364. EmitScalarOpF(Context, Emit, OperFlags.RnRm);
  365. }
  366. public static void EmitScalarTernaryRaOpF(AILEmitterCtx Context, Action Emit)
  367. {
  368. EmitScalarOpF(Context, Emit, OperFlags.RaRnRm);
  369. }
  370. public static void EmitScalarOpF(AILEmitterCtx Context, Action Emit, OperFlags Opers)
  371. {
  372. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  373. int SizeF = Op.Size & 1;
  374. bool Ra = (Opers & OperFlags.Ra) != 0;
  375. bool Rn = (Opers & OperFlags.Rn) != 0;
  376. bool Rm = (Opers & OperFlags.Rm) != 0;
  377. if (Ra)
  378. {
  379. EmitVectorExtractF(Context, ((AOpCodeSimdReg)Op).Ra, 0, SizeF);
  380. }
  381. if (Rn)
  382. {
  383. EmitVectorExtractF(Context, Op.Rn, 0, SizeF);
  384. }
  385. if (Rm)
  386. {
  387. EmitVectorExtractF(Context, ((AOpCodeSimdReg)Op).Rm, 0, SizeF);
  388. }
  389. Emit();
  390. EmitScalarSetF(Context, Op.Rd, SizeF);
  391. }
  392. public static void EmitVectorUnaryOpF(AILEmitterCtx Context, Action Emit)
  393. {
  394. EmitVectorOpF(Context, Emit, OperFlags.Rn);
  395. }
  396. public static void EmitVectorBinaryOpF(AILEmitterCtx Context, Action Emit)
  397. {
  398. EmitVectorOpF(Context, Emit, OperFlags.RnRm);
  399. }
  400. public static void EmitVectorTernaryOpF(AILEmitterCtx Context, Action Emit)
  401. {
  402. EmitVectorOpF(Context, Emit, OperFlags.RdRnRm);
  403. }
  404. public static void EmitVectorOpF(AILEmitterCtx Context, Action Emit, OperFlags Opers)
  405. {
  406. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  407. int SizeF = Op.Size & 1;
  408. int Bytes = Op.GetBitsCount() >> 3;
  409. int Elems = Bytes >> SizeF + 2;
  410. bool Rd = (Opers & OperFlags.Rd) != 0;
  411. bool Rn = (Opers & OperFlags.Rn) != 0;
  412. bool Rm = (Opers & OperFlags.Rm) != 0;
  413. for (int Index = 0; Index < Elems; Index++)
  414. {
  415. if (Rd)
  416. {
  417. EmitVectorExtractF(Context, Op.Rd, Index, SizeF);
  418. }
  419. if (Rn)
  420. {
  421. EmitVectorExtractF(Context, Op.Rn, Index, SizeF);
  422. }
  423. if (Rm)
  424. {
  425. EmitVectorExtractF(Context, ((AOpCodeSimdReg)Op).Rm, Index, SizeF);
  426. }
  427. Emit();
  428. EmitVectorInsertF(Context, Op.Rd, Index, SizeF);
  429. }
  430. if (Op.RegisterSize == ARegisterSize.SIMD64)
  431. {
  432. EmitVectorZeroUpper(Context, Op.Rd);
  433. }
  434. }
  435. public static void EmitVectorBinaryOpByElemF(AILEmitterCtx Context, Action Emit)
  436. {
  437. AOpCodeSimdRegElemF Op = (AOpCodeSimdRegElemF)Context.CurrOp;
  438. EmitVectorOpByElemF(Context, Emit, Op.Index, Ternary: false);
  439. }
  440. public static void EmitVectorTernaryOpByElemF(AILEmitterCtx Context, Action Emit)
  441. {
  442. AOpCodeSimdRegElemF Op = (AOpCodeSimdRegElemF)Context.CurrOp;
  443. EmitVectorOpByElemF(Context, Emit, Op.Index, Ternary: true);
  444. }
  445. public static void EmitVectorOpByElemF(AILEmitterCtx Context, Action Emit, int Elem, bool Ternary)
  446. {
  447. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  448. int SizeF = Op.Size & 1;
  449. int Bytes = Op.GetBitsCount() >> 3;
  450. int Elems = Bytes >> SizeF + 2;
  451. for (int Index = 0; Index < Elems; Index++)
  452. {
  453. if (Ternary)
  454. {
  455. EmitVectorExtractF(Context, Op.Rd, Index, SizeF);
  456. }
  457. EmitVectorExtractF(Context, Op.Rn, Index, SizeF);
  458. EmitVectorExtractF(Context, Op.Rm, Elem, SizeF);
  459. Emit();
  460. EmitVectorInsertTmpF(Context, Index, SizeF);
  461. }
  462. Context.EmitLdvectmp();
  463. Context.EmitStvec(Op.Rd);
  464. if (Op.RegisterSize == ARegisterSize.SIMD64)
  465. {
  466. EmitVectorZeroUpper(Context, Op.Rd);
  467. }
  468. }
  469. public static void EmitVectorUnaryOpSx(AILEmitterCtx Context, Action Emit)
  470. {
  471. EmitVectorOp(Context, Emit, OperFlags.Rn, true);
  472. }
  473. public static void EmitVectorBinaryOpSx(AILEmitterCtx Context, Action Emit)
  474. {
  475. EmitVectorOp(Context, Emit, OperFlags.RnRm, true);
  476. }
  477. public static void EmitVectorTernaryOpSx(AILEmitterCtx Context, Action Emit)
  478. {
  479. EmitVectorOp(Context, Emit, OperFlags.RdRnRm, true);
  480. }
  481. public static void EmitVectorUnaryOpZx(AILEmitterCtx Context, Action Emit)
  482. {
  483. EmitVectorOp(Context, Emit, OperFlags.Rn, false);
  484. }
  485. public static void EmitVectorBinaryOpZx(AILEmitterCtx Context, Action Emit)
  486. {
  487. EmitVectorOp(Context, Emit, OperFlags.RnRm, false);
  488. }
  489. public static void EmitVectorTernaryOpZx(AILEmitterCtx Context, Action Emit)
  490. {
  491. EmitVectorOp(Context, Emit, OperFlags.RdRnRm, false);
  492. }
  493. public static void EmitVectorOp(AILEmitterCtx Context, Action Emit, OperFlags Opers, bool Signed)
  494. {
  495. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  496. int Bytes = Op.GetBitsCount() >> 3;
  497. int Elems = Bytes >> Op.Size;
  498. bool Rd = (Opers & OperFlags.Rd) != 0;
  499. bool Rn = (Opers & OperFlags.Rn) != 0;
  500. bool Rm = (Opers & OperFlags.Rm) != 0;
  501. for (int Index = 0; Index < Elems; Index++)
  502. {
  503. if (Rd)
  504. {
  505. EmitVectorExtract(Context, Op.Rd, Index, Op.Size, Signed);
  506. }
  507. if (Rn)
  508. {
  509. EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
  510. }
  511. if (Rm)
  512. {
  513. EmitVectorExtract(Context, ((AOpCodeSimdReg)Op).Rm, Index, Op.Size, Signed);
  514. }
  515. Emit();
  516. EmitVectorInsert(Context, Op.Rd, Index, Op.Size);
  517. }
  518. if (Op.RegisterSize == ARegisterSize.SIMD64)
  519. {
  520. EmitVectorZeroUpper(Context, Op.Rd);
  521. }
  522. }
  523. public static void EmitVectorBinaryOpByElemSx(AILEmitterCtx Context, Action Emit)
  524. {
  525. AOpCodeSimdRegElem Op = (AOpCodeSimdRegElem)Context.CurrOp;
  526. EmitVectorOpByElem(Context, Emit, Op.Index, false, true);
  527. }
  528. public static void EmitVectorBinaryOpByElemZx(AILEmitterCtx Context, Action Emit)
  529. {
  530. AOpCodeSimdRegElem Op = (AOpCodeSimdRegElem)Context.CurrOp;
  531. EmitVectorOpByElem(Context, Emit, Op.Index, false, false);
  532. }
  533. public static void EmitVectorTernaryOpByElemZx(AILEmitterCtx Context, Action Emit)
  534. {
  535. AOpCodeSimdRegElem Op = (AOpCodeSimdRegElem)Context.CurrOp;
  536. EmitVectorOpByElem(Context, Emit, Op.Index, true, false);
  537. }
  538. public static void EmitVectorOpByElem(AILEmitterCtx Context, Action Emit, int Elem, bool Ternary, bool Signed)
  539. {
  540. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  541. int Bytes = Op.GetBitsCount() >> 3;
  542. int Elems = Bytes >> Op.Size;
  543. EmitVectorExtract(Context, Op.Rm, Elem, Op.Size, Signed);
  544. Context.EmitSttmp();
  545. for (int Index = 0; Index < Elems; Index++)
  546. {
  547. if (Ternary)
  548. {
  549. EmitVectorExtract(Context, Op.Rd, Index, Op.Size, Signed);
  550. }
  551. EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
  552. Context.EmitLdtmp();
  553. Emit();
  554. EmitVectorInsertTmp(Context, Index, Op.Size);
  555. }
  556. Context.EmitLdvectmp();
  557. Context.EmitStvec(Op.Rd);
  558. if (Op.RegisterSize == ARegisterSize.SIMD64)
  559. {
  560. EmitVectorZeroUpper(Context, Op.Rd);
  561. }
  562. }
  563. public static void EmitVectorImmUnaryOp(AILEmitterCtx Context, Action Emit)
  564. {
  565. EmitVectorImmOp(Context, Emit, false);
  566. }
  567. public static void EmitVectorImmBinaryOp(AILEmitterCtx Context, Action Emit)
  568. {
  569. EmitVectorImmOp(Context, Emit, true);
  570. }
  571. public static void EmitVectorImmOp(AILEmitterCtx Context, Action Emit, bool Binary)
  572. {
  573. AOpCodeSimdImm Op = (AOpCodeSimdImm)Context.CurrOp;
  574. int Bytes = Op.GetBitsCount() >> 3;
  575. int Elems = Bytes >> Op.Size;
  576. for (int Index = 0; Index < Elems; Index++)
  577. {
  578. if (Binary)
  579. {
  580. EmitVectorExtractZx(Context, Op.Rd, Index, Op.Size);
  581. }
  582. Context.EmitLdc_I8(Op.Imm);
  583. Emit();
  584. EmitVectorInsert(Context, Op.Rd, Index, Op.Size);
  585. }
  586. if (Op.RegisterSize == ARegisterSize.SIMD64)
  587. {
  588. EmitVectorZeroUpper(Context, Op.Rd);
  589. }
  590. }
  591. public static void EmitVectorWidenRmBinaryOpSx(AILEmitterCtx Context, Action Emit)
  592. {
  593. EmitVectorWidenRmBinaryOp(Context, Emit, true);
  594. }
  595. public static void EmitVectorWidenRmBinaryOpZx(AILEmitterCtx Context, Action Emit)
  596. {
  597. EmitVectorWidenRmBinaryOp(Context, Emit, false);
  598. }
  599. public static void EmitVectorWidenRmBinaryOp(AILEmitterCtx Context, Action Emit, bool Signed)
  600. {
  601. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  602. int Elems = 8 >> Op.Size;
  603. int Part = Op.RegisterSize == ARegisterSize.SIMD128 ? Elems : 0;
  604. for (int Index = 0; Index < Elems; Index++)
  605. {
  606. EmitVectorExtract(Context, Op.Rn, Index, Op.Size + 1, Signed);
  607. EmitVectorExtract(Context, Op.Rm, Part + Index, Op.Size, Signed);
  608. Emit();
  609. EmitVectorInsertTmp(Context, Index, Op.Size + 1);
  610. }
  611. Context.EmitLdvectmp();
  612. Context.EmitStvec(Op.Rd);
  613. }
  614. public static void EmitVectorWidenRnRmBinaryOpSx(AILEmitterCtx Context, Action Emit)
  615. {
  616. EmitVectorWidenRnRmOp(Context, Emit, false, true);
  617. }
  618. public static void EmitVectorWidenRnRmBinaryOpZx(AILEmitterCtx Context, Action Emit)
  619. {
  620. EmitVectorWidenRnRmOp(Context, Emit, false, false);
  621. }
  622. public static void EmitVectorWidenRnRmTernaryOpSx(AILEmitterCtx Context, Action Emit)
  623. {
  624. EmitVectorWidenRnRmOp(Context, Emit, true, true);
  625. }
  626. public static void EmitVectorWidenRnRmTernaryOpZx(AILEmitterCtx Context, Action Emit)
  627. {
  628. EmitVectorWidenRnRmOp(Context, Emit, true, false);
  629. }
  630. public static void EmitVectorWidenRnRmOp(AILEmitterCtx Context, Action Emit, bool Ternary, bool Signed)
  631. {
  632. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  633. int Elems = 8 >> Op.Size;
  634. int Part = Op.RegisterSize == ARegisterSize.SIMD128 ? Elems : 0;
  635. for (int Index = 0; Index < Elems; Index++)
  636. {
  637. if (Ternary)
  638. {
  639. EmitVectorExtract(Context, Op.Rd, Index, Op.Size + 1, Signed);
  640. }
  641. EmitVectorExtract(Context, Op.Rn, Part + Index, Op.Size, Signed);
  642. EmitVectorExtract(Context, Op.Rm, Part + Index, Op.Size, Signed);
  643. Emit();
  644. EmitVectorInsertTmp(Context, Index, Op.Size + 1);
  645. }
  646. Context.EmitLdvectmp();
  647. Context.EmitStvec(Op.Rd);
  648. }
  649. public static void EmitVectorPairwiseOpSx(AILEmitterCtx Context, Action Emit)
  650. {
  651. EmitVectorPairwiseOp(Context, Emit, true);
  652. }
  653. public static void EmitVectorPairwiseOpZx(AILEmitterCtx Context, Action Emit)
  654. {
  655. EmitVectorPairwiseOp(Context, Emit, false);
  656. }
  657. public static void EmitVectorPairwiseOp(AILEmitterCtx Context, Action Emit, bool Signed)
  658. {
  659. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  660. int Words = Op.GetBitsCount() >> 4;
  661. int Pairs = Words >> Op.Size;
  662. for (int Index = 0; Index < Pairs; Index++)
  663. {
  664. int Idx = Index << 1;
  665. EmitVectorExtract(Context, Op.Rn, Idx, Op.Size, Signed);
  666. EmitVectorExtract(Context, Op.Rn, Idx + 1, Op.Size, Signed);
  667. Emit();
  668. EmitVectorExtract(Context, Op.Rm, Idx, Op.Size, Signed);
  669. EmitVectorExtract(Context, Op.Rm, Idx + 1, Op.Size, Signed);
  670. Emit();
  671. EmitVectorInsertTmp(Context, Pairs + Index, Op.Size);
  672. EmitVectorInsertTmp(Context, Index, Op.Size);
  673. }
  674. Context.EmitLdvectmp();
  675. Context.EmitStvec(Op.Rd);
  676. if (Op.RegisterSize == ARegisterSize.SIMD64)
  677. {
  678. EmitVectorZeroUpper(Context, Op.Rd);
  679. }
  680. }
  681. public static void EmitVectorPairwiseOpF(AILEmitterCtx Context, Action Emit)
  682. {
  683. AOpCodeSimdReg Op = (AOpCodeSimdReg)Context.CurrOp;
  684. int SizeF = Op.Size & 1;
  685. int Words = Op.GetBitsCount() >> 4;
  686. int Pairs = Words >> SizeF + 2;
  687. for (int Index = 0; Index < Pairs; Index++)
  688. {
  689. int Idx = Index << 1;
  690. EmitVectorExtractF(Context, Op.Rn, Idx, SizeF);
  691. EmitVectorExtractF(Context, Op.Rn, Idx + 1, SizeF);
  692. Emit();
  693. EmitVectorExtractF(Context, Op.Rm, Idx, SizeF);
  694. EmitVectorExtractF(Context, Op.Rm, Idx + 1, SizeF);
  695. Emit();
  696. EmitVectorInsertTmpF(Context, Pairs + Index, SizeF);
  697. EmitVectorInsertTmpF(Context, Index, SizeF);
  698. }
  699. Context.EmitLdvectmp();
  700. Context.EmitStvec(Op.Rd);
  701. if (Op.RegisterSize == ARegisterSize.SIMD64)
  702. {
  703. EmitVectorZeroUpper(Context, Op.Rd);
  704. }
  705. }
  706. [Flags]
  707. public enum SaturatingFlags
  708. {
  709. Scalar = 1 << 0,
  710. Signed = 1 << 1,
  711. Add = 1 << 2,
  712. Sub = 1 << 3,
  713. Accumulate = 1 << 4,
  714. ScalarSx = Scalar | Signed,
  715. ScalarZx = Scalar,
  716. VectorSx = Signed,
  717. VectorZx = 0
  718. }
  719. public static void EmitScalarSaturatingUnaryOpSx(AILEmitterCtx Context, Action Emit)
  720. {
  721. EmitSaturatingUnaryOpSx(Context, Emit, SaturatingFlags.ScalarSx);
  722. }
  723. public static void EmitVectorSaturatingUnaryOpSx(AILEmitterCtx Context, Action Emit)
  724. {
  725. EmitSaturatingUnaryOpSx(Context, Emit, SaturatingFlags.VectorSx);
  726. }
  727. public static void EmitSaturatingUnaryOpSx(AILEmitterCtx Context, Action Emit, SaturatingFlags Flags)
  728. {
  729. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  730. bool Scalar = (Flags & SaturatingFlags.Scalar) != 0;
  731. int Bytes = Op.GetBitsCount() >> 3;
  732. int Elems = !Scalar ? Bytes >> Op.Size : 1;
  733. if (Scalar)
  734. {
  735. EmitVectorZeroLowerTmp(Context);
  736. }
  737. for (int Index = 0; Index < Elems; Index++)
  738. {
  739. EmitVectorExtractSx(Context, Op.Rn, Index, Op.Size);
  740. Emit();
  741. if (Op.Size <= 2)
  742. {
  743. EmitSatQ(Context, Op.Size, true, true);
  744. }
  745. else /* if (Op.Size == 3) */
  746. {
  747. EmitUnarySignedSatQAbsOrNeg(Context);
  748. }
  749. EmitVectorInsertTmp(Context, Index, Op.Size);
  750. }
  751. Context.EmitLdvectmp();
  752. Context.EmitStvec(Op.Rd);
  753. if ((Op.RegisterSize == ARegisterSize.SIMD64) || Scalar)
  754. {
  755. EmitVectorZeroUpper(Context, Op.Rd);
  756. }
  757. }
  758. public static void EmitScalarSaturatingBinaryOpSx(AILEmitterCtx Context, SaturatingFlags Flags)
  759. {
  760. EmitSaturatingBinaryOp(Context, () => { }, SaturatingFlags.ScalarSx | Flags);
  761. }
  762. public static void EmitScalarSaturatingBinaryOpZx(AILEmitterCtx Context, SaturatingFlags Flags)
  763. {
  764. EmitSaturatingBinaryOp(Context, () => { }, SaturatingFlags.ScalarZx | Flags);
  765. }
  766. public static void EmitVectorSaturatingBinaryOpSx(AILEmitterCtx Context, SaturatingFlags Flags)
  767. {
  768. EmitSaturatingBinaryOp(Context, () => { }, SaturatingFlags.VectorSx | Flags);
  769. }
  770. public static void EmitVectorSaturatingBinaryOpZx(AILEmitterCtx Context, SaturatingFlags Flags)
  771. {
  772. EmitSaturatingBinaryOp(Context, () => { }, SaturatingFlags.VectorZx | Flags);
  773. }
  774. public static void EmitSaturatingBinaryOp(AILEmitterCtx Context, Action Emit, SaturatingFlags Flags)
  775. {
  776. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  777. bool Scalar = (Flags & SaturatingFlags.Scalar) != 0;
  778. bool Signed = (Flags & SaturatingFlags.Signed) != 0;
  779. bool Add = (Flags & SaturatingFlags.Add) != 0;
  780. bool Sub = (Flags & SaturatingFlags.Sub) != 0;
  781. bool Accumulate = (Flags & SaturatingFlags.Accumulate) != 0;
  782. int Bytes = Op.GetBitsCount() >> 3;
  783. int Elems = !Scalar ? Bytes >> Op.Size : 1;
  784. if (Scalar)
  785. {
  786. EmitVectorZeroLowerTmp(Context);
  787. }
  788. if (Add || Sub)
  789. {
  790. for (int Index = 0; Index < Elems; Index++)
  791. {
  792. EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
  793. EmitVectorExtract(Context, ((AOpCodeSimdReg)Op).Rm, Index, Op.Size, Signed);
  794. if (Op.Size <= 2)
  795. {
  796. Context.Emit(Add ? OpCodes.Add : OpCodes.Sub);
  797. EmitSatQ(Context, Op.Size, true, Signed);
  798. }
  799. else /* if (Op.Size == 3) */
  800. {
  801. if (Add)
  802. {
  803. EmitBinarySatQAdd(Context, Signed);
  804. }
  805. else /* if (Sub) */
  806. {
  807. EmitBinarySatQSub(Context, Signed);
  808. }
  809. }
  810. EmitVectorInsertTmp(Context, Index, Op.Size);
  811. }
  812. }
  813. else if (Accumulate)
  814. {
  815. for (int Index = 0; Index < Elems; Index++)
  816. {
  817. EmitVectorExtract(Context, Op.Rn, Index, Op.Size, !Signed);
  818. EmitVectorExtract(Context, Op.Rd, Index, Op.Size, Signed);
  819. if (Op.Size <= 2)
  820. {
  821. Context.Emit(OpCodes.Add);
  822. EmitSatQ(Context, Op.Size, true, Signed);
  823. }
  824. else /* if (Op.Size == 3) */
  825. {
  826. EmitBinarySatQAccumulate(Context, Signed);
  827. }
  828. EmitVectorInsertTmp(Context, Index, Op.Size);
  829. }
  830. }
  831. else
  832. {
  833. for (int Index = 0; Index < Elems; Index++)
  834. {
  835. EmitVectorExtract(Context, Op.Rn, Index, Op.Size, Signed);
  836. EmitVectorExtract(Context, ((AOpCodeSimdReg)Op).Rm, Index, Op.Size, Signed);
  837. Emit();
  838. EmitSatQ(Context, Op.Size, true, Signed);
  839. EmitVectorInsertTmp(Context, Index, Op.Size);
  840. }
  841. }
  842. Context.EmitLdvectmp();
  843. Context.EmitStvec(Op.Rd);
  844. if ((Op.RegisterSize == ARegisterSize.SIMD64) || Scalar)
  845. {
  846. EmitVectorZeroUpper(Context, Op.Rd);
  847. }
  848. }
  849. [Flags]
  850. public enum SaturatingNarrowFlags
  851. {
  852. Scalar = 1 << 0,
  853. SignedSrc = 1 << 1,
  854. SignedDst = 1 << 2,
  855. ScalarSxSx = Scalar | SignedSrc | SignedDst,
  856. ScalarSxZx = Scalar | SignedSrc,
  857. ScalarZxZx = Scalar,
  858. VectorSxSx = SignedSrc | SignedDst,
  859. VectorSxZx = SignedSrc,
  860. VectorZxZx = 0
  861. }
  862. public static void EmitSaturatingNarrowOp(AILEmitterCtx Context, SaturatingNarrowFlags Flags)
  863. {
  864. AOpCodeSimd Op = (AOpCodeSimd)Context.CurrOp;
  865. bool Scalar = (Flags & SaturatingNarrowFlags.Scalar) != 0;
  866. bool SignedSrc = (Flags & SaturatingNarrowFlags.SignedSrc) != 0;
  867. bool SignedDst = (Flags & SaturatingNarrowFlags.SignedDst) != 0;
  868. int Elems = !Scalar ? 8 >> Op.Size : 1;
  869. int Part = !Scalar && (Op.RegisterSize == ARegisterSize.SIMD128) ? Elems : 0;
  870. if (Scalar)
  871. {
  872. EmitVectorZeroLowerTmp(Context);
  873. }
  874. if (Part != 0)
  875. {
  876. Context.EmitLdvec(Op.Rd);
  877. Context.EmitStvectmp();
  878. }
  879. for (int Index = 0; Index < Elems; Index++)
  880. {
  881. EmitVectorExtract(Context, Op.Rn, Index, Op.Size + 1, SignedSrc);
  882. EmitSatQ(Context, Op.Size, SignedSrc, SignedDst);
  883. EmitVectorInsertTmp(Context, Part + Index, Op.Size);
  884. }
  885. Context.EmitLdvectmp();
  886. Context.EmitStvec(Op.Rd);
  887. if (Part == 0)
  888. {
  889. EmitVectorZeroUpper(Context, Op.Rd);
  890. }
  891. }
  892. // TSrc (16bit, 32bit, 64bit; signed, unsigned) > TDst (8bit, 16bit, 32bit; signed, unsigned).
  893. public static void EmitSatQ(
  894. AILEmitterCtx Context,
  895. int SizeDst,
  896. bool SignedSrc,
  897. bool SignedDst)
  898. {
  899. if (SizeDst > 2)
  900. {
  901. throw new ArgumentOutOfRangeException(nameof(SizeDst));
  902. }
  903. Context.EmitLdc_I4(SizeDst);
  904. Context.EmitLdarg(ATranslatedSub.StateArgIdx);
  905. if (SignedSrc)
  906. {
  907. ASoftFallback.EmitCall(Context, SignedDst
  908. ? nameof(ASoftFallback.SignedSrcSignedDstSatQ)
  909. : nameof(ASoftFallback.SignedSrcUnsignedDstSatQ));
  910. }
  911. else
  912. {
  913. ASoftFallback.EmitCall(Context, SignedDst
  914. ? nameof(ASoftFallback.UnsignedSrcSignedDstSatQ)
  915. : nameof(ASoftFallback.UnsignedSrcUnsignedDstSatQ));
  916. }
  917. }
  918. // TSrc (64bit) == TDst (64bit); signed.
  919. public static void EmitUnarySignedSatQAbsOrNeg(AILEmitterCtx Context)
  920. {
  921. if (((AOpCodeSimd)Context.CurrOp).Size < 3)
  922. {
  923. throw new InvalidOperationException();
  924. }
  925. Context.EmitLdarg(ATranslatedSub.StateArgIdx);
  926. ASoftFallback.EmitCall(Context, nameof(ASoftFallback.UnarySignedSatQAbsOrNeg));
  927. }
  928. // TSrcs (64bit) == TDst (64bit); signed, unsigned.
  929. public static void EmitBinarySatQAdd(AILEmitterCtx Context, bool Signed)
  930. {
  931. if (((AOpCodeSimdReg)Context.CurrOp).Size < 3)
  932. {
  933. throw new InvalidOperationException();
  934. }
  935. Context.EmitLdarg(ATranslatedSub.StateArgIdx);
  936. ASoftFallback.EmitCall(Context, Signed
  937. ? nameof(ASoftFallback.BinarySignedSatQAdd)
  938. : nameof(ASoftFallback.BinaryUnsignedSatQAdd));
  939. }
  940. // TSrcs (64bit) == TDst (64bit); signed, unsigned.
  941. public static void EmitBinarySatQSub(AILEmitterCtx Context, bool Signed)
  942. {
  943. if (((AOpCodeSimdReg)Context.CurrOp).Size < 3)
  944. {
  945. throw new InvalidOperationException();
  946. }
  947. Context.EmitLdarg(ATranslatedSub.StateArgIdx);
  948. ASoftFallback.EmitCall(Context, Signed
  949. ? nameof(ASoftFallback.BinarySignedSatQSub)
  950. : nameof(ASoftFallback.BinaryUnsignedSatQSub));
  951. }
  952. // TSrcs (64bit) == TDst (64bit); signed, unsigned.
  953. public static void EmitBinarySatQAccumulate(AILEmitterCtx Context, bool Signed)
  954. {
  955. if (((AOpCodeSimd)Context.CurrOp).Size < 3)
  956. {
  957. throw new InvalidOperationException();
  958. }
  959. Context.EmitLdarg(ATranslatedSub.StateArgIdx);
  960. ASoftFallback.EmitCall(Context, Signed
  961. ? nameof(ASoftFallback.BinarySignedSatQAcc)
  962. : nameof(ASoftFallback.BinaryUnsignedSatQAcc));
  963. }
  964. public static void EmitScalarSet(AILEmitterCtx Context, int Reg, int Size)
  965. {
  966. EmitVectorZeroAll(Context, Reg);
  967. EmitVectorInsert(Context, Reg, 0, Size);
  968. }
  969. public static void EmitScalarSetF(AILEmitterCtx Context, int Reg, int Size)
  970. {
  971. if (AOptimizations.UseSse41 && Size == 0)
  972. {
  973. //If the type is float, we can perform insertion and
  974. //zero the upper bits with a single instruction (INSERTPS);
  975. Context.EmitLdvec(Reg);
  976. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.Sse41VectorInsertScalarSingle));
  977. Context.EmitStvec(Reg);
  978. }
  979. else
  980. {
  981. EmitVectorZeroAll(Context, Reg);
  982. EmitVectorInsertF(Context, Reg, 0, Size);
  983. }
  984. }
  985. public static void EmitVectorExtractSx(AILEmitterCtx Context, int Reg, int Index, int Size)
  986. {
  987. EmitVectorExtract(Context, Reg, Index, Size, true);
  988. }
  989. public static void EmitVectorExtractZx(AILEmitterCtx Context, int Reg, int Index, int Size)
  990. {
  991. EmitVectorExtract(Context, Reg, Index, Size, false);
  992. }
  993. public static void EmitVectorExtract(AILEmitterCtx Context, int Reg, int Index, int Size, bool Signed)
  994. {
  995. ThrowIfInvalid(Index, Size);
  996. IAOpCodeSimd Op = (IAOpCodeSimd)Context.CurrOp;
  997. Context.EmitLdvec(Reg);
  998. Context.EmitLdc_I4(Index);
  999. Context.EmitLdc_I4(Size);
  1000. AVectorHelper.EmitCall(Context, Signed
  1001. ? nameof(AVectorHelper.VectorExtractIntSx)
  1002. : nameof(AVectorHelper.VectorExtractIntZx));
  1003. }
  1004. public static void EmitVectorExtractF(AILEmitterCtx Context, int Reg, int Index, int Size)
  1005. {
  1006. ThrowIfInvalidF(Index, Size);
  1007. Context.EmitLdvec(Reg);
  1008. Context.EmitLdc_I4(Index);
  1009. if (Size == 0)
  1010. {
  1011. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorExtractSingle));
  1012. }
  1013. else if (Size == 1)
  1014. {
  1015. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorExtractDouble));
  1016. }
  1017. else
  1018. {
  1019. throw new ArgumentOutOfRangeException(nameof(Size));
  1020. }
  1021. }
  1022. public static void EmitVectorZeroAll(AILEmitterCtx Context, int Rd)
  1023. {
  1024. if (AOptimizations.UseSse2)
  1025. {
  1026. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorSingleZero));
  1027. Context.EmitStvec(Rd);
  1028. }
  1029. else
  1030. {
  1031. EmitVectorZeroLower(Context, Rd);
  1032. EmitVectorZeroUpper(Context, Rd);
  1033. }
  1034. }
  1035. public static void EmitVectorZeroLower(AILEmitterCtx Context, int Rd)
  1036. {
  1037. EmitVectorInsert(Context, Rd, 0, 3, 0);
  1038. }
  1039. public static void EmitVectorZeroLowerTmp(AILEmitterCtx Context)
  1040. {
  1041. EmitVectorInsertTmp(Context, 0, 3, 0);
  1042. }
  1043. public static void EmitVectorZeroUpper(AILEmitterCtx Context, int Reg)
  1044. {
  1045. if (AOptimizations.UseSse2)
  1046. {
  1047. //TODO: Use MoveScalar once it is fixed, as of the
  1048. //time of writing it just crashes the JIT.
  1049. EmitLdvecWithUnsignedCast(Context, Reg, 3);
  1050. Type[] Types = new Type[] { typeof(Vector128<ulong>), typeof(byte) };
  1051. //Context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.MoveScalar), Types));
  1052. Context.EmitLdc_I4(8);
  1053. Context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.ShiftLeftLogical128BitLane), Types));
  1054. Context.EmitLdc_I4(8);
  1055. Context.EmitCall(typeof(Sse2).GetMethod(nameof(Sse2.ShiftRightLogical128BitLane), Types));
  1056. EmitStvecWithUnsignedCast(Context, Reg, 3);
  1057. }
  1058. else
  1059. {
  1060. EmitVectorInsert(Context, Reg, 1, 3, 0);
  1061. }
  1062. }
  1063. public static void EmitVectorZero32_128(AILEmitterCtx Context, int Reg)
  1064. {
  1065. Context.EmitLdvec(Reg);
  1066. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorZero32_128));
  1067. Context.EmitStvec(Reg);
  1068. }
  1069. public static void EmitVectorInsert(AILEmitterCtx Context, int Reg, int Index, int Size)
  1070. {
  1071. ThrowIfInvalid(Index, Size);
  1072. Context.EmitLdvec(Reg);
  1073. Context.EmitLdc_I4(Index);
  1074. Context.EmitLdc_I4(Size);
  1075. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertInt));
  1076. Context.EmitStvec(Reg);
  1077. }
  1078. public static void EmitVectorInsertTmp(AILEmitterCtx Context, int Index, int Size)
  1079. {
  1080. ThrowIfInvalid(Index, Size);
  1081. Context.EmitLdvectmp();
  1082. Context.EmitLdc_I4(Index);
  1083. Context.EmitLdc_I4(Size);
  1084. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertInt));
  1085. Context.EmitStvectmp();
  1086. }
  1087. public static void EmitVectorInsert(AILEmitterCtx Context, int Reg, int Index, int Size, long Value)
  1088. {
  1089. ThrowIfInvalid(Index, Size);
  1090. Context.EmitLdc_I8(Value);
  1091. Context.EmitLdvec(Reg);
  1092. Context.EmitLdc_I4(Index);
  1093. Context.EmitLdc_I4(Size);
  1094. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertInt));
  1095. Context.EmitStvec(Reg);
  1096. }
  1097. public static void EmitVectorInsertTmp(AILEmitterCtx Context, int Index, int Size, long Value)
  1098. {
  1099. ThrowIfInvalid(Index, Size);
  1100. Context.EmitLdc_I8(Value);
  1101. Context.EmitLdvectmp();
  1102. Context.EmitLdc_I4(Index);
  1103. Context.EmitLdc_I4(Size);
  1104. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertInt));
  1105. Context.EmitStvectmp();
  1106. }
  1107. public static void EmitVectorInsertF(AILEmitterCtx Context, int Reg, int Index, int Size)
  1108. {
  1109. ThrowIfInvalidF(Index, Size);
  1110. Context.EmitLdvec(Reg);
  1111. Context.EmitLdc_I4(Index);
  1112. if (Size == 0)
  1113. {
  1114. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertSingle));
  1115. }
  1116. else if (Size == 1)
  1117. {
  1118. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertDouble));
  1119. }
  1120. else
  1121. {
  1122. throw new ArgumentOutOfRangeException(nameof(Size));
  1123. }
  1124. Context.EmitStvec(Reg);
  1125. }
  1126. public static void EmitVectorInsertTmpF(AILEmitterCtx Context, int Index, int Size)
  1127. {
  1128. ThrowIfInvalidF(Index, Size);
  1129. Context.EmitLdvectmp();
  1130. Context.EmitLdc_I4(Index);
  1131. if (Size == 0)
  1132. {
  1133. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertSingle));
  1134. }
  1135. else if (Size == 1)
  1136. {
  1137. AVectorHelper.EmitCall(Context, nameof(AVectorHelper.VectorInsertDouble));
  1138. }
  1139. else
  1140. {
  1141. throw new ArgumentOutOfRangeException(nameof(Size));
  1142. }
  1143. Context.EmitStvectmp();
  1144. }
  1145. private static void ThrowIfInvalid(int Index, int Size)
  1146. {
  1147. if ((uint)Size > 3)
  1148. {
  1149. throw new ArgumentOutOfRangeException(nameof(Size));
  1150. }
  1151. if ((uint)Index >= 16 >> Size)
  1152. {
  1153. throw new ArgumentOutOfRangeException(nameof(Index));
  1154. }
  1155. }
  1156. private static void ThrowIfInvalidF(int Index, int Size)
  1157. {
  1158. if ((uint)Size > 1)
  1159. {
  1160. throw new ArgumentOutOfRangeException(nameof(Size));
  1161. }
  1162. if ((uint)Index >= 4 >> Size)
  1163. {
  1164. throw new ArgumentOutOfRangeException(nameof(Index));
  1165. }
  1166. }
  1167. }
  1168. }