ASoftFallback.cs 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922
  1. using ChocolArm64.State;
  2. using ChocolArm64.Translation;
  3. using System;
  4. using System.Runtime.CompilerServices;
  5. using System.Runtime.Intrinsics;
  6. using System.Runtime.Intrinsics.X86;
  7. namespace ChocolArm64.Instruction
  8. {
  9. using static AVectorHelper;
  10. static class ASoftFallback
  11. {
  12. public static void EmitCall(AILEmitterCtx Context, string MthdName)
  13. {
  14. Context.EmitCall(typeof(ASoftFallback), MthdName);
  15. }
  16. #region "ShrImm_64"
  17. public static long SignedShrImm_64(long Value, long RoundConst, int Shift)
  18. {
  19. if (RoundConst == 0L)
  20. {
  21. if (Shift <= 63)
  22. {
  23. return Value >> Shift;
  24. }
  25. else /* if (Shift == 64) */
  26. {
  27. if (Value < 0L)
  28. {
  29. return -1L;
  30. }
  31. else
  32. {
  33. return 0L;
  34. }
  35. }
  36. }
  37. else /* if (RoundConst == 1L << (Shift - 1)) */
  38. {
  39. if (Shift <= 63)
  40. {
  41. long Add = Value + RoundConst;
  42. if ((~Value & (Value ^ Add)) < 0L)
  43. {
  44. return (long)((ulong)Add >> Shift);
  45. }
  46. else
  47. {
  48. return Add >> Shift;
  49. }
  50. }
  51. else /* if (Shift == 64) */
  52. {
  53. return 0L;
  54. }
  55. }
  56. }
  57. public static ulong UnsignedShrImm_64(ulong Value, long RoundConst, int Shift)
  58. {
  59. if (RoundConst == 0L)
  60. {
  61. if (Shift <= 63)
  62. {
  63. return Value >> Shift;
  64. }
  65. else /* if (Shift == 64) */
  66. {
  67. return 0UL;
  68. }
  69. }
  70. else /* if (RoundConst == 1L << (Shift - 1)) */
  71. {
  72. ulong Add = Value + (ulong)RoundConst;
  73. if ((Add < Value) && (Add < (ulong)RoundConst))
  74. {
  75. if (Shift <= 63)
  76. {
  77. return (Add >> Shift) | (0x8000000000000000UL >> (Shift - 1));
  78. }
  79. else /* if (Shift == 64) */
  80. {
  81. return 1UL;
  82. }
  83. }
  84. else
  85. {
  86. if (Shift <= 63)
  87. {
  88. return Add >> Shift;
  89. }
  90. else /* if (Shift == 64) */
  91. {
  92. return 0UL;
  93. }
  94. }
  95. }
  96. }
  97. #endregion
  98. #region "Saturating"
  99. public static long SignedSrcSignedDstSatQ(long Op, int Size, AThreadState State)
  100. {
  101. int ESize = 8 << Size;
  102. long TMaxValue = (1L << (ESize - 1)) - 1L;
  103. long TMinValue = -(1L << (ESize - 1));
  104. if (Op > TMaxValue)
  105. {
  106. State.SetFpsrFlag(FPSR.QC);
  107. return TMaxValue;
  108. }
  109. else if (Op < TMinValue)
  110. {
  111. State.SetFpsrFlag(FPSR.QC);
  112. return TMinValue;
  113. }
  114. else
  115. {
  116. return Op;
  117. }
  118. }
  119. public static ulong SignedSrcUnsignedDstSatQ(long Op, int Size, AThreadState State)
  120. {
  121. int ESize = 8 << Size;
  122. ulong TMaxValue = (1UL << ESize) - 1UL;
  123. ulong TMinValue = 0UL;
  124. if (Op > (long)TMaxValue)
  125. {
  126. State.SetFpsrFlag(FPSR.QC);
  127. return TMaxValue;
  128. }
  129. else if (Op < (long)TMinValue)
  130. {
  131. State.SetFpsrFlag(FPSR.QC);
  132. return TMinValue;
  133. }
  134. else
  135. {
  136. return (ulong)Op;
  137. }
  138. }
  139. public static long UnsignedSrcSignedDstSatQ(ulong Op, int Size, AThreadState State)
  140. {
  141. int ESize = 8 << Size;
  142. long TMaxValue = (1L << (ESize - 1)) - 1L;
  143. if (Op > (ulong)TMaxValue)
  144. {
  145. State.SetFpsrFlag(FPSR.QC);
  146. return TMaxValue;
  147. }
  148. else
  149. {
  150. return (long)Op;
  151. }
  152. }
  153. public static ulong UnsignedSrcUnsignedDstSatQ(ulong Op, int Size, AThreadState State)
  154. {
  155. int ESize = 8 << Size;
  156. ulong TMaxValue = (1UL << ESize) - 1UL;
  157. if (Op > TMaxValue)
  158. {
  159. State.SetFpsrFlag(FPSR.QC);
  160. return TMaxValue;
  161. }
  162. else
  163. {
  164. return Op;
  165. }
  166. }
  167. public static long UnarySignedSatQAbsOrNeg(long Op, AThreadState State)
  168. {
  169. if (Op == long.MinValue)
  170. {
  171. State.SetFpsrFlag(FPSR.QC);
  172. return long.MaxValue;
  173. }
  174. else
  175. {
  176. return Op;
  177. }
  178. }
  179. public static long BinarySignedSatQAdd(long Op1, long Op2, AThreadState State)
  180. {
  181. long Add = Op1 + Op2;
  182. if ((~(Op1 ^ Op2) & (Op1 ^ Add)) < 0L)
  183. {
  184. State.SetFpsrFlag(FPSR.QC);
  185. if (Op1 < 0L)
  186. {
  187. return long.MinValue;
  188. }
  189. else
  190. {
  191. return long.MaxValue;
  192. }
  193. }
  194. else
  195. {
  196. return Add;
  197. }
  198. }
  199. public static ulong BinaryUnsignedSatQAdd(ulong Op1, ulong Op2, AThreadState State)
  200. {
  201. ulong Add = Op1 + Op2;
  202. if ((Add < Op1) && (Add < Op2))
  203. {
  204. State.SetFpsrFlag(FPSR.QC);
  205. return ulong.MaxValue;
  206. }
  207. else
  208. {
  209. return Add;
  210. }
  211. }
  212. public static long BinarySignedSatQSub(long Op1, long Op2, AThreadState State)
  213. {
  214. long Sub = Op1 - Op2;
  215. if (((Op1 ^ Op2) & (Op1 ^ Sub)) < 0L)
  216. {
  217. State.SetFpsrFlag(FPSR.QC);
  218. if (Op1 < 0L)
  219. {
  220. return long.MinValue;
  221. }
  222. else
  223. {
  224. return long.MaxValue;
  225. }
  226. }
  227. else
  228. {
  229. return Sub;
  230. }
  231. }
  232. public static ulong BinaryUnsignedSatQSub(ulong Op1, ulong Op2, AThreadState State)
  233. {
  234. ulong Sub = Op1 - Op2;
  235. if (Op1 < Op2)
  236. {
  237. State.SetFpsrFlag(FPSR.QC);
  238. return ulong.MinValue;
  239. }
  240. else
  241. {
  242. return Sub;
  243. }
  244. }
  245. public static long BinarySignedSatQAcc(ulong Op1, long Op2, AThreadState State)
  246. {
  247. if (Op1 <= (ulong)long.MaxValue)
  248. {
  249. // Op1 from ulong.MinValue to (ulong)long.MaxValue
  250. // Op2 from long.MinValue to long.MaxValue
  251. long Add = (long)Op1 + Op2;
  252. if ((~Op2 & Add) < 0L)
  253. {
  254. State.SetFpsrFlag(FPSR.QC);
  255. return long.MaxValue;
  256. }
  257. else
  258. {
  259. return Add;
  260. }
  261. }
  262. else if (Op2 >= 0L)
  263. {
  264. // Op1 from (ulong)long.MaxValue + 1UL to ulong.MaxValue
  265. // Op2 from (long)ulong.MinValue to long.MaxValue
  266. State.SetFpsrFlag(FPSR.QC);
  267. return long.MaxValue;
  268. }
  269. else
  270. {
  271. // Op1 from (ulong)long.MaxValue + 1UL to ulong.MaxValue
  272. // Op2 from long.MinValue to (long)ulong.MinValue - 1L
  273. ulong Add = Op1 + (ulong)Op2;
  274. if (Add > (ulong)long.MaxValue)
  275. {
  276. State.SetFpsrFlag(FPSR.QC);
  277. return long.MaxValue;
  278. }
  279. else
  280. {
  281. return (long)Add;
  282. }
  283. }
  284. }
  285. public static ulong BinaryUnsignedSatQAcc(long Op1, ulong Op2, AThreadState State)
  286. {
  287. if (Op1 >= 0L)
  288. {
  289. // Op1 from (long)ulong.MinValue to long.MaxValue
  290. // Op2 from ulong.MinValue to ulong.MaxValue
  291. ulong Add = (ulong)Op1 + Op2;
  292. if ((Add < (ulong)Op1) && (Add < Op2))
  293. {
  294. State.SetFpsrFlag(FPSR.QC);
  295. return ulong.MaxValue;
  296. }
  297. else
  298. {
  299. return Add;
  300. }
  301. }
  302. else if (Op2 > (ulong)long.MaxValue)
  303. {
  304. // Op1 from long.MinValue to (long)ulong.MinValue - 1L
  305. // Op2 from (ulong)long.MaxValue + 1UL to ulong.MaxValue
  306. return (ulong)Op1 + Op2;
  307. }
  308. else
  309. {
  310. // Op1 from long.MinValue to (long)ulong.MinValue - 1L
  311. // Op2 from ulong.MinValue to (ulong)long.MaxValue
  312. long Add = Op1 + (long)Op2;
  313. if (Add < (long)ulong.MinValue)
  314. {
  315. State.SetFpsrFlag(FPSR.QC);
  316. return ulong.MinValue;
  317. }
  318. else
  319. {
  320. return (ulong)Add;
  321. }
  322. }
  323. }
  324. #endregion
  325. #region "Count"
  326. public static ulong CountLeadingSigns(ulong Value, int Size) // Size is 8, 16, 32 or 64 (SIMD&FP or Base Inst.).
  327. {
  328. Value ^= Value >> 1;
  329. int HighBit = Size - 2;
  330. for (int Bit = HighBit; Bit >= 0; Bit--)
  331. {
  332. if (((Value >> Bit) & 0b1) != 0)
  333. {
  334. return (ulong)(HighBit - Bit);
  335. }
  336. }
  337. return (ulong)(Size - 1);
  338. }
  339. private static readonly byte[] ClzNibbleTbl = { 4, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 };
  340. public static ulong CountLeadingZeros(ulong Value, int Size) // Size is 8, 16, 32 or 64 (SIMD&FP or Base Inst.).
  341. {
  342. if (Value == 0ul)
  343. {
  344. return (ulong)Size;
  345. }
  346. int NibbleIdx = Size;
  347. int PreCount, Count = 0;
  348. do
  349. {
  350. NibbleIdx -= 4;
  351. PreCount = ClzNibbleTbl[(Value >> NibbleIdx) & 0b1111];
  352. Count += PreCount;
  353. }
  354. while (PreCount == 4);
  355. return (ulong)Count;
  356. }
  357. public static ulong CountSetBits8(ulong Value) // "Size" is 8 (SIMD&FP Inst.).
  358. {
  359. if (Value == 0xfful)
  360. {
  361. return 8ul;
  362. }
  363. Value = ((Value >> 1) & 0x55ul) + (Value & 0x55ul);
  364. Value = ((Value >> 2) & 0x33ul) + (Value & 0x33ul);
  365. return (Value >> 4) + (Value & 0x0ful);
  366. }
  367. #endregion
  368. #region "Crc32"
  369. private const uint Crc32RevPoly = 0xedb88320;
  370. private const uint Crc32cRevPoly = 0x82f63b78;
  371. public static uint Crc32b(uint Crc, byte Val) => Crc32 (Crc, Crc32RevPoly, Val);
  372. public static uint Crc32h(uint Crc, ushort Val) => Crc32h(Crc, Crc32RevPoly, Val);
  373. public static uint Crc32w(uint Crc, uint Val) => Crc32w(Crc, Crc32RevPoly, Val);
  374. public static uint Crc32x(uint Crc, ulong Val) => Crc32x(Crc, Crc32RevPoly, Val);
  375. public static uint Crc32cb(uint Crc, byte Val) => Crc32 (Crc, Crc32cRevPoly, Val);
  376. public static uint Crc32ch(uint Crc, ushort Val) => Crc32h(Crc, Crc32cRevPoly, Val);
  377. public static uint Crc32cw(uint Crc, uint Val) => Crc32w(Crc, Crc32cRevPoly, Val);
  378. public static uint Crc32cx(uint Crc, ulong Val) => Crc32x(Crc, Crc32cRevPoly, Val);
  379. private static uint Crc32h(uint Crc, uint Poly, ushort Val)
  380. {
  381. Crc = Crc32(Crc, Poly, (byte)(Val >> 0));
  382. Crc = Crc32(Crc, Poly, (byte)(Val >> 8));
  383. return Crc;
  384. }
  385. private static uint Crc32w(uint Crc, uint Poly, uint Val)
  386. {
  387. Crc = Crc32(Crc, Poly, (byte)(Val >> 0 ));
  388. Crc = Crc32(Crc, Poly, (byte)(Val >> 8 ));
  389. Crc = Crc32(Crc, Poly, (byte)(Val >> 16));
  390. Crc = Crc32(Crc, Poly, (byte)(Val >> 24));
  391. return Crc;
  392. }
  393. private static uint Crc32x(uint Crc, uint Poly, ulong Val)
  394. {
  395. Crc = Crc32(Crc, Poly, (byte)(Val >> 0 ));
  396. Crc = Crc32(Crc, Poly, (byte)(Val >> 8 ));
  397. Crc = Crc32(Crc, Poly, (byte)(Val >> 16));
  398. Crc = Crc32(Crc, Poly, (byte)(Val >> 24));
  399. Crc = Crc32(Crc, Poly, (byte)(Val >> 32));
  400. Crc = Crc32(Crc, Poly, (byte)(Val >> 40));
  401. Crc = Crc32(Crc, Poly, (byte)(Val >> 48));
  402. Crc = Crc32(Crc, Poly, (byte)(Val >> 56));
  403. return Crc;
  404. }
  405. private static uint Crc32(uint Crc, uint Poly, byte Val)
  406. {
  407. Crc ^= Val;
  408. for (int Bit = 7; Bit >= 0; Bit--)
  409. {
  410. uint Mask = (uint)(-(int)(Crc & 1));
  411. Crc = (Crc >> 1) ^ (Poly & Mask);
  412. }
  413. return Crc;
  414. }
  415. #endregion
  416. #region "Aes"
  417. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  418. public static Vector128<float> Decrypt(Vector128<float> value, Vector128<float> roundKey)
  419. {
  420. if (!Sse.IsSupported)
  421. {
  422. throw new PlatformNotSupportedException();
  423. }
  424. return ACryptoHelper.AESInvSubBytes(ACryptoHelper.AESInvShiftRows(Sse.Xor(value, roundKey)));
  425. }
  426. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  427. public static Vector128<float> Encrypt(Vector128<float> value, Vector128<float> roundKey)
  428. {
  429. if (!Sse.IsSupported)
  430. {
  431. throw new PlatformNotSupportedException();
  432. }
  433. return ACryptoHelper.AESSubBytes(ACryptoHelper.AESShiftRows(Sse.Xor(value, roundKey)));
  434. }
  435. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  436. public static Vector128<float> InverseMixColumns(Vector128<float> value)
  437. {
  438. return ACryptoHelper.AESInvMixColumns(value);
  439. }
  440. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  441. public static Vector128<float> MixColumns(Vector128<float> value)
  442. {
  443. return ACryptoHelper.AESMixColumns(value);
  444. }
  445. #endregion
  446. #region "Sha1"
  447. public static Vector128<float> HashChoose(Vector128<float> hash_abcd, uint hash_e, Vector128<float> wk)
  448. {
  449. for (int e = 0; e <= 3; e++)
  450. {
  451. uint t = SHAchoose((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2),
  452. (uint)VectorExtractIntZx(hash_abcd, (byte)2, 2),
  453. (uint)VectorExtractIntZx(hash_abcd, (byte)3, 2));
  454. hash_e += Rol((uint)VectorExtractIntZx(hash_abcd, (byte)0, 2), 5) + t;
  455. hash_e += (uint)VectorExtractIntZx(wk, (byte)e, 2);
  456. t = Rol((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2), 30);
  457. hash_abcd = VectorInsertInt((ulong)t, hash_abcd, (byte)1, 2);
  458. Rol32_160(ref hash_e, ref hash_abcd);
  459. }
  460. return hash_abcd;
  461. }
  462. public static uint FixedRotate(uint hash_e)
  463. {
  464. return hash_e.Rol(30);
  465. }
  466. public static Vector128<float> HashMajority(Vector128<float> hash_abcd, uint hash_e, Vector128<float> wk)
  467. {
  468. for (int e = 0; e <= 3; e++)
  469. {
  470. uint t = SHAmajority((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2),
  471. (uint)VectorExtractIntZx(hash_abcd, (byte)2, 2),
  472. (uint)VectorExtractIntZx(hash_abcd, (byte)3, 2));
  473. hash_e += Rol((uint)VectorExtractIntZx(hash_abcd, (byte)0, 2), 5) + t;
  474. hash_e += (uint)VectorExtractIntZx(wk, (byte)e, 2);
  475. t = Rol((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2), 30);
  476. hash_abcd = VectorInsertInt((ulong)t, hash_abcd, (byte)1, 2);
  477. Rol32_160(ref hash_e, ref hash_abcd);
  478. }
  479. return hash_abcd;
  480. }
  481. public static Vector128<float> HashParity(Vector128<float> hash_abcd, uint hash_e, Vector128<float> wk)
  482. {
  483. for (int e = 0; e <= 3; e++)
  484. {
  485. uint t = SHAparity((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2),
  486. (uint)VectorExtractIntZx(hash_abcd, (byte)2, 2),
  487. (uint)VectorExtractIntZx(hash_abcd, (byte)3, 2));
  488. hash_e += Rol((uint)VectorExtractIntZx(hash_abcd, (byte)0, 2), 5) + t;
  489. hash_e += (uint)VectorExtractIntZx(wk, (byte)e, 2);
  490. t = Rol((uint)VectorExtractIntZx(hash_abcd, (byte)1, 2), 30);
  491. hash_abcd = VectorInsertInt((ulong)t, hash_abcd, (byte)1, 2);
  492. Rol32_160(ref hash_e, ref hash_abcd);
  493. }
  494. return hash_abcd;
  495. }
  496. public static Vector128<float> Sha1SchedulePart1(Vector128<float> w0_3, Vector128<float> w4_7, Vector128<float> w8_11)
  497. {
  498. if (!Sse.IsSupported)
  499. {
  500. throw new PlatformNotSupportedException();
  501. }
  502. Vector128<float> result = new Vector128<float>();
  503. ulong t2 = VectorExtractIntZx(w4_7, (byte)0, 3);
  504. ulong t1 = VectorExtractIntZx(w0_3, (byte)1, 3);
  505. result = VectorInsertInt((ulong)t1, result, (byte)0, 3);
  506. result = VectorInsertInt((ulong)t2, result, (byte)1, 3);
  507. return Sse.Xor(result, Sse.Xor(w0_3, w8_11));
  508. }
  509. public static Vector128<float> Sha1SchedulePart2(Vector128<float> tw0_3, Vector128<float> w12_15)
  510. {
  511. if (!Sse2.IsSupported)
  512. {
  513. throw new PlatformNotSupportedException();
  514. }
  515. Vector128<float> result = new Vector128<float>();
  516. Vector128<float> T = Sse.Xor(tw0_3, Sse.StaticCast<uint, float>(
  517. Sse2.ShiftRightLogical128BitLane(Sse.StaticCast<float, uint>(w12_15), (byte)4)));
  518. uint tE0 = (uint)VectorExtractIntZx(T, (byte)0, 2);
  519. uint tE1 = (uint)VectorExtractIntZx(T, (byte)1, 2);
  520. uint tE2 = (uint)VectorExtractIntZx(T, (byte)2, 2);
  521. uint tE3 = (uint)VectorExtractIntZx(T, (byte)3, 2);
  522. result = VectorInsertInt((ulong)tE0.Rol(1), result, (byte)0, 2);
  523. result = VectorInsertInt((ulong)tE1.Rol(1), result, (byte)1, 2);
  524. result = VectorInsertInt((ulong)tE2.Rol(1), result, (byte)2, 2);
  525. return VectorInsertInt((ulong)(tE3.Rol(1) ^ tE0.Rol(2)), result, (byte)3, 2);
  526. }
  527. private static void Rol32_160(ref uint y, ref Vector128<float> X)
  528. {
  529. if (!Sse2.IsSupported)
  530. {
  531. throw new PlatformNotSupportedException();
  532. }
  533. uint xE3 = (uint)VectorExtractIntZx(X, (byte)3, 2);
  534. X = Sse.StaticCast<uint, float>(Sse2.ShiftLeftLogical128BitLane(Sse.StaticCast<float, uint>(X), (byte)4));
  535. X = VectorInsertInt((ulong)y, X, (byte)0, 2);
  536. y = xE3;
  537. }
  538. private static uint SHAchoose(uint x, uint y, uint z)
  539. {
  540. return ((y ^ z) & x) ^ z;
  541. }
  542. private static uint SHAmajority(uint x, uint y, uint z)
  543. {
  544. return (x & y) | ((x | y) & z);
  545. }
  546. private static uint SHAparity(uint x, uint y, uint z)
  547. {
  548. return x ^ y ^ z;
  549. }
  550. private static uint Rol(this uint value, int count)
  551. {
  552. return (value << count) | (value >> (32 - count));
  553. }
  554. #endregion
  555. #region "Sha256"
  556. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  557. public static Vector128<float> HashLower(Vector128<float> hash_abcd, Vector128<float> hash_efgh, Vector128<float> wk)
  558. {
  559. return SHA256hash(hash_abcd, hash_efgh, wk, true);
  560. }
  561. [MethodImpl(MethodImplOptions.AggressiveInlining)]
  562. public static Vector128<float> HashUpper(Vector128<float> hash_efgh, Vector128<float> hash_abcd, Vector128<float> wk)
  563. {
  564. return SHA256hash(hash_abcd, hash_efgh, wk, false);
  565. }
  566. public static Vector128<float> Sha256SchedulePart1(Vector128<float> w0_3, Vector128<float> w4_7)
  567. {
  568. Vector128<float> result = new Vector128<float>();
  569. for (int e = 0; e <= 3; e++)
  570. {
  571. uint elt = (uint)VectorExtractIntZx(e <= 2 ? w0_3 : w4_7, (byte)(e <= 2 ? e + 1 : 0), 2);
  572. elt = elt.Ror(7) ^ elt.Ror(18) ^ elt.Lsr(3);
  573. elt += (uint)VectorExtractIntZx(w0_3, (byte)e, 2);
  574. result = VectorInsertInt((ulong)elt, result, (byte)e, 2);
  575. }
  576. return result;
  577. }
  578. public static Vector128<float> Sha256SchedulePart2(Vector128<float> w0_3, Vector128<float> w8_11, Vector128<float> w12_15)
  579. {
  580. Vector128<float> result = new Vector128<float>();
  581. ulong T1 = VectorExtractIntZx(w12_15, (byte)1, 3);
  582. for (int e = 0; e <= 1; e++)
  583. {
  584. uint elt = T1.ULongPart(e);
  585. elt = elt.Ror(17) ^ elt.Ror(19) ^ elt.Lsr(10);
  586. elt += (uint)VectorExtractIntZx(w0_3, (byte)e, 2);
  587. elt += (uint)VectorExtractIntZx(w8_11, (byte)(e + 1), 2);
  588. result = VectorInsertInt((ulong)elt, result, (byte)e, 2);
  589. }
  590. T1 = VectorExtractIntZx(result, (byte)0, 3);
  591. for (int e = 2; e <= 3; e++)
  592. {
  593. uint elt = T1.ULongPart(e - 2);
  594. elt = elt.Ror(17) ^ elt.Ror(19) ^ elt.Lsr(10);
  595. elt += (uint)VectorExtractIntZx(w0_3, (byte)e, 2);
  596. elt += (uint)VectorExtractIntZx(e == 2 ? w8_11 : w12_15, (byte)(e == 2 ? 3 : 0), 2);
  597. result = VectorInsertInt((ulong)elt, result, (byte)e, 2);
  598. }
  599. return result;
  600. }
  601. private static Vector128<float> SHA256hash(Vector128<float> X, Vector128<float> Y, Vector128<float> W, bool part1)
  602. {
  603. for (int e = 0; e <= 3; e++)
  604. {
  605. uint chs = SHAchoose((uint)VectorExtractIntZx(Y, (byte)0, 2),
  606. (uint)VectorExtractIntZx(Y, (byte)1, 2),
  607. (uint)VectorExtractIntZx(Y, (byte)2, 2));
  608. uint maj = SHAmajority((uint)VectorExtractIntZx(X, (byte)0, 2),
  609. (uint)VectorExtractIntZx(X, (byte)1, 2),
  610. (uint)VectorExtractIntZx(X, (byte)2, 2));
  611. uint t1 = (uint)VectorExtractIntZx(Y, (byte)3, 2);
  612. t1 += SHAhashSIGMA1((uint)VectorExtractIntZx(Y, (byte)0, 2)) + chs;
  613. t1 += (uint)VectorExtractIntZx(W, (byte)e, 2);
  614. uint t2 = t1 + (uint)VectorExtractIntZx(X, (byte)3, 2);
  615. X = VectorInsertInt((ulong)t2, X, (byte)3, 2);
  616. t2 = t1 + SHAhashSIGMA0((uint)VectorExtractIntZx(X, (byte)0, 2)) + maj;
  617. Y = VectorInsertInt((ulong)t2, Y, (byte)3, 2);
  618. Rol32_256(ref Y, ref X);
  619. }
  620. return part1 ? X : Y;
  621. }
  622. private static void Rol32_256(ref Vector128<float> Y, ref Vector128<float> X)
  623. {
  624. if (!Sse2.IsSupported)
  625. {
  626. throw new PlatformNotSupportedException();
  627. }
  628. uint yE3 = (uint)VectorExtractIntZx(Y, (byte)3, 2);
  629. uint xE3 = (uint)VectorExtractIntZx(X, (byte)3, 2);
  630. Y = Sse.StaticCast<uint, float>(Sse2.ShiftLeftLogical128BitLane(Sse.StaticCast<float, uint>(Y), (byte)4));
  631. X = Sse.StaticCast<uint, float>(Sse2.ShiftLeftLogical128BitLane(Sse.StaticCast<float, uint>(X), (byte)4));
  632. Y = VectorInsertInt((ulong)xE3, Y, (byte)0, 2);
  633. X = VectorInsertInt((ulong)yE3, X, (byte)0, 2);
  634. }
  635. private static uint SHAhashSIGMA0(uint x)
  636. {
  637. return x.Ror(2) ^ x.Ror(13) ^ x.Ror(22);
  638. }
  639. private static uint SHAhashSIGMA1(uint x)
  640. {
  641. return x.Ror(6) ^ x.Ror(11) ^ x.Ror(25);
  642. }
  643. private static uint Ror(this uint value, int count)
  644. {
  645. return (value >> count) | (value << (32 - count));
  646. }
  647. private static uint Lsr(this uint value, int count)
  648. {
  649. return value >> count;
  650. }
  651. private static uint ULongPart(this ulong value, int part)
  652. {
  653. return part == 0
  654. ? (uint)(value & 0xFFFFFFFFUL)
  655. : (uint)(value >> 32);
  656. }
  657. #endregion
  658. #region "Reverse"
  659. public static uint ReverseBits8(uint Value)
  660. {
  661. Value = ((Value & 0xaa) >> 1) | ((Value & 0x55) << 1);
  662. Value = ((Value & 0xcc) >> 2) | ((Value & 0x33) << 2);
  663. return (Value >> 4) | ((Value & 0x0f) << 4);
  664. }
  665. public static uint ReverseBits32(uint Value)
  666. {
  667. Value = ((Value & 0xaaaaaaaa) >> 1) | ((Value & 0x55555555) << 1);
  668. Value = ((Value & 0xcccccccc) >> 2) | ((Value & 0x33333333) << 2);
  669. Value = ((Value & 0xf0f0f0f0) >> 4) | ((Value & 0x0f0f0f0f) << 4);
  670. Value = ((Value & 0xff00ff00) >> 8) | ((Value & 0x00ff00ff) << 8);
  671. return (Value >> 16) | (Value << 16);
  672. }
  673. public static ulong ReverseBits64(ulong Value)
  674. {
  675. Value = ((Value & 0xaaaaaaaaaaaaaaaa) >> 1 ) | ((Value & 0x5555555555555555) << 1 );
  676. Value = ((Value & 0xcccccccccccccccc) >> 2 ) | ((Value & 0x3333333333333333) << 2 );
  677. Value = ((Value & 0xf0f0f0f0f0f0f0f0) >> 4 ) | ((Value & 0x0f0f0f0f0f0f0f0f) << 4 );
  678. Value = ((Value & 0xff00ff00ff00ff00) >> 8 ) | ((Value & 0x00ff00ff00ff00ff) << 8 );
  679. Value = ((Value & 0xffff0000ffff0000) >> 16) | ((Value & 0x0000ffff0000ffff) << 16);
  680. return (Value >> 32) | (Value << 32);
  681. }
  682. public static uint ReverseBytes16_32(uint Value) => (uint)ReverseBytes16_64(Value);
  683. public static uint ReverseBytes32_32(uint Value) => (uint)ReverseBytes32_64(Value);
  684. public static ulong ReverseBytes16_64(ulong Value) => ReverseBytes(Value, RevSize.Rev16);
  685. public static ulong ReverseBytes32_64(ulong Value) => ReverseBytes(Value, RevSize.Rev32);
  686. public static ulong ReverseBytes64(ulong Value) => ReverseBytes(Value, RevSize.Rev64);
  687. private enum RevSize
  688. {
  689. Rev16,
  690. Rev32,
  691. Rev64
  692. }
  693. private static ulong ReverseBytes(ulong Value, RevSize Size)
  694. {
  695. Value = ((Value & 0xff00ff00ff00ff00) >> 8) | ((Value & 0x00ff00ff00ff00ff) << 8);
  696. if (Size == RevSize.Rev16)
  697. {
  698. return Value;
  699. }
  700. Value = ((Value & 0xffff0000ffff0000) >> 16) | ((Value & 0x0000ffff0000ffff) << 16);
  701. if (Size == RevSize.Rev32)
  702. {
  703. return Value;
  704. }
  705. Value = ((Value & 0xffffffff00000000) >> 32) | ((Value & 0x00000000ffffffff) << 32);
  706. if (Size == RevSize.Rev64)
  707. {
  708. return Value;
  709. }
  710. throw new ArgumentException(nameof(Size));
  711. }
  712. #endregion
  713. #region "MultiplyHigh"
  714. public static long SMulHi128(long Left, long Right)
  715. {
  716. long Result = (long)UMulHi128((ulong)Left, (ulong)Right);
  717. if (Left < 0)
  718. {
  719. Result -= Right;
  720. }
  721. if (Right < 0)
  722. {
  723. Result -= Left;
  724. }
  725. return Result;
  726. }
  727. public static ulong UMulHi128(ulong Left, ulong Right)
  728. {
  729. ulong LHigh = Left >> 32;
  730. ulong LLow = Left & 0xFFFFFFFF;
  731. ulong RHigh = Right >> 32;
  732. ulong RLow = Right & 0xFFFFFFFF;
  733. ulong Z2 = LLow * RLow;
  734. ulong T = LHigh * RLow + (Z2 >> 32);
  735. ulong Z1 = T & 0xFFFFFFFF;
  736. ulong Z0 = T >> 32;
  737. Z1 += LLow * RHigh;
  738. return LHigh * RHigh + Z0 + (Z1 >> 32);
  739. }
  740. #endregion
  741. }
  742. }