You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

436 line
12 KiB

  1. // Copyright 2016 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build 386,!gccgo,!appengine
  5. #include "textflag.h"
  6. DATA iv0<>+0x00(SB)/4, $0x6a09e667
  7. DATA iv0<>+0x04(SB)/4, $0xbb67ae85
  8. DATA iv0<>+0x08(SB)/4, $0x3c6ef372
  9. DATA iv0<>+0x0c(SB)/4, $0xa54ff53a
  10. GLOBL iv0<>(SB), (NOPTR+RODATA), $16
  11. DATA iv1<>+0x00(SB)/4, $0x510e527f
  12. DATA iv1<>+0x04(SB)/4, $0x9b05688c
  13. DATA iv1<>+0x08(SB)/4, $0x1f83d9ab
  14. DATA iv1<>+0x0c(SB)/4, $0x5be0cd19
  15. GLOBL iv1<>(SB), (NOPTR+RODATA), $16
  16. DATA rol16<>+0x00(SB)/8, $0x0504070601000302
  17. DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
  18. GLOBL rol16<>(SB), (NOPTR+RODATA), $16
  19. DATA rol8<>+0x00(SB)/8, $0x0407060500030201
  20. DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09
  21. GLOBL rol8<>(SB), (NOPTR+RODATA), $16
  22. DATA counter<>+0x00(SB)/8, $0x40
  23. DATA counter<>+0x08(SB)/8, $0x0
  24. GLOBL counter<>(SB), (NOPTR+RODATA), $16
  25. #define ROTL_SSE2(n, t, v) \
  26. MOVO v, t; \
  27. PSLLL $n, t; \
  28. PSRLL $(32-n), v; \
  29. PXOR t, v
  30. #define ROTL_SSSE3(c, v) \
  31. PSHUFB c, v
  32. #define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \
  33. PADDL m0, v0; \
  34. PADDL v1, v0; \
  35. PXOR v0, v3; \
  36. ROTL_SSE2(16, t, v3); \
  37. PADDL v3, v2; \
  38. PXOR v2, v1; \
  39. ROTL_SSE2(20, t, v1); \
  40. PADDL m1, v0; \
  41. PADDL v1, v0; \
  42. PXOR v0, v3; \
  43. ROTL_SSE2(24, t, v3); \
  44. PADDL v3, v2; \
  45. PXOR v2, v1; \
  46. ROTL_SSE2(25, t, v1); \
  47. PSHUFL $0x39, v1, v1; \
  48. PSHUFL $0x4E, v2, v2; \
  49. PSHUFL $0x93, v3, v3; \
  50. PADDL m2, v0; \
  51. PADDL v1, v0; \
  52. PXOR v0, v3; \
  53. ROTL_SSE2(16, t, v3); \
  54. PADDL v3, v2; \
  55. PXOR v2, v1; \
  56. ROTL_SSE2(20, t, v1); \
  57. PADDL m3, v0; \
  58. PADDL v1, v0; \
  59. PXOR v0, v3; \
  60. ROTL_SSE2(24, t, v3); \
  61. PADDL v3, v2; \
  62. PXOR v2, v1; \
  63. ROTL_SSE2(25, t, v1); \
  64. PSHUFL $0x39, v3, v3; \
  65. PSHUFL $0x4E, v2, v2; \
  66. PSHUFL $0x93, v1, v1
  67. #define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \
  68. PADDL m0, v0; \
  69. PADDL v1, v0; \
  70. PXOR v0, v3; \
  71. ROTL_SSSE3(c16, v3); \
  72. PADDL v3, v2; \
  73. PXOR v2, v1; \
  74. ROTL_SSE2(20, t, v1); \
  75. PADDL m1, v0; \
  76. PADDL v1, v0; \
  77. PXOR v0, v3; \
  78. ROTL_SSSE3(c8, v3); \
  79. PADDL v3, v2; \
  80. PXOR v2, v1; \
  81. ROTL_SSE2(25, t, v1); \
  82. PSHUFL $0x39, v1, v1; \
  83. PSHUFL $0x4E, v2, v2; \
  84. PSHUFL $0x93, v3, v3; \
  85. PADDL m2, v0; \
  86. PADDL v1, v0; \
  87. PXOR v0, v3; \
  88. ROTL_SSSE3(c16, v3); \
  89. PADDL v3, v2; \
  90. PXOR v2, v1; \
  91. ROTL_SSE2(20, t, v1); \
  92. PADDL m3, v0; \
  93. PADDL v1, v0; \
  94. PXOR v0, v3; \
  95. ROTL_SSSE3(c8, v3); \
  96. PADDL v3, v2; \
  97. PXOR v2, v1; \
  98. ROTL_SSE2(25, t, v1); \
  99. PSHUFL $0x39, v3, v3; \
  100. PSHUFL $0x4E, v2, v2; \
  101. PSHUFL $0x93, v1, v1
  102. #define PRECOMPUTE(dst, off, src, t) \
  103. MOVL 0*4(src), t; \
  104. MOVL t, 0*4+off+0(dst); \
  105. MOVL t, 9*4+off+64(dst); \
  106. MOVL t, 5*4+off+128(dst); \
  107. MOVL t, 14*4+off+192(dst); \
  108. MOVL t, 4*4+off+256(dst); \
  109. MOVL t, 2*4+off+320(dst); \
  110. MOVL t, 8*4+off+384(dst); \
  111. MOVL t, 12*4+off+448(dst); \
  112. MOVL t, 3*4+off+512(dst); \
  113. MOVL t, 15*4+off+576(dst); \
  114. MOVL 1*4(src), t; \
  115. MOVL t, 4*4+off+0(dst); \
  116. MOVL t, 8*4+off+64(dst); \
  117. MOVL t, 14*4+off+128(dst); \
  118. MOVL t, 5*4+off+192(dst); \
  119. MOVL t, 12*4+off+256(dst); \
  120. MOVL t, 11*4+off+320(dst); \
  121. MOVL t, 1*4+off+384(dst); \
  122. MOVL t, 6*4+off+448(dst); \
  123. MOVL t, 10*4+off+512(dst); \
  124. MOVL t, 3*4+off+576(dst); \
  125. MOVL 2*4(src), t; \
  126. MOVL t, 1*4+off+0(dst); \
  127. MOVL t, 13*4+off+64(dst); \
  128. MOVL t, 6*4+off+128(dst); \
  129. MOVL t, 8*4+off+192(dst); \
  130. MOVL t, 2*4+off+256(dst); \
  131. MOVL t, 0*4+off+320(dst); \
  132. MOVL t, 14*4+off+384(dst); \
  133. MOVL t, 11*4+off+448(dst); \
  134. MOVL t, 12*4+off+512(dst); \
  135. MOVL t, 4*4+off+576(dst); \
  136. MOVL 3*4(src), t; \
  137. MOVL t, 5*4+off+0(dst); \
  138. MOVL t, 15*4+off+64(dst); \
  139. MOVL t, 9*4+off+128(dst); \
  140. MOVL t, 1*4+off+192(dst); \
  141. MOVL t, 11*4+off+256(dst); \
  142. MOVL t, 7*4+off+320(dst); \
  143. MOVL t, 13*4+off+384(dst); \
  144. MOVL t, 3*4+off+448(dst); \
  145. MOVL t, 6*4+off+512(dst); \
  146. MOVL t, 10*4+off+576(dst); \
  147. MOVL 4*4(src), t; \
  148. MOVL t, 2*4+off+0(dst); \
  149. MOVL t, 1*4+off+64(dst); \
  150. MOVL t, 15*4+off+128(dst); \
  151. MOVL t, 10*4+off+192(dst); \
  152. MOVL t, 6*4+off+256(dst); \
  153. MOVL t, 8*4+off+320(dst); \
  154. MOVL t, 3*4+off+384(dst); \
  155. MOVL t, 13*4+off+448(dst); \
  156. MOVL t, 14*4+off+512(dst); \
  157. MOVL t, 5*4+off+576(dst); \
  158. MOVL 5*4(src), t; \
  159. MOVL t, 6*4+off+0(dst); \
  160. MOVL t, 11*4+off+64(dst); \
  161. MOVL t, 2*4+off+128(dst); \
  162. MOVL t, 9*4+off+192(dst); \
  163. MOVL t, 1*4+off+256(dst); \
  164. MOVL t, 13*4+off+320(dst); \
  165. MOVL t, 4*4+off+384(dst); \
  166. MOVL t, 8*4+off+448(dst); \
  167. MOVL t, 15*4+off+512(dst); \
  168. MOVL t, 7*4+off+576(dst); \
  169. MOVL 6*4(src), t; \
  170. MOVL t, 3*4+off+0(dst); \
  171. MOVL t, 7*4+off+64(dst); \
  172. MOVL t, 13*4+off+128(dst); \
  173. MOVL t, 12*4+off+192(dst); \
  174. MOVL t, 10*4+off+256(dst); \
  175. MOVL t, 1*4+off+320(dst); \
  176. MOVL t, 9*4+off+384(dst); \
  177. MOVL t, 14*4+off+448(dst); \
  178. MOVL t, 0*4+off+512(dst); \
  179. MOVL t, 6*4+off+576(dst); \
  180. MOVL 7*4(src), t; \
  181. MOVL t, 7*4+off+0(dst); \
  182. MOVL t, 14*4+off+64(dst); \
  183. MOVL t, 10*4+off+128(dst); \
  184. MOVL t, 0*4+off+192(dst); \
  185. MOVL t, 5*4+off+256(dst); \
  186. MOVL t, 9*4+off+320(dst); \
  187. MOVL t, 12*4+off+384(dst); \
  188. MOVL t, 1*4+off+448(dst); \
  189. MOVL t, 13*4+off+512(dst); \
  190. MOVL t, 2*4+off+576(dst); \
  191. MOVL 8*4(src), t; \
  192. MOVL t, 8*4+off+0(dst); \
  193. MOVL t, 5*4+off+64(dst); \
  194. MOVL t, 4*4+off+128(dst); \
  195. MOVL t, 15*4+off+192(dst); \
  196. MOVL t, 14*4+off+256(dst); \
  197. MOVL t, 3*4+off+320(dst); \
  198. MOVL t, 11*4+off+384(dst); \
  199. MOVL t, 10*4+off+448(dst); \
  200. MOVL t, 7*4+off+512(dst); \
  201. MOVL t, 1*4+off+576(dst); \
  202. MOVL 9*4(src), t; \
  203. MOVL t, 12*4+off+0(dst); \
  204. MOVL t, 2*4+off+64(dst); \
  205. MOVL t, 11*4+off+128(dst); \
  206. MOVL t, 4*4+off+192(dst); \
  207. MOVL t, 0*4+off+256(dst); \
  208. MOVL t, 15*4+off+320(dst); \
  209. MOVL t, 10*4+off+384(dst); \
  210. MOVL t, 7*4+off+448(dst); \
  211. MOVL t, 5*4+off+512(dst); \
  212. MOVL t, 9*4+off+576(dst); \
  213. MOVL 10*4(src), t; \
  214. MOVL t, 9*4+off+0(dst); \
  215. MOVL t, 4*4+off+64(dst); \
  216. MOVL t, 8*4+off+128(dst); \
  217. MOVL t, 13*4+off+192(dst); \
  218. MOVL t, 3*4+off+256(dst); \
  219. MOVL t, 5*4+off+320(dst); \
  220. MOVL t, 7*4+off+384(dst); \
  221. MOVL t, 15*4+off+448(dst); \
  222. MOVL t, 11*4+off+512(dst); \
  223. MOVL t, 0*4+off+576(dst); \
  224. MOVL 11*4(src), t; \
  225. MOVL t, 13*4+off+0(dst); \
  226. MOVL t, 10*4+off+64(dst); \
  227. MOVL t, 0*4+off+128(dst); \
  228. MOVL t, 3*4+off+192(dst); \
  229. MOVL t, 9*4+off+256(dst); \
  230. MOVL t, 6*4+off+320(dst); \
  231. MOVL t, 15*4+off+384(dst); \
  232. MOVL t, 4*4+off+448(dst); \
  233. MOVL t, 2*4+off+512(dst); \
  234. MOVL t, 12*4+off+576(dst); \
  235. MOVL 12*4(src), t; \
  236. MOVL t, 10*4+off+0(dst); \
  237. MOVL t, 12*4+off+64(dst); \
  238. MOVL t, 1*4+off+128(dst); \
  239. MOVL t, 6*4+off+192(dst); \
  240. MOVL t, 13*4+off+256(dst); \
  241. MOVL t, 4*4+off+320(dst); \
  242. MOVL t, 0*4+off+384(dst); \
  243. MOVL t, 2*4+off+448(dst); \
  244. MOVL t, 8*4+off+512(dst); \
  245. MOVL t, 14*4+off+576(dst); \
  246. MOVL 13*4(src), t; \
  247. MOVL t, 14*4+off+0(dst); \
  248. MOVL t, 3*4+off+64(dst); \
  249. MOVL t, 7*4+off+128(dst); \
  250. MOVL t, 2*4+off+192(dst); \
  251. MOVL t, 15*4+off+256(dst); \
  252. MOVL t, 12*4+off+320(dst); \
  253. MOVL t, 6*4+off+384(dst); \
  254. MOVL t, 0*4+off+448(dst); \
  255. MOVL t, 9*4+off+512(dst); \
  256. MOVL t, 11*4+off+576(dst); \
  257. MOVL 14*4(src), t; \
  258. MOVL t, 11*4+off+0(dst); \
  259. MOVL t, 0*4+off+64(dst); \
  260. MOVL t, 12*4+off+128(dst); \
  261. MOVL t, 7*4+off+192(dst); \
  262. MOVL t, 8*4+off+256(dst); \
  263. MOVL t, 14*4+off+320(dst); \
  264. MOVL t, 2*4+off+384(dst); \
  265. MOVL t, 5*4+off+448(dst); \
  266. MOVL t, 1*4+off+512(dst); \
  267. MOVL t, 13*4+off+576(dst); \
  268. MOVL 15*4(src), t; \
  269. MOVL t, 15*4+off+0(dst); \
  270. MOVL t, 6*4+off+64(dst); \
  271. MOVL t, 3*4+off+128(dst); \
  272. MOVL t, 11*4+off+192(dst); \
  273. MOVL t, 7*4+off+256(dst); \
  274. MOVL t, 10*4+off+320(dst); \
  275. MOVL t, 5*4+off+384(dst); \
  276. MOVL t, 9*4+off+448(dst); \
  277. MOVL t, 4*4+off+512(dst); \
  278. MOVL t, 8*4+off+576(dst)
  279. // func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
  280. TEXT ·hashBlocksSSE2(SB), 0, $672-24 // frame = 656 + 16 byte alignment
  281. MOVL h+0(FP), AX
  282. MOVL c+4(FP), BX
  283. MOVL flag+8(FP), CX
  284. MOVL blocks_base+12(FP), SI
  285. MOVL blocks_len+16(FP), DX
  286. MOVL SP, BP
  287. MOVL SP, DI
  288. ADDL $15, DI
  289. ANDL $~15, DI
  290. MOVL DI, SP
  291. MOVL CX, 8(SP)
  292. MOVL 0(BX), CX
  293. MOVL CX, 0(SP)
  294. MOVL 4(BX), CX
  295. MOVL CX, 4(SP)
  296. XORL CX, CX
  297. MOVL CX, 12(SP)
  298. MOVOU 0(AX), X0
  299. MOVOU 16(AX), X1
  300. MOVOU counter<>(SB), X2
  301. loop:
  302. MOVO X0, X4
  303. MOVO X1, X5
  304. MOVOU iv0<>(SB), X6
  305. MOVOU iv1<>(SB), X7
  306. MOVO 0(SP), X3
  307. PADDQ X2, X3
  308. PXOR X3, X7
  309. MOVO X3, 0(SP)
  310. PRECOMPUTE(SP, 16, SI, CX)
  311. ROUND_SSE2(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3)
  312. ROUND_SSE2(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3)
  313. ROUND_SSE2(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3)
  314. ROUND_SSE2(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3)
  315. ROUND_SSE2(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3)
  316. ROUND_SSE2(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3)
  317. ROUND_SSE2(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3)
  318. ROUND_SSE2(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3)
  319. ROUND_SSE2(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3)
  320. ROUND_SSE2(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3)
  321. PXOR X4, X0
  322. PXOR X5, X1
  323. PXOR X6, X0
  324. PXOR X7, X1
  325. LEAL 64(SI), SI
  326. SUBL $64, DX
  327. JNE loop
  328. MOVL 0(SP), CX
  329. MOVL CX, 0(BX)
  330. MOVL 4(SP), CX
  331. MOVL CX, 4(BX)
  332. MOVOU X0, 0(AX)
  333. MOVOU X1, 16(AX)
  334. MOVL BP, SP
  335. RET
  336. // func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
  337. TEXT ·hashBlocksSSSE3(SB), 0, $704-24 // frame = 688 + 16 byte alignment
  338. MOVL h+0(FP), AX
  339. MOVL c+4(FP), BX
  340. MOVL flag+8(FP), CX
  341. MOVL blocks_base+12(FP), SI
  342. MOVL blocks_len+16(FP), DX
  343. MOVL SP, BP
  344. MOVL SP, DI
  345. ADDL $15, DI
  346. ANDL $~15, DI
  347. MOVL DI, SP
  348. MOVL CX, 8(SP)
  349. MOVL 0(BX), CX
  350. MOVL CX, 0(SP)
  351. MOVL 4(BX), CX
  352. MOVL CX, 4(SP)
  353. XORL CX, CX
  354. MOVL CX, 12(SP)
  355. MOVOU 0(AX), X0
  356. MOVOU 16(AX), X1
  357. MOVOU counter<>(SB), X2
  358. loop:
  359. MOVO X0, 656(SP)
  360. MOVO X1, 672(SP)
  361. MOVO X0, X4
  362. MOVO X1, X5
  363. MOVOU iv0<>(SB), X6
  364. MOVOU iv1<>(SB), X7
  365. MOVO 0(SP), X3
  366. PADDQ X2, X3
  367. PXOR X3, X7
  368. MOVO X3, 0(SP)
  369. MOVOU rol16<>(SB), X0
  370. MOVOU rol8<>(SB), X1
  371. PRECOMPUTE(SP, 16, SI, CX)
  372. ROUND_SSSE3(X4, X5, X6, X7, 16(SP), 32(SP), 48(SP), 64(SP), X3, X0, X1)
  373. ROUND_SSSE3(X4, X5, X6, X7, 16+64(SP), 32+64(SP), 48+64(SP), 64+64(SP), X3, X0, X1)
  374. ROUND_SSSE3(X4, X5, X6, X7, 16+128(SP), 32+128(SP), 48+128(SP), 64+128(SP), X3, X0, X1)
  375. ROUND_SSSE3(X4, X5, X6, X7, 16+192(SP), 32+192(SP), 48+192(SP), 64+192(SP), X3, X0, X1)
  376. ROUND_SSSE3(X4, X5, X6, X7, 16+256(SP), 32+256(SP), 48+256(SP), 64+256(SP), X3, X0, X1)
  377. ROUND_SSSE3(X4, X5, X6, X7, 16+320(SP), 32+320(SP), 48+320(SP), 64+320(SP), X3, X0, X1)
  378. ROUND_SSSE3(X4, X5, X6, X7, 16+384(SP), 32+384(SP), 48+384(SP), 64+384(SP), X3, X0, X1)
  379. ROUND_SSSE3(X4, X5, X6, X7, 16+448(SP), 32+448(SP), 48+448(SP), 64+448(SP), X3, X0, X1)
  380. ROUND_SSSE3(X4, X5, X6, X7, 16+512(SP), 32+512(SP), 48+512(SP), 64+512(SP), X3, X0, X1)
  381. ROUND_SSSE3(X4, X5, X6, X7, 16+576(SP), 32+576(SP), 48+576(SP), 64+576(SP), X3, X0, X1)
  382. MOVO 656(SP), X0
  383. MOVO 672(SP), X1
  384. PXOR X4, X0
  385. PXOR X5, X1
  386. PXOR X6, X0
  387. PXOR X7, X1
  388. LEAL 64(SI), SI
  389. SUBL $64, DX
  390. JNE loop
  391. MOVL 0(SP), CX
  392. MOVL CX, 0(BX)
  393. MOVL 4(SP), CX
  394. MOVL CX, 4(BX)
  395. MOVOU X0, 0(AX)
  396. MOVOU X1, 16(AX)
  397. MOVL BP, SP
  398. RET