31 #if !defined(POLARSSL_CONFIG_FILE)
34 #include POLARSSL_CONFIG_FILE
37 #if defined(POLARSSL_AESNI_C)
42 #if defined(POLARSSL_HAVE_X86_64)
47 int aesni_supports(
unsigned int what )
50 static unsigned int c = 0;
54 asm(
"movl $1, %%eax \n"
58 :
"eax",
"ebx",
"edx" );
62 return( ( c & what ) != 0 );
75 #define AESDEC ".byte 0x66,0x0F,0x38,0xDE,"
76 #define AESDECLAST ".byte 0x66,0x0F,0x38,0xDF,"
77 #define AESENC ".byte 0x66,0x0F,0x38,0xDC,"
78 #define AESENCLAST ".byte 0x66,0x0F,0x38,0xDD,"
79 #define AESIMC ".byte 0x66,0x0F,0x38,0xDB,"
80 #define AESKEYGENA ".byte 0x66,0x0F,0x3A,0xDF,"
81 #define PCLMULQDQ ".byte 0x66,0x0F,0x3A,0x44,"
83 #define xmm0_xmm0 "0xC0"
84 #define xmm0_xmm1 "0xC8"
85 #define xmm0_xmm2 "0xD0"
86 #define xmm0_xmm3 "0xD8"
87 #define xmm0_xmm4 "0xE0"
88 #define xmm1_xmm0 "0xC1"
89 #define xmm1_xmm2 "0xD1"
96 const unsigned char input[16],
97 unsigned char output[16] )
99 asm(
"movdqu (%3), %%xmm0 \n"
100 "movdqu (%1), %%xmm1 \n"
101 "pxor %%xmm1, %%xmm0 \n"
108 "movdqu (%1), %%xmm1 \n"
109 AESENC xmm1_xmm0
"\n"
113 "movdqu (%1), %%xmm1 \n"
114 AESENCLAST xmm1_xmm0
"\n"
118 "movdqu (%1), %%xmm1 \n"
119 AESDEC xmm1_xmm0
"\n"
123 "movdqu (%1), %%xmm1 \n"
124 AESDECLAST xmm1_xmm0
"\n"
127 "movdqu %%xmm0, (%4) \n"
129 :
"r" (ctx->
nr),
"r" (ctx->
rk),
"r" (mode),
"r" (input),
"r" (output)
130 :
"memory",
"cc",
"xmm0",
"xmm1" );
140 void aesni_gcm_mult(
unsigned char c[16],
141 const unsigned char a[16],
142 const unsigned char b[16] )
144 unsigned char aa[16], bb[16], cc[16];
148 for( i = 0; i < 16; i++ )
154 asm(
"movdqu (%0), %%xmm0 \n"
155 "movdqu (%1), %%xmm1 \n"
161 "movdqa %%xmm1, %%xmm2 \n"
162 "movdqa %%xmm1, %%xmm3 \n"
163 "movdqa %%xmm1, %%xmm4 \n"
164 PCLMULQDQ xmm0_xmm1
",0x00 \n"
165 PCLMULQDQ xmm0_xmm2
",0x11 \n"
166 PCLMULQDQ xmm0_xmm3
",0x10 \n"
167 PCLMULQDQ xmm0_xmm4
",0x01 \n"
168 "pxor %%xmm3, %%xmm4 \n"
169 "movdqa %%xmm4, %%xmm3 \n"
170 "psrldq $8, %%xmm4 \n"
171 "pslldq $8, %%xmm3 \n"
172 "pxor %%xmm4, %%xmm2 \n"
173 "pxor %%xmm3, %%xmm1 \n"
179 "movdqa %%xmm1, %%xmm3 \n"
180 "movdqa %%xmm2, %%xmm4 \n"
181 "psllq $1, %%xmm1 \n"
182 "psllq $1, %%xmm2 \n"
183 "psrlq $63, %%xmm3 \n"
184 "psrlq $63, %%xmm4 \n"
185 "movdqa %%xmm3, %%xmm5 \n"
186 "pslldq $8, %%xmm3 \n"
187 "pslldq $8, %%xmm4 \n"
188 "psrldq $8, %%xmm5 \n"
189 "por %%xmm3, %%xmm1 \n"
190 "por %%xmm4, %%xmm2 \n"
191 "por %%xmm5, %%xmm2 \n"
199 "movdqa %%xmm1, %%xmm3 \n"
200 "movdqa %%xmm1, %%xmm4 \n"
201 "movdqa %%xmm1, %%xmm5 \n"
202 "psllq $63, %%xmm3 \n"
203 "psllq $62, %%xmm4 \n"
204 "psllq $57, %%xmm5 \n"
207 "pxor %%xmm4, %%xmm3 \n"
208 "pxor %%xmm5, %%xmm3 \n"
209 "pslldq $8, %%xmm3 \n"
210 "pxor %%xmm3, %%xmm1 \n"
213 "movdqa %%xmm1,%%xmm0 \n"
214 "movdqa %%xmm1,%%xmm4 \n"
215 "movdqa %%xmm1,%%xmm5 \n"
216 "psrlq $1, %%xmm0 \n"
217 "psrlq $2, %%xmm4 \n"
218 "psrlq $7, %%xmm5 \n"
219 "pxor %%xmm4, %%xmm0 \n"
220 "pxor %%xmm5, %%xmm0 \n"
223 "movdqa %%xmm1,%%xmm3 \n"
224 "movdqa %%xmm1,%%xmm4 \n"
225 "movdqa %%xmm1,%%xmm5 \n"
226 "psllq $63, %%xmm3 \n"
227 "psllq $62, %%xmm4 \n"
228 "psllq $57, %%xmm5 \n"
229 "pxor %%xmm4, %%xmm3 \n"
230 "pxor %%xmm5, %%xmm3 \n"
231 "psrldq $8, %%xmm3 \n"
232 "pxor %%xmm3, %%xmm0 \n"
233 "pxor %%xmm1, %%xmm0 \n"
234 "pxor %%xmm2, %%xmm0 \n"
236 "movdqu %%xmm0, (%2) \n"
238 :
"r" (aa),
"r" (bb),
"r" (cc)
239 :
"memory",
"cc",
"xmm0",
"xmm1",
"xmm2",
"xmm3",
"xmm4",
"xmm5" );
242 for( i = 0; i < 16; i++ )
251 void aesni_inverse_key(
unsigned char *invkey,
252 const unsigned char *fwdkey,
int nr )
254 unsigned char *ik = invkey;
255 const unsigned char *fk = fwdkey + 16 * nr;
257 memcpy( ik, fk, 16 );
259 for( fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16 )
260 asm(
"movdqu (%0), %%xmm0 \n"
261 AESIMC xmm0_xmm0
"\n"
262 "movdqu %%xmm0, (%1) \n"
265 :
"memory",
"xmm0" );
267 memcpy( ik, fk, 16 );
273 static void aesni_setkey_enc_128(
unsigned char *rk,
274 const unsigned char *key )
276 asm(
"movdqu (%1), %%xmm0 \n"
277 "movdqu %%xmm0, (%0) \n"
291 "pshufd $0xff, %%xmm1, %%xmm1 \n"
292 "pxor %%xmm0, %%xmm1 \n"
293 "pslldq $4, %%xmm0 \n"
294 "pxor %%xmm0, %%xmm1 \n"
295 "pslldq $4, %%xmm0 \n"
296 "pxor %%xmm0, %%xmm1 \n"
297 "pslldq $4, %%xmm0 \n"
298 "pxor %%xmm1, %%xmm0 \n"
300 "movdqu %%xmm0, (%0) \n"
305 AESKEYGENA xmm0_xmm1
",0x01 \ncall 1b \n"
306 AESKEYGENA xmm0_xmm1
",0x02 \ncall 1b \n"
307 AESKEYGENA xmm0_xmm1
",0x04 \ncall 1b \n"
308 AESKEYGENA xmm0_xmm1
",0x08 \ncall 1b \n"
309 AESKEYGENA xmm0_xmm1
",0x10 \ncall 1b \n"
310 AESKEYGENA xmm0_xmm1
",0x20 \ncall 1b \n"
311 AESKEYGENA xmm0_xmm1
",0x40 \ncall 1b \n"
312 AESKEYGENA xmm0_xmm1
",0x80 \ncall 1b \n"
313 AESKEYGENA xmm0_xmm1
",0x1B \ncall 1b \n"
314 AESKEYGENA xmm0_xmm1
",0x36 \ncall 1b \n"
316 :
"r" (rk),
"r" (key)
317 :
"memory",
"cc",
"0" );
323 static void aesni_setkey_enc_192(
unsigned char *rk,
324 const unsigned char *key )
326 asm(
"movdqu (%1), %%xmm0 \n"
327 "movdqu %%xmm0, (%0) \n"
329 "movq 16(%1), %%xmm1 \n"
330 "movq %%xmm1, (%0) \n"
344 "pshufd $0x55, %%xmm2, %%xmm2 \n"
345 "pxor %%xmm0, %%xmm2 \n"
346 "pslldq $4, %%xmm0 \n"
347 "pxor %%xmm0, %%xmm2 \n"
348 "pslldq $4, %%xmm0 \n"
349 "pxor %%xmm0, %%xmm2 \n"
350 "pslldq $4, %%xmm0 \n"
351 "pxor %%xmm2, %%xmm0 \n"
352 "movdqu %%xmm0, (%0) \n"
354 "pshufd $0xff, %%xmm0, %%xmm2 \n"
355 "pxor %%xmm1, %%xmm2 \n"
356 "pslldq $4, %%xmm1 \n"
357 "pxor %%xmm2, %%xmm1 \n"
358 "movq %%xmm1, (%0) \n"
363 AESKEYGENA xmm1_xmm2
",0x01 \ncall 1b \n"
364 AESKEYGENA xmm1_xmm2
",0x02 \ncall 1b \n"
365 AESKEYGENA xmm1_xmm2
",0x04 \ncall 1b \n"
366 AESKEYGENA xmm1_xmm2
",0x08 \ncall 1b \n"
367 AESKEYGENA xmm1_xmm2
",0x10 \ncall 1b \n"
368 AESKEYGENA xmm1_xmm2
",0x20 \ncall 1b \n"
369 AESKEYGENA xmm1_xmm2
",0x40 \ncall 1b \n"
370 AESKEYGENA xmm1_xmm2
",0x80 \ncall 1b \n"
373 :
"r" (rk),
"r" (key)
374 :
"memory",
"cc",
"0" );
380 static void aesni_setkey_enc_256(
unsigned char *rk,
381 const unsigned char *key )
383 asm(
"movdqu (%1), %%xmm0 \n"
384 "movdqu %%xmm0, (%0) \n"
386 "movdqu 16(%1), %%xmm1 \n"
387 "movdqu %%xmm1, (%0) \n"
400 "pshufd $0xff, %%xmm2, %%xmm2 \n"
401 "pxor %%xmm0, %%xmm2 \n"
402 "pslldq $4, %%xmm0 \n"
403 "pxor %%xmm0, %%xmm2 \n"
404 "pslldq $4, %%xmm0 \n"
405 "pxor %%xmm0, %%xmm2 \n"
406 "pslldq $4, %%xmm0 \n"
407 "pxor %%xmm2, %%xmm0 \n"
409 "movdqu %%xmm0, (%0) \n"
413 AESKEYGENA xmm0_xmm2
",0x00 \n"
414 "pshufd $0xaa, %%xmm2, %%xmm2 \n"
415 "pxor %%xmm1, %%xmm2 \n"
416 "pslldq $4, %%xmm1 \n"
417 "pxor %%xmm1, %%xmm2 \n"
418 "pslldq $4, %%xmm1 \n"
419 "pxor %%xmm1, %%xmm2 \n"
420 "pslldq $4, %%xmm1 \n"
421 "pxor %%xmm2, %%xmm1 \n"
423 "movdqu %%xmm1, (%0) \n"
431 AESKEYGENA xmm1_xmm2
",0x01 \ncall 1b \n"
432 AESKEYGENA xmm1_xmm2
",0x02 \ncall 1b \n"
433 AESKEYGENA xmm1_xmm2
",0x04 \ncall 1b \n"
434 AESKEYGENA xmm1_xmm2
",0x08 \ncall 1b \n"
435 AESKEYGENA xmm1_xmm2
",0x10 \ncall 1b \n"
436 AESKEYGENA xmm1_xmm2
",0x20 \ncall 1b \n"
437 AESKEYGENA xmm1_xmm2
",0x40 \ncall 1b \n"
439 :
"r" (rk),
"r" (key)
440 :
"memory",
"cc",
"0" );
446 int aesni_setkey_enc(
unsigned char *rk,
447 const unsigned char *key,
452 case 128: aesni_setkey_enc_128( rk, key );
break;
453 case 192: aesni_setkey_enc_192( rk, key );
break;
454 case 256: aesni_setkey_enc_256( rk, key );
break;
Configuration options (set of defines)
AES-NI for hardware AES acceleration on some Intel processors.
#define POLARSSL_ERR_AES_INVALID_KEY_LENGTH
Invalid key length.