| default rel |
| %define XMMWORD |
| %define YMMWORD |
| %define ZMMWORD |
| section .text code align=64 |
| |
| |
| |
| ALIGN 32 |
| _aesni_ctr32_ghash_6x: |
| |
| vmovdqu xmm2,XMMWORD[32+r11] |
| sub rdx,6 |
| vpxor xmm4,xmm4,xmm4 |
| vmovdqu xmm15,XMMWORD[((0-128))+rcx] |
| vpaddb xmm10,xmm1,xmm2 |
| vpaddb xmm11,xmm10,xmm2 |
| vpaddb xmm12,xmm11,xmm2 |
| vpaddb xmm13,xmm12,xmm2 |
| vpaddb xmm14,xmm13,xmm2 |
| vpxor xmm9,xmm1,xmm15 |
| vmovdqu XMMWORD[(16+8)+rsp],xmm4 |
| jmp NEAR $L$oop6x |
| |
| ALIGN 32 |
| $L$oop6x: |
| add ebx,100663296 |
| jc NEAR $L$handle_ctr32 |
| vmovdqu xmm3,XMMWORD[((0-32))+r9] |
| vpaddb xmm1,xmm14,xmm2 |
| vpxor xmm10,xmm10,xmm15 |
| vpxor xmm11,xmm11,xmm15 |
| |
| $L$resume_ctr32: |
| vmovdqu XMMWORD[r8],xmm1 |
| vpclmulqdq xmm5,xmm7,xmm3,0x10 |
| vpxor xmm12,xmm12,xmm15 |
| vmovups xmm2,XMMWORD[((16-128))+rcx] |
| vpclmulqdq xmm6,xmm7,xmm3,0x01 |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| xor r12,r12 |
| cmp r15,r14 |
| |
| vaesenc xmm9,xmm9,xmm2 |
| vmovdqu xmm0,XMMWORD[((48+8))+rsp] |
| vpxor xmm13,xmm13,xmm15 |
| vpclmulqdq xmm1,xmm7,xmm3,0x00 |
| vaesenc xmm10,xmm10,xmm2 |
| vpxor xmm14,xmm14,xmm15 |
| setnc r12b |
| vpclmulqdq xmm7,xmm7,xmm3,0x11 |
| vaesenc xmm11,xmm11,xmm2 |
| vmovdqu xmm3,XMMWORD[((16-32))+r9] |
| neg r12 |
| vaesenc xmm12,xmm12,xmm2 |
| vpxor xmm6,xmm6,xmm5 |
| vpclmulqdq xmm5,xmm0,xmm3,0x00 |
| vpxor xmm8,xmm8,xmm4 |
| vaesenc xmm13,xmm13,xmm2 |
| vpxor xmm4,xmm1,xmm5 |
| and r12,0x60 |
| vmovups xmm15,XMMWORD[((32-128))+rcx] |
| vpclmulqdq xmm1,xmm0,xmm3,0x10 |
| vaesenc xmm14,xmm14,xmm2 |
| |
| vpclmulqdq xmm2,xmm0,xmm3,0x01 |
| lea r14,[r12*1+r14] |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] |
| vpclmulqdq xmm3,xmm0,xmm3,0x11 |
| vmovdqu xmm0,XMMWORD[((64+8))+rsp] |
| vaesenc xmm10,xmm10,xmm15 |
| movbe r13,QWORD[88+r14] |
| vaesenc xmm11,xmm11,xmm15 |
| movbe r12,QWORD[80+r14] |
| vaesenc xmm12,xmm12,xmm15 |
| mov QWORD[((32+8))+rsp],r13 |
| vaesenc xmm13,xmm13,xmm15 |
| mov QWORD[((40+8))+rsp],r12 |
| vmovdqu xmm5,XMMWORD[((48-32))+r9] |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vmovups xmm15,XMMWORD[((48-128))+rcx] |
| vpxor xmm6,xmm6,xmm1 |
| vpclmulqdq xmm1,xmm0,xmm5,0x00 |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm6,xmm6,xmm2 |
| vpclmulqdq xmm2,xmm0,xmm5,0x10 |
| vaesenc xmm10,xmm10,xmm15 |
| vpxor xmm7,xmm7,xmm3 |
| vpclmulqdq xmm3,xmm0,xmm5,0x01 |
| vaesenc xmm11,xmm11,xmm15 |
| vpclmulqdq xmm5,xmm0,xmm5,0x11 |
| vmovdqu xmm0,XMMWORD[((80+8))+rsp] |
| vaesenc xmm12,xmm12,xmm15 |
| vaesenc xmm13,xmm13,xmm15 |
| vpxor xmm4,xmm4,xmm1 |
| vmovdqu xmm1,XMMWORD[((64-32))+r9] |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vmovups xmm15,XMMWORD[((64-128))+rcx] |
| vpxor xmm6,xmm6,xmm2 |
| vpclmulqdq xmm2,xmm0,xmm1,0x00 |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm6,xmm6,xmm3 |
| vpclmulqdq xmm3,xmm0,xmm1,0x10 |
| vaesenc xmm10,xmm10,xmm15 |
| movbe r13,QWORD[72+r14] |
| vpxor xmm7,xmm7,xmm5 |
| vpclmulqdq xmm5,xmm0,xmm1,0x01 |
| vaesenc xmm11,xmm11,xmm15 |
| movbe r12,QWORD[64+r14] |
| vpclmulqdq xmm1,xmm0,xmm1,0x11 |
| vmovdqu xmm0,XMMWORD[((96+8))+rsp] |
| vaesenc xmm12,xmm12,xmm15 |
| mov QWORD[((48+8))+rsp],r13 |
| vaesenc xmm13,xmm13,xmm15 |
| mov QWORD[((56+8))+rsp],r12 |
| vpxor xmm4,xmm4,xmm2 |
| vmovdqu xmm2,XMMWORD[((96-32))+r9] |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vmovups xmm15,XMMWORD[((80-128))+rcx] |
| vpxor xmm6,xmm6,xmm3 |
| vpclmulqdq xmm3,xmm0,xmm2,0x00 |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm6,xmm6,xmm5 |
| vpclmulqdq xmm5,xmm0,xmm2,0x10 |
| vaesenc xmm10,xmm10,xmm15 |
| movbe r13,QWORD[56+r14] |
| vpxor xmm7,xmm7,xmm1 |
| vpclmulqdq xmm1,xmm0,xmm2,0x01 |
| vpxor xmm8,xmm8,XMMWORD[((112+8))+rsp] |
| vaesenc xmm11,xmm11,xmm15 |
| movbe r12,QWORD[48+r14] |
| vpclmulqdq xmm2,xmm0,xmm2,0x11 |
| vaesenc xmm12,xmm12,xmm15 |
| mov QWORD[((64+8))+rsp],r13 |
| vaesenc xmm13,xmm13,xmm15 |
| mov QWORD[((72+8))+rsp],r12 |
| vpxor xmm4,xmm4,xmm3 |
| vmovdqu xmm3,XMMWORD[((112-32))+r9] |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vmovups xmm15,XMMWORD[((96-128))+rcx] |
| vpxor xmm6,xmm6,xmm5 |
| vpclmulqdq xmm5,xmm8,xmm3,0x10 |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm6,xmm6,xmm1 |
| vpclmulqdq xmm1,xmm8,xmm3,0x01 |
| vaesenc xmm10,xmm10,xmm15 |
| movbe r13,QWORD[40+r14] |
| vpxor xmm7,xmm7,xmm2 |
| vpclmulqdq xmm2,xmm8,xmm3,0x00 |
| vaesenc xmm11,xmm11,xmm15 |
| movbe r12,QWORD[32+r14] |
| vpclmulqdq xmm8,xmm8,xmm3,0x11 |
| vaesenc xmm12,xmm12,xmm15 |
| mov QWORD[((80+8))+rsp],r13 |
| vaesenc xmm13,xmm13,xmm15 |
| mov QWORD[((88+8))+rsp],r12 |
| vpxor xmm6,xmm6,xmm5 |
| vaesenc xmm14,xmm14,xmm15 |
| vpxor xmm6,xmm6,xmm1 |
| |
| vmovups xmm15,XMMWORD[((112-128))+rcx] |
| vpslldq xmm5,xmm6,8 |
| vpxor xmm4,xmm4,xmm2 |
| vmovdqu xmm3,XMMWORD[16+r11] |
| |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm7,xmm7,xmm8 |
| vaesenc xmm10,xmm10,xmm15 |
| vpxor xmm4,xmm4,xmm5 |
| movbe r13,QWORD[24+r14] |
| vaesenc xmm11,xmm11,xmm15 |
| movbe r12,QWORD[16+r14] |
| vpalignr xmm0,xmm4,xmm4,8 |
| vpclmulqdq xmm4,xmm4,xmm3,0x10 |
| mov QWORD[((96+8))+rsp],r13 |
| vaesenc xmm12,xmm12,xmm15 |
| mov QWORD[((104+8))+rsp],r12 |
| vaesenc xmm13,xmm13,xmm15 |
| vmovups xmm1,XMMWORD[((128-128))+rcx] |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vaesenc xmm9,xmm9,xmm1 |
| vmovups xmm15,XMMWORD[((144-128))+rcx] |
| vaesenc xmm10,xmm10,xmm1 |
| vpsrldq xmm6,xmm6,8 |
| vaesenc xmm11,xmm11,xmm1 |
| vpxor xmm7,xmm7,xmm6 |
| vaesenc xmm12,xmm12,xmm1 |
| vpxor xmm4,xmm4,xmm0 |
| movbe r13,QWORD[8+r14] |
| vaesenc xmm13,xmm13,xmm1 |
| movbe r12,QWORD[r14] |
| vaesenc xmm14,xmm14,xmm1 |
| vmovups xmm1,XMMWORD[((160-128))+rcx] |
| cmp ebp,11 |
| jb NEAR $L$enc_tail |
| |
| vaesenc xmm9,xmm9,xmm15 |
| vaesenc xmm10,xmm10,xmm15 |
| vaesenc xmm11,xmm11,xmm15 |
| vaesenc xmm12,xmm12,xmm15 |
| vaesenc xmm13,xmm13,xmm15 |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vaesenc xmm9,xmm9,xmm1 |
| vaesenc xmm10,xmm10,xmm1 |
| vaesenc xmm11,xmm11,xmm1 |
| vaesenc xmm12,xmm12,xmm1 |
| vaesenc xmm13,xmm13,xmm1 |
| vmovups xmm15,XMMWORD[((176-128))+rcx] |
| vaesenc xmm14,xmm14,xmm1 |
| vmovups xmm1,XMMWORD[((192-128))+rcx] |
| je NEAR $L$enc_tail |
| |
| vaesenc xmm9,xmm9,xmm15 |
| vaesenc xmm10,xmm10,xmm15 |
| vaesenc xmm11,xmm11,xmm15 |
| vaesenc xmm12,xmm12,xmm15 |
| vaesenc xmm13,xmm13,xmm15 |
| vaesenc xmm14,xmm14,xmm15 |
| |
| vaesenc xmm9,xmm9,xmm1 |
| vaesenc xmm10,xmm10,xmm1 |
| vaesenc xmm11,xmm11,xmm1 |
| vaesenc xmm12,xmm12,xmm1 |
| vaesenc xmm13,xmm13,xmm1 |
| vmovups xmm15,XMMWORD[((208-128))+rcx] |
| vaesenc xmm14,xmm14,xmm1 |
| vmovups xmm1,XMMWORD[((224-128))+rcx] |
| jmp NEAR $L$enc_tail |
| |
| ALIGN 32 |
| $L$handle_ctr32: |
| vmovdqu xmm0,XMMWORD[r11] |
| vpshufb xmm6,xmm1,xmm0 |
| vmovdqu xmm5,XMMWORD[48+r11] |
| vpaddd xmm10,xmm6,XMMWORD[64+r11] |
| vpaddd xmm11,xmm6,xmm5 |
| vmovdqu xmm3,XMMWORD[((0-32))+r9] |
| vpaddd xmm12,xmm10,xmm5 |
| vpshufb xmm10,xmm10,xmm0 |
| vpaddd xmm13,xmm11,xmm5 |
| vpshufb xmm11,xmm11,xmm0 |
| vpxor xmm10,xmm10,xmm15 |
| vpaddd xmm14,xmm12,xmm5 |
| vpshufb xmm12,xmm12,xmm0 |
| vpxor xmm11,xmm11,xmm15 |
| vpaddd xmm1,xmm13,xmm5 |
| vpshufb xmm13,xmm13,xmm0 |
| vpshufb xmm14,xmm14,xmm0 |
| vpshufb xmm1,xmm1,xmm0 |
| jmp NEAR $L$resume_ctr32 |
| |
| ALIGN 32 |
| $L$enc_tail: |
| vaesenc xmm9,xmm9,xmm15 |
| vmovdqu XMMWORD[(16+8)+rsp],xmm7 |
| vpalignr xmm8,xmm4,xmm4,8 |
| vaesenc xmm10,xmm10,xmm15 |
| vpclmulqdq xmm4,xmm4,xmm3,0x10 |
| vpxor xmm2,xmm1,XMMWORD[rdi] |
| vaesenc xmm11,xmm11,xmm15 |
| vpxor xmm0,xmm1,XMMWORD[16+rdi] |
| vaesenc xmm12,xmm12,xmm15 |
| vpxor xmm5,xmm1,XMMWORD[32+rdi] |
| vaesenc xmm13,xmm13,xmm15 |
| vpxor xmm6,xmm1,XMMWORD[48+rdi] |
| vaesenc xmm14,xmm14,xmm15 |
| vpxor xmm7,xmm1,XMMWORD[64+rdi] |
| vpxor xmm3,xmm1,XMMWORD[80+rdi] |
| vmovdqu xmm1,XMMWORD[r8] |
| |
| vaesenclast xmm9,xmm9,xmm2 |
| vmovdqu xmm2,XMMWORD[32+r11] |
| vaesenclast xmm10,xmm10,xmm0 |
| vpaddb xmm0,xmm1,xmm2 |
| mov QWORD[((112+8))+rsp],r13 |
| lea rdi,[96+rdi] |
| vaesenclast xmm11,xmm11,xmm5 |
| vpaddb xmm5,xmm0,xmm2 |
| mov QWORD[((120+8))+rsp],r12 |
| lea rsi,[96+rsi] |
| vmovdqu xmm15,XMMWORD[((0-128))+rcx] |
| vaesenclast xmm12,xmm12,xmm6 |
| vpaddb xmm6,xmm5,xmm2 |
| vaesenclast xmm13,xmm13,xmm7 |
| vpaddb xmm7,xmm6,xmm2 |
| vaesenclast xmm14,xmm14,xmm3 |
| vpaddb xmm3,xmm7,xmm2 |
| |
| add r10,0x60 |
| sub rdx,0x6 |
| jc NEAR $L$6x_done |
| |
| vmovups XMMWORD[(-96)+rsi],xmm9 |
| vpxor xmm9,xmm1,xmm15 |
| vmovups XMMWORD[(-80)+rsi],xmm10 |
| vmovdqa xmm10,xmm0 |
| vmovups XMMWORD[(-64)+rsi],xmm11 |
| vmovdqa xmm11,xmm5 |
| vmovups XMMWORD[(-48)+rsi],xmm12 |
| vmovdqa xmm12,xmm6 |
| vmovups XMMWORD[(-32)+rsi],xmm13 |
| vmovdqa xmm13,xmm7 |
| vmovups XMMWORD[(-16)+rsi],xmm14 |
| vmovdqa xmm14,xmm3 |
| vmovdqu xmm7,XMMWORD[((32+8))+rsp] |
| jmp NEAR $L$oop6x |
| |
| $L$6x_done: |
| vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] |
| vpxor xmm8,xmm8,xmm4 |
| |
| DB 0F3h,0C3h ;repret |
| |
| |
| global aesni_gcm_decrypt |
| |
| ALIGN 32 |
| aesni_gcm_decrypt: |
| mov QWORD[8+rsp],rdi ;WIN64 prologue |
| mov QWORD[16+rsp],rsi |
| mov rax,rsp |
| $L$SEH_begin_aesni_gcm_decrypt: |
| mov rdi,rcx |
| mov rsi,rdx |
| mov rdx,r8 |
| mov rcx,r9 |
| mov r8,QWORD[40+rsp] |
| mov r9,QWORD[48+rsp] |
| |
| |
| |
| xor r10,r10 |
| |
| |
| |
| cmp rdx,0x60 |
| jb NEAR $L$gcm_dec_abort |
| |
| lea rax,[rsp] |
| |
| push rbx |
| |
| push rbp |
| |
| push r12 |
| |
| push r13 |
| |
| push r14 |
| |
| push r15 |
| |
| lea rsp,[((-168))+rsp] |
| movaps XMMWORD[(-216)+rax],xmm6 |
| movaps XMMWORD[(-200)+rax],xmm7 |
| movaps XMMWORD[(-184)+rax],xmm8 |
| movaps XMMWORD[(-168)+rax],xmm9 |
| movaps XMMWORD[(-152)+rax],xmm10 |
| movaps XMMWORD[(-136)+rax],xmm11 |
| movaps XMMWORD[(-120)+rax],xmm12 |
| movaps XMMWORD[(-104)+rax],xmm13 |
| movaps XMMWORD[(-88)+rax],xmm14 |
| movaps XMMWORD[(-72)+rax],xmm15 |
| $L$gcm_dec_body: |
| vzeroupper |
| |
| vmovdqu xmm1,XMMWORD[r8] |
| add rsp,-128 |
| mov ebx,DWORD[12+r8] |
| lea r11,[$L$bswap_mask] |
| lea r14,[((-128))+rcx] |
| mov r15,0xf80 |
| vmovdqu xmm8,XMMWORD[r9] |
| and rsp,-128 |
| vmovdqu xmm0,XMMWORD[r11] |
| lea rcx,[128+rcx] |
| lea r9,[((32+32))+r9] |
| mov ebp,DWORD[((240-128))+rcx] |
| vpshufb xmm8,xmm8,xmm0 |
| |
| and r14,r15 |
| and r15,rsp |
| sub r15,r14 |
| jc NEAR $L$dec_no_key_aliasing |
| cmp r15,768 |
| jnc NEAR $L$dec_no_key_aliasing |
| sub rsp,r15 |
| $L$dec_no_key_aliasing: |
| |
| vmovdqu xmm7,XMMWORD[80+rdi] |
| lea r14,[rdi] |
| vmovdqu xmm4,XMMWORD[64+rdi] |
| |
| |
| |
| |
| |
| |
| |
| lea r15,[((-192))+rdx*1+rdi] |
| |
| vmovdqu xmm5,XMMWORD[48+rdi] |
| shr rdx,4 |
| xor r10,r10 |
| vmovdqu xmm6,XMMWORD[32+rdi] |
| vpshufb xmm7,xmm7,xmm0 |
| vmovdqu xmm2,XMMWORD[16+rdi] |
| vpshufb xmm4,xmm4,xmm0 |
| vmovdqu xmm3,XMMWORD[rdi] |
| vpshufb xmm5,xmm5,xmm0 |
| vmovdqu XMMWORD[48+rsp],xmm4 |
| vpshufb xmm6,xmm6,xmm0 |
| vmovdqu XMMWORD[64+rsp],xmm5 |
| vpshufb xmm2,xmm2,xmm0 |
| vmovdqu XMMWORD[80+rsp],xmm6 |
| vpshufb xmm3,xmm3,xmm0 |
| vmovdqu XMMWORD[96+rsp],xmm2 |
| vmovdqu XMMWORD[112+rsp],xmm3 |
| |
| call _aesni_ctr32_ghash_6x |
| |
| vmovups XMMWORD[(-96)+rsi],xmm9 |
| vmovups XMMWORD[(-80)+rsi],xmm10 |
| vmovups XMMWORD[(-64)+rsi],xmm11 |
| vmovups XMMWORD[(-48)+rsi],xmm12 |
| vmovups XMMWORD[(-32)+rsi],xmm13 |
| vmovups XMMWORD[(-16)+rsi],xmm14 |
| |
| vpshufb xmm8,xmm8,XMMWORD[r11] |
| vmovdqu XMMWORD[(-64)+r9],xmm8 |
| |
| vzeroupper |
| movaps xmm6,XMMWORD[((-216))+rax] |
| movaps xmm7,XMMWORD[((-200))+rax] |
| movaps xmm8,XMMWORD[((-184))+rax] |
| movaps xmm9,XMMWORD[((-168))+rax] |
| movaps xmm10,XMMWORD[((-152))+rax] |
| movaps xmm11,XMMWORD[((-136))+rax] |
| movaps xmm12,XMMWORD[((-120))+rax] |
| movaps xmm13,XMMWORD[((-104))+rax] |
| movaps xmm14,XMMWORD[((-88))+rax] |
| movaps xmm15,XMMWORD[((-72))+rax] |
| mov r15,QWORD[((-48))+rax] |
| |
| mov r14,QWORD[((-40))+rax] |
| |
| mov r13,QWORD[((-32))+rax] |
| |
| mov r12,QWORD[((-24))+rax] |
| |
| mov rbp,QWORD[((-16))+rax] |
| |
| mov rbx,QWORD[((-8))+rax] |
| |
| lea rsp,[rax] |
| |
| $L$gcm_dec_abort: |
| mov rax,r10 |
| mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| mov rsi,QWORD[16+rsp] |
| DB 0F3h,0C3h ;repret |
| |
| $L$SEH_end_aesni_gcm_decrypt: |
| |
| ALIGN 32 |
| _aesni_ctr32_6x: |
| |
| vmovdqu xmm4,XMMWORD[((0-128))+rcx] |
| vmovdqu xmm2,XMMWORD[32+r11] |
| lea r13,[((-1))+rbp] |
| vmovups xmm15,XMMWORD[((16-128))+rcx] |
| lea r12,[((32-128))+rcx] |
| vpxor xmm9,xmm1,xmm4 |
| add ebx,100663296 |
| jc NEAR $L$handle_ctr32_2 |
| vpaddb xmm10,xmm1,xmm2 |
| vpaddb xmm11,xmm10,xmm2 |
| vpxor xmm10,xmm10,xmm4 |
| vpaddb xmm12,xmm11,xmm2 |
| vpxor xmm11,xmm11,xmm4 |
| vpaddb xmm13,xmm12,xmm2 |
| vpxor xmm12,xmm12,xmm4 |
| vpaddb xmm14,xmm13,xmm2 |
| vpxor xmm13,xmm13,xmm4 |
| vpaddb xmm1,xmm14,xmm2 |
| vpxor xmm14,xmm14,xmm4 |
| jmp NEAR $L$oop_ctr32 |
| |
| ALIGN 16 |
| $L$oop_ctr32: |
| vaesenc xmm9,xmm9,xmm15 |
| vaesenc xmm10,xmm10,xmm15 |
| vaesenc xmm11,xmm11,xmm15 |
| vaesenc xmm12,xmm12,xmm15 |
| vaesenc xmm13,xmm13,xmm15 |
| vaesenc xmm14,xmm14,xmm15 |
| vmovups xmm15,XMMWORD[r12] |
| lea r12,[16+r12] |
| dec r13d |
| jnz NEAR $L$oop_ctr32 |
| |
| vmovdqu xmm3,XMMWORD[r12] |
| vaesenc xmm9,xmm9,xmm15 |
| vpxor xmm4,xmm3,XMMWORD[rdi] |
| vaesenc xmm10,xmm10,xmm15 |
| vpxor xmm5,xmm3,XMMWORD[16+rdi] |
| vaesenc xmm11,xmm11,xmm15 |
| vpxor xmm6,xmm3,XMMWORD[32+rdi] |
| vaesenc xmm12,xmm12,xmm15 |
| vpxor xmm8,xmm3,XMMWORD[48+rdi] |
| vaesenc xmm13,xmm13,xmm15 |
| vpxor xmm2,xmm3,XMMWORD[64+rdi] |
| vaesenc xmm14,xmm14,xmm15 |
| vpxor xmm3,xmm3,XMMWORD[80+rdi] |
| lea rdi,[96+rdi] |
| |
| vaesenclast xmm9,xmm9,xmm4 |
| vaesenclast xmm10,xmm10,xmm5 |
| vaesenclast xmm11,xmm11,xmm6 |
| vaesenclast xmm12,xmm12,xmm8 |
| vaesenclast xmm13,xmm13,xmm2 |
| vaesenclast xmm14,xmm14,xmm3 |
| vmovups XMMWORD[rsi],xmm9 |
| vmovups XMMWORD[16+rsi],xmm10 |
| vmovups XMMWORD[32+rsi],xmm11 |
| vmovups XMMWORD[48+rsi],xmm12 |
| vmovups XMMWORD[64+rsi],xmm13 |
| vmovups XMMWORD[80+rsi],xmm14 |
| lea rsi,[96+rsi] |
| |
| DB 0F3h,0C3h ;repret |
| ALIGN 32 |
| $L$handle_ctr32_2: |
| vpshufb xmm6,xmm1,xmm0 |
| vmovdqu xmm5,XMMWORD[48+r11] |
| vpaddd xmm10,xmm6,XMMWORD[64+r11] |
| vpaddd xmm11,xmm6,xmm5 |
| vpaddd xmm12,xmm10,xmm5 |
| vpshufb xmm10,xmm10,xmm0 |
| vpaddd xmm13,xmm11,xmm5 |
| vpshufb xmm11,xmm11,xmm0 |
| vpxor xmm10,xmm10,xmm4 |
| vpaddd xmm14,xmm12,xmm5 |
| vpshufb xmm12,xmm12,xmm0 |
| vpxor xmm11,xmm11,xmm4 |
| vpaddd xmm1,xmm13,xmm5 |
| vpshufb xmm13,xmm13,xmm0 |
| vpxor xmm12,xmm12,xmm4 |
| vpshufb xmm14,xmm14,xmm0 |
| vpxor xmm13,xmm13,xmm4 |
| vpshufb xmm1,xmm1,xmm0 |
| vpxor xmm14,xmm14,xmm4 |
| jmp NEAR $L$oop_ctr32 |
| |
| |
| |
| global aesni_gcm_encrypt |
| |
| ALIGN 32 |
| aesni_gcm_encrypt: |
| mov QWORD[8+rsp],rdi ;WIN64 prologue |
| mov QWORD[16+rsp],rsi |
| mov rax,rsp |
| $L$SEH_begin_aesni_gcm_encrypt: |
| mov rdi,rcx |
| mov rsi,rdx |
| mov rdx,r8 |
| mov rcx,r9 |
| mov r8,QWORD[40+rsp] |
| mov r9,QWORD[48+rsp] |
| |
| |
| |
| xor r10,r10 |
| |
| |
| |
| |
| cmp rdx,0x60*3 |
| jb NEAR $L$gcm_enc_abort |
| |
| lea rax,[rsp] |
| |
| push rbx |
| |
| push rbp |
| |
| push r12 |
| |
| push r13 |
| |
| push r14 |
| |
| push r15 |
| |
| lea rsp,[((-168))+rsp] |
| movaps XMMWORD[(-216)+rax],xmm6 |
| movaps XMMWORD[(-200)+rax],xmm7 |
| movaps XMMWORD[(-184)+rax],xmm8 |
| movaps XMMWORD[(-168)+rax],xmm9 |
| movaps XMMWORD[(-152)+rax],xmm10 |
| movaps XMMWORD[(-136)+rax],xmm11 |
| movaps XMMWORD[(-120)+rax],xmm12 |
| movaps XMMWORD[(-104)+rax],xmm13 |
| movaps XMMWORD[(-88)+rax],xmm14 |
| movaps XMMWORD[(-72)+rax],xmm15 |
| $L$gcm_enc_body: |
| vzeroupper |
| |
| vmovdqu xmm1,XMMWORD[r8] |
| add rsp,-128 |
| mov ebx,DWORD[12+r8] |
| lea r11,[$L$bswap_mask] |
| lea r14,[((-128))+rcx] |
| mov r15,0xf80 |
| lea rcx,[128+rcx] |
| vmovdqu xmm0,XMMWORD[r11] |
| and rsp,-128 |
| mov ebp,DWORD[((240-128))+rcx] |
| |
| and r14,r15 |
| and r15,rsp |
| sub r15,r14 |
| jc NEAR $L$enc_no_key_aliasing |
| cmp r15,768 |
| jnc NEAR $L$enc_no_key_aliasing |
| sub rsp,r15 |
| $L$enc_no_key_aliasing: |
| |
| lea r14,[rsi] |
| |
| |
| |
| |
| |
| |
| |
| |
| lea r15,[((-192))+rdx*1+rsi] |
| |
| shr rdx,4 |
| |
| call _aesni_ctr32_6x |
| vpshufb xmm8,xmm9,xmm0 |
| vpshufb xmm2,xmm10,xmm0 |
| vmovdqu XMMWORD[112+rsp],xmm8 |
| vpshufb xmm4,xmm11,xmm0 |
| vmovdqu XMMWORD[96+rsp],xmm2 |
| vpshufb xmm5,xmm12,xmm0 |
| vmovdqu XMMWORD[80+rsp],xmm4 |
| vpshufb xmm6,xmm13,xmm0 |
| vmovdqu XMMWORD[64+rsp],xmm5 |
| vpshufb xmm7,xmm14,xmm0 |
| vmovdqu XMMWORD[48+rsp],xmm6 |
| |
| call _aesni_ctr32_6x |
| |
| vmovdqu xmm8,XMMWORD[r9] |
| lea r9,[((32+32))+r9] |
| sub rdx,12 |
| mov r10,0x60*2 |
| vpshufb xmm8,xmm8,xmm0 |
| |
| call _aesni_ctr32_ghash_6x |
| vmovdqu xmm7,XMMWORD[32+rsp] |
| vmovdqu xmm0,XMMWORD[r11] |
| vmovdqu xmm3,XMMWORD[((0-32))+r9] |
| vpunpckhqdq xmm1,xmm7,xmm7 |
| vmovdqu xmm15,XMMWORD[((32-32))+r9] |
| vmovups XMMWORD[(-96)+rsi],xmm9 |
| vpshufb xmm9,xmm9,xmm0 |
| vpxor xmm1,xmm1,xmm7 |
| vmovups XMMWORD[(-80)+rsi],xmm10 |
| vpshufb xmm10,xmm10,xmm0 |
| vmovups XMMWORD[(-64)+rsi],xmm11 |
| vpshufb xmm11,xmm11,xmm0 |
| vmovups XMMWORD[(-48)+rsi],xmm12 |
| vpshufb xmm12,xmm12,xmm0 |
| vmovups XMMWORD[(-32)+rsi],xmm13 |
| vpshufb xmm13,xmm13,xmm0 |
| vmovups XMMWORD[(-16)+rsi],xmm14 |
| vpshufb xmm14,xmm14,xmm0 |
| vmovdqu XMMWORD[16+rsp],xmm9 |
| vmovdqu xmm6,XMMWORD[48+rsp] |
| vmovdqu xmm0,XMMWORD[((16-32))+r9] |
| vpunpckhqdq xmm2,xmm6,xmm6 |
| vpclmulqdq xmm5,xmm7,xmm3,0x00 |
| vpxor xmm2,xmm2,xmm6 |
| vpclmulqdq xmm7,xmm7,xmm3,0x11 |
| vpclmulqdq xmm1,xmm1,xmm15,0x00 |
| |
| vmovdqu xmm9,XMMWORD[64+rsp] |
| vpclmulqdq xmm4,xmm6,xmm0,0x00 |
| vmovdqu xmm3,XMMWORD[((48-32))+r9] |
| vpxor xmm4,xmm4,xmm5 |
| vpunpckhqdq xmm5,xmm9,xmm9 |
| vpclmulqdq xmm6,xmm6,xmm0,0x11 |
| vpxor xmm5,xmm5,xmm9 |
| vpxor xmm6,xmm6,xmm7 |
| vpclmulqdq xmm2,xmm2,xmm15,0x10 |
| vmovdqu xmm15,XMMWORD[((80-32))+r9] |
| vpxor xmm2,xmm2,xmm1 |
| |
| vmovdqu xmm1,XMMWORD[80+rsp] |
| vpclmulqdq xmm7,xmm9,xmm3,0x00 |
| vmovdqu xmm0,XMMWORD[((64-32))+r9] |
| vpxor xmm7,xmm7,xmm4 |
| vpunpckhqdq xmm4,xmm1,xmm1 |
| vpclmulqdq xmm9,xmm9,xmm3,0x11 |
| vpxor xmm4,xmm4,xmm1 |
| vpxor xmm9,xmm9,xmm6 |
| vpclmulqdq xmm5,xmm5,xmm15,0x00 |
| vpxor xmm5,xmm5,xmm2 |
| |
| vmovdqu xmm2,XMMWORD[96+rsp] |
| vpclmulqdq xmm6,xmm1,xmm0,0x00 |
| vmovdqu xmm3,XMMWORD[((96-32))+r9] |
| vpxor xmm6,xmm6,xmm7 |
| vpunpckhqdq xmm7,xmm2,xmm2 |
| vpclmulqdq xmm1,xmm1,xmm0,0x11 |
| vpxor xmm7,xmm7,xmm2 |
| vpxor xmm1,xmm1,xmm9 |
| vpclmulqdq xmm4,xmm4,xmm15,0x10 |
| vmovdqu xmm15,XMMWORD[((128-32))+r9] |
| vpxor xmm4,xmm4,xmm5 |
| |
| vpxor xmm8,xmm8,XMMWORD[112+rsp] |
| vpclmulqdq xmm5,xmm2,xmm3,0x00 |
| vmovdqu xmm0,XMMWORD[((112-32))+r9] |
| vpunpckhqdq xmm9,xmm8,xmm8 |
| vpxor xmm5,xmm5,xmm6 |
| vpclmulqdq xmm2,xmm2,xmm3,0x11 |
| vpxor xmm9,xmm9,xmm8 |
| vpxor xmm2,xmm2,xmm1 |
| vpclmulqdq xmm7,xmm7,xmm15,0x00 |
| vpxor xmm4,xmm7,xmm4 |
| |
| vpclmulqdq xmm6,xmm8,xmm0,0x00 |
| vmovdqu xmm3,XMMWORD[((0-32))+r9] |
| vpunpckhqdq xmm1,xmm14,xmm14 |
| vpclmulqdq xmm8,xmm8,xmm0,0x11 |
| vpxor xmm1,xmm1,xmm14 |
| vpxor xmm5,xmm6,xmm5 |
| vpclmulqdq xmm9,xmm9,xmm15,0x10 |
| vmovdqu xmm15,XMMWORD[((32-32))+r9] |
| vpxor xmm7,xmm8,xmm2 |
| vpxor xmm6,xmm9,xmm4 |
| |
| vmovdqu xmm0,XMMWORD[((16-32))+r9] |
| vpxor xmm9,xmm7,xmm5 |
| vpclmulqdq xmm4,xmm14,xmm3,0x00 |
| vpxor xmm6,xmm6,xmm9 |
| vpunpckhqdq xmm2,xmm13,xmm13 |
| vpclmulqdq xmm14,xmm14,xmm3,0x11 |
| vpxor xmm2,xmm2,xmm13 |
| vpslldq xmm9,xmm6,8 |
| vpclmulqdq xmm1,xmm1,xmm15,0x00 |
| vpxor xmm8,xmm5,xmm9 |
| vpsrldq xmm6,xmm6,8 |
| vpxor xmm7,xmm7,xmm6 |
| |
| vpclmulqdq xmm5,xmm13,xmm0,0x00 |
| vmovdqu xmm3,XMMWORD[((48-32))+r9] |
| vpxor xmm5,xmm5,xmm4 |
| vpunpckhqdq xmm9,xmm12,xmm12 |
| vpclmulqdq xmm13,xmm13,xmm0,0x11 |
| vpxor xmm9,xmm9,xmm12 |
| vpxor xmm13,xmm13,xmm14 |
| vpalignr xmm14,xmm8,xmm8,8 |
| vpclmulqdq xmm2,xmm2,xmm15,0x10 |
| vmovdqu xmm15,XMMWORD[((80-32))+r9] |
| vpxor xmm2,xmm2,xmm1 |
| |
| vpclmulqdq xmm4,xmm12,xmm3,0x00 |
| vmovdqu xmm0,XMMWORD[((64-32))+r9] |
| vpxor xmm4,xmm4,xmm5 |
| vpunpckhqdq xmm1,xmm11,xmm11 |
| vpclmulqdq xmm12,xmm12,xmm3,0x11 |
| vpxor xmm1,xmm1,xmm11 |
| vpxor xmm12,xmm12,xmm13 |
| vxorps xmm7,xmm7,XMMWORD[16+rsp] |
| vpclmulqdq xmm9,xmm9,xmm15,0x00 |
| vpxor xmm9,xmm9,xmm2 |
| |
| vpclmulqdq xmm8,xmm8,XMMWORD[16+r11],0x10 |
| vxorps xmm8,xmm8,xmm14 |
| |
| vpclmulqdq xmm5,xmm11,xmm0,0x00 |
| vmovdqu xmm3,XMMWORD[((96-32))+r9] |
| vpxor xmm5,xmm5,xmm4 |
| vpunpckhqdq xmm2,xmm10,xmm10 |
| vpclmulqdq xmm11,xmm11,xmm0,0x11 |
| vpxor xmm2,xmm2,xmm10 |
| vpalignr xmm14,xmm8,xmm8,8 |
| vpxor xmm11,xmm11,xmm12 |
| vpclmulqdq xmm1,xmm1,xmm15,0x10 |
| vmovdqu xmm15,XMMWORD[((128-32))+r9] |
| vpxor xmm1,xmm1,xmm9 |
| |
| vxorps xmm14,xmm14,xmm7 |
| vpclmulqdq xmm8,xmm8,XMMWORD[16+r11],0x10 |
| vxorps xmm8,xmm8,xmm14 |
| |
| vpclmulqdq xmm4,xmm10,xmm3,0x00 |
| vmovdqu xmm0,XMMWORD[((112-32))+r9] |
| vpxor xmm4,xmm4,xmm5 |
| vpunpckhqdq xmm9,xmm8,xmm8 |
| vpclmulqdq xmm10,xmm10,xmm3,0x11 |
| vpxor xmm9,xmm9,xmm8 |
| vpxor xmm10,xmm10,xmm11 |
| vpclmulqdq xmm2,xmm2,xmm15,0x00 |
| vpxor xmm2,xmm2,xmm1 |
| |
| vpclmulqdq xmm5,xmm8,xmm0,0x00 |
| vpclmulqdq xmm7,xmm8,xmm0,0x11 |
| vpxor xmm5,xmm5,xmm4 |
| vpclmulqdq xmm6,xmm9,xmm15,0x10 |
| vpxor xmm7,xmm7,xmm10 |
| vpxor xmm6,xmm6,xmm2 |
| |
| vpxor xmm4,xmm7,xmm5 |
| vpxor xmm6,xmm6,xmm4 |
| vpslldq xmm1,xmm6,8 |
| vmovdqu xmm3,XMMWORD[16+r11] |
| vpsrldq xmm6,xmm6,8 |
| vpxor xmm8,xmm5,xmm1 |
| vpxor xmm7,xmm7,xmm6 |
| |
| vpalignr xmm2,xmm8,xmm8,8 |
| vpclmulqdq xmm8,xmm8,xmm3,0x10 |
| vpxor xmm8,xmm8,xmm2 |
| |
| vpalignr xmm2,xmm8,xmm8,8 |
| vpclmulqdq xmm8,xmm8,xmm3,0x10 |
| vpxor xmm2,xmm2,xmm7 |
| vpxor xmm8,xmm8,xmm2 |
| vpshufb xmm8,xmm8,XMMWORD[r11] |
| vmovdqu XMMWORD[(-64)+r9],xmm8 |
| |
| vzeroupper |
| movaps xmm6,XMMWORD[((-216))+rax] |
| movaps xmm7,XMMWORD[((-200))+rax] |
| movaps xmm8,XMMWORD[((-184))+rax] |
| movaps xmm9,XMMWORD[((-168))+rax] |
| movaps xmm10,XMMWORD[((-152))+rax] |
| movaps xmm11,XMMWORD[((-136))+rax] |
| movaps xmm12,XMMWORD[((-120))+rax] |
| movaps xmm13,XMMWORD[((-104))+rax] |
| movaps xmm14,XMMWORD[((-88))+rax] |
| movaps xmm15,XMMWORD[((-72))+rax] |
| mov r15,QWORD[((-48))+rax] |
| |
| mov r14,QWORD[((-40))+rax] |
| |
| mov r13,QWORD[((-32))+rax] |
| |
| mov r12,QWORD[((-24))+rax] |
| |
| mov rbp,QWORD[((-16))+rax] |
| |
| mov rbx,QWORD[((-8))+rax] |
| |
| lea rsp,[rax] |
| |
| $L$gcm_enc_abort: |
| mov rax,r10 |
| mov rdi,QWORD[8+rsp] ;WIN64 epilogue |
| mov rsi,QWORD[16+rsp] |
| DB 0F3h,0C3h ;repret |
| |
| $L$SEH_end_aesni_gcm_encrypt: |
| ALIGN 64 |
| $L$bswap_mask: |
| DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 |
| $L$poly: |
| DB 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2 |
| $L$one_msb: |
| DB 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 |
| $L$two_lsb: |
| DB 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 |
| $L$one_lsb: |
| DB 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 |
| DB 65,69,83,45,78,73,32,71,67,77,32,109,111,100,117,108 |
| DB 101,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82 |
| DB 89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112 |
| DB 114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 |
| ALIGN 64 |
| EXTERN __imp_RtlVirtualUnwind |
| |
| ALIGN 16 |
| gcm_se_handler: |
| push rsi |
| push rdi |
| push rbx |
| push rbp |
| push r12 |
| push r13 |
| push r14 |
| push r15 |
| pushfq |
| sub rsp,64 |
| |
| mov rax,QWORD[120+r8] |
| mov rbx,QWORD[248+r8] |
| |
| mov rsi,QWORD[8+r9] |
| mov r11,QWORD[56+r9] |
| |
| mov r10d,DWORD[r11] |
| lea r10,[r10*1+rsi] |
| cmp rbx,r10 |
| jb NEAR $L$common_seh_tail |
| |
| mov rax,QWORD[152+r8] |
| |
| mov r10d,DWORD[4+r11] |
| lea r10,[r10*1+rsi] |
| cmp rbx,r10 |
| jae NEAR $L$common_seh_tail |
| |
| mov rax,QWORD[120+r8] |
| |
| mov r15,QWORD[((-48))+rax] |
| mov r14,QWORD[((-40))+rax] |
| mov r13,QWORD[((-32))+rax] |
| mov r12,QWORD[((-24))+rax] |
| mov rbp,QWORD[((-16))+rax] |
| mov rbx,QWORD[((-8))+rax] |
| mov QWORD[240+r8],r15 |
| mov QWORD[232+r8],r14 |
| mov QWORD[224+r8],r13 |
| mov QWORD[216+r8],r12 |
| mov QWORD[160+r8],rbp |
| mov QWORD[144+r8],rbx |
| |
| lea rsi,[((-216))+rax] |
| lea rdi,[512+r8] |
| mov ecx,20 |
| DD 0xa548f3fc |
| |
| $L$common_seh_tail: |
| mov rdi,QWORD[8+rax] |
| mov rsi,QWORD[16+rax] |
| mov QWORD[152+r8],rax |
| mov QWORD[168+r8],rsi |
| mov QWORD[176+r8],rdi |
| |
| mov rdi,QWORD[40+r9] |
| mov rsi,r8 |
| mov ecx,154 |
| DD 0xa548f3fc |
| |
| mov rsi,r9 |
| xor rcx,rcx |
| mov rdx,QWORD[8+rsi] |
| mov r8,QWORD[rsi] |
| mov r9,QWORD[16+rsi] |
| mov r10,QWORD[40+rsi] |
| lea r11,[56+rsi] |
| lea r12,[24+rsi] |
| mov QWORD[32+rsp],r10 |
| mov QWORD[40+rsp],r11 |
| mov QWORD[48+rsp],r12 |
| mov QWORD[56+rsp],rcx |
| call QWORD[__imp_RtlVirtualUnwind] |
| |
| mov eax,1 |
| add rsp,64 |
| popfq |
| pop r15 |
| pop r14 |
| pop r13 |
| pop r12 |
| pop rbp |
| pop rbx |
| pop rdi |
| pop rsi |
| DB 0F3h,0C3h ;repret |
| |
| |
| section .pdata rdata align=4 |
| ALIGN 4 |
| DD $L$SEH_begin_aesni_gcm_decrypt wrt ..imagebase |
| DD $L$SEH_end_aesni_gcm_decrypt wrt ..imagebase |
| DD $L$SEH_gcm_dec_info wrt ..imagebase |
| |
| DD $L$SEH_begin_aesni_gcm_encrypt wrt ..imagebase |
| DD $L$SEH_end_aesni_gcm_encrypt wrt ..imagebase |
| DD $L$SEH_gcm_enc_info wrt ..imagebase |
| section .xdata rdata align=8 |
| ALIGN 8 |
| $L$SEH_gcm_dec_info: |
| DB 9,0,0,0 |
| DD gcm_se_handler wrt ..imagebase |
| DD $L$gcm_dec_body wrt ..imagebase,$L$gcm_dec_abort wrt ..imagebase |
| $L$SEH_gcm_enc_info: |
| DB 9,0,0,0 |
| DD gcm_se_handler wrt ..imagebase |
| DD $L$gcm_enc_body wrt ..imagebase,$L$gcm_enc_abort wrt ..imagebase |
