1168 lines
24 KiB
ArmAsm
1168 lines
24 KiB
ArmAsm
// This file is generated from a similarly-named Perl script in the BoringSSL
|
|
// source tree. Do not edit by hand.
|
|
|
|
#include <ring-core/asm_base.h>
|
|
|
|
#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__)
|
|
.section __DATA,__const
|
|
.p2align 4
|
|
|
|
|
|
L$bswap_mask:
|
|
.quad 0x08090a0b0c0d0e0f, 0x0001020304050607
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
L$gfpoly:
|
|
.quad 1, 0xc200000000000000
|
|
|
|
|
|
L$gfpoly_and_internal_carrybit:
|
|
.quad 1, 0xc200000000000001
|
|
|
|
.p2align 5
|
|
|
|
L$ctr_pattern:
|
|
.quad 0, 0
|
|
.quad 1, 0
|
|
L$inc_2blocks:
|
|
.quad 2, 0
|
|
.quad 2, 0
|
|
|
|
.text
|
|
.globl _gcm_init_vpclmulqdq_avx2
|
|
.private_extern _gcm_init_vpclmulqdq_avx2
|
|
|
|
.p2align 5
|
|
_gcm_init_vpclmulqdq_avx2:
|
|
|
|
|
|
_CET_ENDBR
|
|
|
|
|
|
|
|
|
|
|
|
vpshufd $0x4e,(%rsi),%xmm3
|
|
|
|
|
|
|
|
|
|
|
|
vpshufd $0xd3,%xmm3,%xmm0
|
|
vpsrad $31,%xmm0,%xmm0
|
|
vpaddq %xmm3,%xmm3,%xmm3
|
|
vpand L$gfpoly_and_internal_carrybit(%rip),%xmm0,%xmm0
|
|
vpxor %xmm0,%xmm3,%xmm3
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm6
|
|
|
|
|
|
vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0
|
|
vpclmulqdq $0x01,%xmm3,%xmm3,%xmm1
|
|
vpclmulqdq $0x10,%xmm3,%xmm3,%xmm2
|
|
vpxor %xmm2,%xmm1,%xmm1
|
|
vpclmulqdq $0x01,%xmm0,%xmm6,%xmm2
|
|
vpshufd $0x4e,%xmm0,%xmm0
|
|
vpxor %xmm0,%xmm1,%xmm1
|
|
vpxor %xmm2,%xmm1,%xmm1
|
|
vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5
|
|
vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0
|
|
vpshufd $0x4e,%xmm1,%xmm1
|
|
vpxor %xmm1,%xmm5,%xmm5
|
|
vpxor %xmm0,%xmm5,%xmm5
|
|
|
|
|
|
|
|
vinserti128 $1,%xmm3,%ymm5,%ymm3
|
|
vinserti128 $1,%xmm5,%ymm5,%ymm5
|
|
|
|
|
|
.byte 0xc4,0xe3,0x65,0x44,0xc5,0x00
|
|
.byte 0xc4,0xe3,0x65,0x44,0xcd,0x01
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd5,0x10
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xd0,0x01
|
|
vpshufd $0x4e,%ymm0,%ymm0
|
|
vpxor %ymm0,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x65,0x44,0xe5,0x11
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xc1,0x01
|
|
vpshufd $0x4e,%ymm1,%ymm1
|
|
vpxor %ymm1,%ymm4,%ymm4
|
|
vpxor %ymm0,%ymm4,%ymm4
|
|
|
|
|
|
|
|
vmovdqu %ymm3,96(%rdi)
|
|
vmovdqu %ymm4,64(%rdi)
|
|
|
|
|
|
|
|
vpunpcklqdq %ymm3,%ymm4,%ymm0
|
|
vpunpckhqdq %ymm3,%ymm4,%ymm1
|
|
vpxor %ymm1,%ymm0,%ymm0
|
|
vmovdqu %ymm0,128+32(%rdi)
|
|
|
|
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xc5,0x00
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xcd,0x01
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd5,0x10
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xd0,0x01
|
|
vpshufd $0x4e,%ymm0,%ymm0
|
|
vpxor %ymm0,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xdd,0x11
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xc1,0x01
|
|
vpshufd $0x4e,%ymm1,%ymm1
|
|
vpxor %ymm1,%ymm3,%ymm3
|
|
vpxor %ymm0,%ymm3,%ymm3
|
|
|
|
.byte 0xc4,0xe3,0x65,0x44,0xc5,0x00
|
|
.byte 0xc4,0xe3,0x65,0x44,0xcd,0x01
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd5,0x10
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xd0,0x01
|
|
vpshufd $0x4e,%ymm0,%ymm0
|
|
vpxor %ymm0,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
.byte 0xc4,0xe3,0x65,0x44,0xe5,0x11
|
|
.byte 0xc4,0xe3,0x4d,0x44,0xc1,0x01
|
|
vpshufd $0x4e,%ymm1,%ymm1
|
|
vpxor %ymm1,%ymm4,%ymm4
|
|
vpxor %ymm0,%ymm4,%ymm4
|
|
|
|
vmovdqu %ymm3,32(%rdi)
|
|
vmovdqu %ymm4,0(%rdi)
|
|
|
|
|
|
|
|
vpunpcklqdq %ymm3,%ymm4,%ymm0
|
|
vpunpckhqdq %ymm3,%ymm4,%ymm1
|
|
vpxor %ymm1,%ymm0,%ymm0
|
|
vmovdqu %ymm0,128(%rdi)
|
|
|
|
vzeroupper
|
|
ret
|
|
|
|
|
|
|
|
.globl _gcm_ghash_vpclmulqdq_avx2_1
|
|
.private_extern _gcm_ghash_vpclmulqdq_avx2_1
|
|
|
|
.p2align 5
|
|
_gcm_ghash_vpclmulqdq_avx2_1:
|
|
|
|
|
|
_CET_ENDBR
|
|
|
|
|
|
|
|
|
|
|
|
|
|
vmovdqu L$bswap_mask(%rip),%xmm6
|
|
vmovdqu L$gfpoly(%rip),%xmm7
|
|
|
|
|
|
vmovdqu (%rdi),%xmm5
|
|
vpshufb %xmm6,%xmm5,%xmm5
|
|
|
|
|
|
|
|
L$ghash_lastblock:
|
|
vmovdqu (%rdx),%xmm0
|
|
vpshufb %xmm6,%xmm0,%xmm0
|
|
vpxor %xmm0,%xmm5,%xmm5
|
|
vmovdqu 128-16(%rsi),%xmm0
|
|
vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1
|
|
vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2
|
|
vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3
|
|
vpxor %xmm3,%xmm2,%xmm2
|
|
vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3
|
|
vpshufd $0x4e,%xmm1,%xmm1
|
|
vpxor %xmm1,%xmm2,%xmm2
|
|
vpxor %xmm3,%xmm2,%xmm2
|
|
vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5
|
|
vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1
|
|
vpshufd $0x4e,%xmm2,%xmm2
|
|
vpxor %xmm2,%xmm5,%xmm5
|
|
vpxor %xmm1,%xmm5,%xmm5
|
|
|
|
|
|
L$ghash_done:
|
|
|
|
vpshufb %xmm6,%xmm5,%xmm5
|
|
vmovdqu %xmm5,(%rdi)
|
|
|
|
vzeroupper
|
|
ret
|
|
|
|
|
|
|
|
.globl _aes_gcm_enc_update_vaes_avx2
|
|
.private_extern _aes_gcm_enc_update_vaes_avx2
|
|
|
|
.p2align 5
|
|
_aes_gcm_enc_update_vaes_avx2:
|
|
|
|
|
|
_CET_ENDBR
|
|
pushq %r12
|
|
|
|
|
|
movq 16(%rsp),%r12
|
|
#ifdef BORINGSSL_DISPATCH_TEST
|
|
|
|
movb $1,_BORINGSSL_function_hit+8(%rip)
|
|
#endif
|
|
vbroadcasti128 L$bswap_mask(%rip),%ymm0
|
|
|
|
|
|
|
|
vmovdqu (%r12),%xmm1
|
|
vpshufb %xmm0,%xmm1,%xmm1
|
|
vbroadcasti128 (%r8),%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm11
|
|
|
|
|
|
|
|
movl 240(%rcx),%r10d
|
|
leal -20(,%r10,4),%r10d
|
|
|
|
|
|
|
|
|
|
leaq 96(%rcx,%r10,4),%r11
|
|
vbroadcasti128 (%rcx),%ymm9
|
|
vbroadcasti128 (%r11),%ymm10
|
|
|
|
|
|
vpaddd L$ctr_pattern(%rip),%ymm11,%ymm11
|
|
|
|
|
|
|
|
cmpq $127,%rdx
|
|
jbe L$crypt_loop_4x_done__func1
|
|
|
|
vmovdqu 128(%r9),%ymm7
|
|
vmovdqu 128+32(%r9),%ymm8
|
|
|
|
|
|
|
|
vmovdqu L$inc_2blocks(%rip),%ymm2
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm14
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm15
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
|
|
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
vpxor %ymm9,%ymm14,%ymm14
|
|
vpxor %ymm9,%ymm15,%ymm15
|
|
|
|
leaq 16(%rcx),%rax
|
|
L$vaesenc_loop_first_4_vecs__func1:
|
|
vbroadcasti128 (%rax),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
addq $16,%rax
|
|
cmpq %rax,%r11
|
|
jne L$vaesenc_loop_first_4_vecs__func1
|
|
vpxor 0(%rdi),%ymm10,%ymm2
|
|
vpxor 32(%rdi),%ymm10,%ymm3
|
|
vpxor 64(%rdi),%ymm10,%ymm5
|
|
vpxor 96(%rdi),%ymm10,%ymm6
|
|
.byte 0xc4,0x62,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdd,0xeb
|
|
.byte 0xc4,0x62,0x0d,0xdd,0xf5
|
|
.byte 0xc4,0x62,0x05,0xdd,0xfe
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %ymm13,32(%rsi)
|
|
vmovdqu %ymm14,64(%rsi)
|
|
vmovdqu %ymm15,96(%rsi)
|
|
|
|
subq $-128,%rdi
|
|
addq $-128,%rdx
|
|
cmpq $127,%rdx
|
|
jbe L$ghash_last_ciphertext_4x__func1
|
|
.p2align 4
|
|
L$crypt_loop_4x__func1:
|
|
|
|
|
|
|
|
|
|
vmovdqu L$inc_2blocks(%rip),%ymm2
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm14
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm15
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
|
|
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
vpxor %ymm9,%ymm14,%ymm14
|
|
vpxor %ymm9,%ymm15,%ymm15
|
|
|
|
cmpl $24,%r10d
|
|
jl L$aes128__func1
|
|
je L$aes192__func1
|
|
|
|
vbroadcasti128 -208(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vbroadcasti128 -192(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
L$aes192__func1:
|
|
vbroadcasti128 -176(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vbroadcasti128 -160(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
L$aes128__func1:
|
|
prefetcht0 512(%rdi)
|
|
prefetcht0 512+64(%rdi)
|
|
|
|
vmovdqu 0(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 0(%r9),%ymm4
|
|
vpxor %ymm1,%ymm3,%ymm3
|
|
.byte 0xc4,0xe3,0x65,0x44,0xec,0x00
|
|
.byte 0xc4,0xe3,0x65,0x44,0xcc,0x11
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xf7,0x00
|
|
|
|
vbroadcasti128 -144(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vbroadcasti128 -128(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vmovdqu 32(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 32(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xd7,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -112(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vmovdqu 64(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 64(%r9),%ymm4
|
|
|
|
vbroadcasti128 -96(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
|
|
vbroadcasti128 -80(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x00
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
|
|
vmovdqu 96(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
|
|
vbroadcasti128 -64(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vmovdqu 96(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -48(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm1,%ymm6,%ymm6
|
|
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm4
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd5,0x01
|
|
vpshufd $0x4e,%ymm5,%ymm5
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -32(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd6,0x01
|
|
vpshufd $0x4e,%ymm6,%ymm6
|
|
vpxor %ymm6,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
|
|
vbroadcasti128 -16(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vextracti128 $1,%ymm1,%xmm2
|
|
vpxor %xmm2,%xmm1,%xmm1
|
|
|
|
|
|
subq $-128,%rsi
|
|
vpxor 0(%rdi),%ymm10,%ymm2
|
|
vpxor 32(%rdi),%ymm10,%ymm3
|
|
vpxor 64(%rdi),%ymm10,%ymm5
|
|
vpxor 96(%rdi),%ymm10,%ymm6
|
|
.byte 0xc4,0x62,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdd,0xeb
|
|
.byte 0xc4,0x62,0x0d,0xdd,0xf5
|
|
.byte 0xc4,0x62,0x05,0xdd,0xfe
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %ymm13,32(%rsi)
|
|
vmovdqu %ymm14,64(%rsi)
|
|
vmovdqu %ymm15,96(%rsi)
|
|
|
|
subq $-128,%rdi
|
|
|
|
addq $-128,%rdx
|
|
cmpq $127,%rdx
|
|
ja L$crypt_loop_4x__func1
|
|
L$ghash_last_ciphertext_4x__func1:
|
|
|
|
vmovdqu 0(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 0(%r9),%ymm4
|
|
vpxor %ymm1,%ymm3,%ymm3
|
|
.byte 0xc4,0xe3,0x65,0x44,0xec,0x00
|
|
.byte 0xc4,0xe3,0x65,0x44,0xcc,0x11
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xf7,0x00
|
|
|
|
vmovdqu 32(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 32(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xd7,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vmovdqu 64(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 64(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x00
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
|
|
vmovdqu 96(%rsi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 96(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm1,%ymm6,%ymm6
|
|
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm4
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd5,0x01
|
|
vpshufd $0x4e,%ymm5,%ymm5
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd6,0x01
|
|
vpshufd $0x4e,%ymm6,%ymm6
|
|
vpxor %ymm6,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vextracti128 $1,%ymm1,%xmm2
|
|
vpxor %xmm2,%xmm1,%xmm1
|
|
|
|
subq $-128,%rsi
|
|
L$crypt_loop_4x_done__func1:
|
|
|
|
testq %rdx,%rdx
|
|
jz L$done__func1
|
|
|
|
|
|
|
|
|
|
|
|
leaq 128(%r9),%r8
|
|
subq %rdx,%r8
|
|
|
|
|
|
vpxor %xmm5,%xmm5,%xmm5
|
|
vpxor %xmm6,%xmm6,%xmm6
|
|
vpxor %xmm7,%xmm7,%xmm7
|
|
|
|
cmpq $64,%rdx
|
|
jb L$lessthan64bytes__func1
|
|
|
|
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
leaq 16(%rcx),%rax
|
|
L$vaesenc_loop_tail_1__func1:
|
|
vbroadcasti128 (%rax),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
addq $16,%rax
|
|
cmpq %rax,%r11
|
|
jne L$vaesenc_loop_tail_1__func1
|
|
.byte 0xc4,0x42,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x42,0x15,0xdd,0xea
|
|
|
|
|
|
vmovdqu 0(%rdi),%ymm2
|
|
vmovdqu 32(%rdi),%ymm3
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vpxor %ymm3,%ymm13,%ymm13
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %ymm13,32(%rsi)
|
|
|
|
|
|
vpshufb %ymm0,%ymm12,%ymm12
|
|
vpshufb %ymm0,%ymm13,%ymm13
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
vmovdqu 32(%r8),%ymm3
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xea,0x00
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xf2,0x01
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xfa,0x11
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x00
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x01
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x11
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
|
|
addq $64,%r8
|
|
addq $64,%rdi
|
|
addq $64,%rsi
|
|
subq $64,%rdx
|
|
jz L$reduce__func1
|
|
|
|
vpxor %xmm1,%xmm1,%xmm1
|
|
|
|
|
|
L$lessthan64bytes__func1:
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
leaq 16(%rcx),%rax
|
|
L$vaesenc_loop_tail_2__func1:
|
|
vbroadcasti128 (%rax),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
addq $16,%rax
|
|
cmpq %rax,%r11
|
|
jne L$vaesenc_loop_tail_2__func1
|
|
.byte 0xc4,0x42,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x42,0x15,0xdd,0xea
|
|
|
|
|
|
|
|
|
|
cmpq $32,%rdx
|
|
jb L$xor_one_block__func1
|
|
je L$xor_two_blocks__func1
|
|
|
|
L$xor_three_blocks__func1:
|
|
vmovdqu 0(%rdi),%ymm2
|
|
vmovdqu 32(%rdi),%xmm3
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vpxor %xmm3,%xmm13,%xmm13
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %xmm13,32(%rsi)
|
|
|
|
vpshufb %ymm0,%ymm12,%ymm12
|
|
vpshufb %xmm0,%xmm13,%xmm13
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
vmovdqu 32(%r8),%xmm3
|
|
vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
jmp L$ghash_mul_one_vec_unreduced__func1
|
|
|
|
L$xor_two_blocks__func1:
|
|
vmovdqu (%rdi),%ymm2
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vmovdqu %ymm12,(%rsi)
|
|
vpshufb %ymm0,%ymm12,%ymm12
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
jmp L$ghash_mul_one_vec_unreduced__func1
|
|
|
|
L$xor_one_block__func1:
|
|
vmovdqu (%rdi),%xmm2
|
|
vpxor %xmm2,%xmm12,%xmm12
|
|
vmovdqu %xmm12,(%rsi)
|
|
vpshufb %xmm0,%xmm12,%xmm12
|
|
vpxor %xmm1,%xmm12,%xmm12
|
|
vmovdqu (%r8),%xmm2
|
|
|
|
L$ghash_mul_one_vec_unreduced__func1:
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x00
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x01
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x11
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
|
|
L$reduce__func1:
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xdd,0x01
|
|
vpshufd $0x4e,%ymm5,%ymm5
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm3,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xde,0x01
|
|
vpshufd $0x4e,%ymm6,%ymm6
|
|
vpxor %ymm6,%ymm7,%ymm7
|
|
vpxor %ymm3,%ymm7,%ymm7
|
|
vextracti128 $1,%ymm7,%xmm1
|
|
vpxor %xmm7,%xmm1,%xmm1
|
|
|
|
L$done__func1:
|
|
|
|
vpshufb %xmm0,%xmm1,%xmm1
|
|
vmovdqu %xmm1,(%r12)
|
|
|
|
vzeroupper
|
|
popq %r12
|
|
|
|
ret
|
|
|
|
|
|
|
|
.globl _aes_gcm_dec_update_vaes_avx2
|
|
.private_extern _aes_gcm_dec_update_vaes_avx2
|
|
|
|
.p2align 5
|
|
_aes_gcm_dec_update_vaes_avx2:
|
|
|
|
|
|
_CET_ENDBR
|
|
pushq %r12
|
|
|
|
|
|
movq 16(%rsp),%r12
|
|
vbroadcasti128 L$bswap_mask(%rip),%ymm0
|
|
|
|
|
|
|
|
vmovdqu (%r12),%xmm1
|
|
vpshufb %xmm0,%xmm1,%xmm1
|
|
vbroadcasti128 (%r8),%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm11
|
|
|
|
|
|
|
|
movl 240(%rcx),%r10d
|
|
leal -20(,%r10,4),%r10d
|
|
|
|
|
|
|
|
|
|
leaq 96(%rcx,%r10,4),%r11
|
|
vbroadcasti128 (%rcx),%ymm9
|
|
vbroadcasti128 (%r11),%ymm10
|
|
|
|
|
|
vpaddd L$ctr_pattern(%rip),%ymm11,%ymm11
|
|
|
|
|
|
|
|
cmpq $127,%rdx
|
|
jbe L$crypt_loop_4x_done__func2
|
|
|
|
vmovdqu 128(%r9),%ymm7
|
|
vmovdqu 128+32(%r9),%ymm8
|
|
.p2align 4
|
|
L$crypt_loop_4x__func2:
|
|
|
|
|
|
|
|
|
|
vmovdqu L$inc_2blocks(%rip),%ymm2
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm14
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm15
|
|
vpaddd %ymm2,%ymm11,%ymm11
|
|
|
|
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
vpxor %ymm9,%ymm14,%ymm14
|
|
vpxor %ymm9,%ymm15,%ymm15
|
|
|
|
cmpl $24,%r10d
|
|
jl L$aes128__func2
|
|
je L$aes192__func2
|
|
|
|
vbroadcasti128 -208(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vbroadcasti128 -192(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
L$aes192__func2:
|
|
vbroadcasti128 -176(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vbroadcasti128 -160(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
L$aes128__func2:
|
|
prefetcht0 512(%rdi)
|
|
prefetcht0 512+64(%rdi)
|
|
|
|
vmovdqu 0(%rdi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 0(%r9),%ymm4
|
|
vpxor %ymm1,%ymm3,%ymm3
|
|
.byte 0xc4,0xe3,0x65,0x44,0xec,0x00
|
|
.byte 0xc4,0xe3,0x65,0x44,0xcc,0x11
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xf7,0x00
|
|
|
|
vbroadcasti128 -144(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vbroadcasti128 -128(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vmovdqu 32(%rdi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 32(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xd7,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -112(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vmovdqu 64(%rdi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
vmovdqu 64(%r9),%ymm4
|
|
|
|
vbroadcasti128 -96(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
|
|
vbroadcasti128 -80(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x00
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
|
|
vmovdqu 96(%rdi),%ymm3
|
|
vpshufb %ymm0,%ymm3,%ymm3
|
|
|
|
vbroadcasti128 -64(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vmovdqu 96(%r9),%ymm4
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x00
|
|
vpxor %ymm2,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x65,0x44,0xd4,0x11
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
vpunpckhqdq %ymm3,%ymm3,%ymm2
|
|
vpxor %ymm3,%ymm2,%ymm2
|
|
.byte 0xc4,0xc3,0x6d,0x44,0xd0,0x10
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -48(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm1,%ymm6,%ymm6
|
|
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm4
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd5,0x01
|
|
vpshufd $0x4e,%ymm5,%ymm5
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm2,%ymm6,%ymm6
|
|
|
|
vbroadcasti128 -32(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
|
|
.byte 0xc4,0xe3,0x5d,0x44,0xd6,0x01
|
|
vpshufd $0x4e,%ymm6,%ymm6
|
|
vpxor %ymm6,%ymm1,%ymm1
|
|
vpxor %ymm2,%ymm1,%ymm1
|
|
|
|
vbroadcasti128 -16(%r11),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
.byte 0xc4,0x62,0x0d,0xdc,0xf2
|
|
.byte 0xc4,0x62,0x05,0xdc,0xfa
|
|
|
|
vextracti128 $1,%ymm1,%xmm2
|
|
vpxor %xmm2,%xmm1,%xmm1
|
|
|
|
|
|
|
|
vpxor 0(%rdi),%ymm10,%ymm2
|
|
vpxor 32(%rdi),%ymm10,%ymm3
|
|
vpxor 64(%rdi),%ymm10,%ymm5
|
|
vpxor 96(%rdi),%ymm10,%ymm6
|
|
.byte 0xc4,0x62,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdd,0xeb
|
|
.byte 0xc4,0x62,0x0d,0xdd,0xf5
|
|
.byte 0xc4,0x62,0x05,0xdd,0xfe
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %ymm13,32(%rsi)
|
|
vmovdqu %ymm14,64(%rsi)
|
|
vmovdqu %ymm15,96(%rsi)
|
|
|
|
subq $-128,%rdi
|
|
subq $-128,%rsi
|
|
addq $-128,%rdx
|
|
cmpq $127,%rdx
|
|
ja L$crypt_loop_4x__func2
|
|
L$crypt_loop_4x_done__func2:
|
|
|
|
testq %rdx,%rdx
|
|
jz L$done__func2
|
|
|
|
|
|
|
|
|
|
|
|
leaq 128(%r9),%r8
|
|
subq %rdx,%r8
|
|
|
|
|
|
vpxor %xmm5,%xmm5,%xmm5
|
|
vpxor %xmm6,%xmm6,%xmm6
|
|
vpxor %xmm7,%xmm7,%xmm7
|
|
|
|
cmpq $64,%rdx
|
|
jb L$lessthan64bytes__func2
|
|
|
|
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
leaq 16(%rcx),%rax
|
|
L$vaesenc_loop_tail_1__func2:
|
|
vbroadcasti128 (%rax),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
addq $16,%rax
|
|
cmpq %rax,%r11
|
|
jne L$vaesenc_loop_tail_1__func2
|
|
.byte 0xc4,0x42,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x42,0x15,0xdd,0xea
|
|
|
|
|
|
vmovdqu 0(%rdi),%ymm2
|
|
vmovdqu 32(%rdi),%ymm3
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vpxor %ymm3,%ymm13,%ymm13
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %ymm13,32(%rsi)
|
|
|
|
|
|
vpshufb %ymm0,%ymm2,%ymm12
|
|
vpshufb %ymm0,%ymm3,%ymm13
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
vmovdqu 32(%r8),%ymm3
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xea,0x00
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xf2,0x01
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xfa,0x11
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x00
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x01
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x15,0x44,0xe3,0x11
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
|
|
addq $64,%r8
|
|
addq $64,%rdi
|
|
addq $64,%rsi
|
|
subq $64,%rdx
|
|
jz L$reduce__func2
|
|
|
|
vpxor %xmm1,%xmm1,%xmm1
|
|
|
|
|
|
L$lessthan64bytes__func2:
|
|
vpshufb %ymm0,%ymm11,%ymm12
|
|
vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11
|
|
vpshufb %ymm0,%ymm11,%ymm13
|
|
vpxor %ymm9,%ymm12,%ymm12
|
|
vpxor %ymm9,%ymm13,%ymm13
|
|
leaq 16(%rcx),%rax
|
|
L$vaesenc_loop_tail_2__func2:
|
|
vbroadcasti128 (%rax),%ymm2
|
|
.byte 0xc4,0x62,0x1d,0xdc,0xe2
|
|
.byte 0xc4,0x62,0x15,0xdc,0xea
|
|
addq $16,%rax
|
|
cmpq %rax,%r11
|
|
jne L$vaesenc_loop_tail_2__func2
|
|
.byte 0xc4,0x42,0x1d,0xdd,0xe2
|
|
.byte 0xc4,0x42,0x15,0xdd,0xea
|
|
|
|
|
|
|
|
|
|
cmpq $32,%rdx
|
|
jb L$xor_one_block__func2
|
|
je L$xor_two_blocks__func2
|
|
|
|
L$xor_three_blocks__func2:
|
|
vmovdqu 0(%rdi),%ymm2
|
|
vmovdqu 32(%rdi),%xmm3
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vpxor %xmm3,%xmm13,%xmm13
|
|
vmovdqu %ymm12,0(%rsi)
|
|
vmovdqu %xmm13,32(%rsi)
|
|
|
|
vpshufb %ymm0,%ymm2,%ymm12
|
|
vpshufb %xmm0,%xmm3,%xmm13
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
vmovdqu 32(%r8),%xmm3
|
|
vpclmulqdq $0x00,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
vpclmulqdq $0x01,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
vpclmulqdq $0x10,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
vpclmulqdq $0x11,%xmm3,%xmm13,%xmm4
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
jmp L$ghash_mul_one_vec_unreduced__func2
|
|
|
|
L$xor_two_blocks__func2:
|
|
vmovdqu (%rdi),%ymm2
|
|
vpxor %ymm2,%ymm12,%ymm12
|
|
vmovdqu %ymm12,(%rsi)
|
|
vpshufb %ymm0,%ymm2,%ymm12
|
|
vpxor %ymm1,%ymm12,%ymm12
|
|
vmovdqu (%r8),%ymm2
|
|
jmp L$ghash_mul_one_vec_unreduced__func2
|
|
|
|
L$xor_one_block__func2:
|
|
vmovdqu (%rdi),%xmm2
|
|
vpxor %xmm2,%xmm12,%xmm12
|
|
vmovdqu %xmm12,(%rsi)
|
|
vpshufb %xmm0,%xmm2,%xmm12
|
|
vpxor %xmm1,%xmm12,%xmm12
|
|
vmovdqu (%r8),%xmm2
|
|
|
|
L$ghash_mul_one_vec_unreduced__func2:
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x00
|
|
vpxor %ymm4,%ymm5,%ymm5
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x01
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x10
|
|
vpxor %ymm4,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x1d,0x44,0xe2,0x11
|
|
vpxor %ymm4,%ymm7,%ymm7
|
|
|
|
L$reduce__func2:
|
|
|
|
vbroadcasti128 L$gfpoly(%rip),%ymm2
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xdd,0x01
|
|
vpshufd $0x4e,%ymm5,%ymm5
|
|
vpxor %ymm5,%ymm6,%ymm6
|
|
vpxor %ymm3,%ymm6,%ymm6
|
|
.byte 0xc4,0xe3,0x6d,0x44,0xde,0x01
|
|
vpshufd $0x4e,%ymm6,%ymm6
|
|
vpxor %ymm6,%ymm7,%ymm7
|
|
vpxor %ymm3,%ymm7,%ymm7
|
|
vextracti128 $1,%ymm7,%xmm1
|
|
vpxor %xmm7,%xmm1,%xmm1
|
|
|
|
L$done__func2:
|
|
|
|
vpshufb %xmm0,%xmm1,%xmm1
|
|
vmovdqu %xmm1,(%r12)
|
|
|
|
vzeroupper
|
|
popq %r12
|
|
|
|
ret
|
|
|
|
|
|
|
|
#endif
|