[pcsx_rearmed.git] / deps / libchdr / deps / lzma-22.01 / src / Asm / x86 / AesOpt.asm

; AesOpt.asm -- AES optimized code for x86 AES hardware instructions\r
; 2021-12-25 : Igor Pavlov : Public domain\r
\r
include 7zAsm.asm\r
\r
ifdef __ASMC__\r
  use_vaes_256 equ 1\r
else\r
ifdef ymm0\r
  use_vaes_256 equ 1\r
endif\r
endif\r
\r
\r
ifdef use_vaes_256\r
  ECHO "++ VAES 256"\r
else\r
  ECHO "-- NO VAES 256"\r
endif\r
\r
ifdef x64\r
  ECHO "x86-64"\r
else\r
  ECHO "x86"\r
if (IS_CDECL gt 0)\r
  ECHO "ABI : CDECL"\r
else\r
  ECHO "ABI : no CDECL : FASTCALL"\r
endif\r
endif\r
\r
if (IS_LINUX gt 0)\r
  ECHO "ABI : LINUX"\r
else\r
  ECHO "ABI : WINDOWS"\r
endif\r
\r
MY_ASM_START\r
\r
ifndef x64\r
    .686\r
    .xmm\r
endif\r
\r
\r
; MY_ALIGN EQU ALIGN(64)\r
MY_ALIGN EQU\r
\r
SEG_ALIGN EQU MY_ALIGN\r
\r
MY_SEG_PROC macro name:req, numParams:req\r
    ; seg_name equ @CatStr(_TEXT$, name)\r
    ; seg_name SEGMENT SEG_ALIGN 'CODE'\r
    MY_PROC name, numParams\r
endm\r
\r
MY_SEG_ENDP macro\r
    ; seg_name ENDS\r
endm\r
\r
\r
NUM_AES_KEYS_MAX equ 15\r
\r
; the number of push operators in function PROLOG\r
if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
num_regs_push   equ 2\r
stack_param_offset equ (REG_SIZE * (1 + num_regs_push))\r
endif\r
\r
ifdef x64\r
    num_param   equ REG_ABI_PARAM_2\r
else\r
  if (IS_CDECL gt 0)\r
    ;   size_t     size\r
    ;   void *     data\r
    ;   UInt32 *   aes\r
    ;   ret-ip <- (r4)\r
    aes_OFFS    equ (stack_param_offset)\r
    data_OFFS   equ (REG_SIZE + aes_OFFS)\r
    size_OFFS   equ (REG_SIZE + data_OFFS)\r
    num_param   equ [r4 + size_OFFS]\r
  else\r
    num_param   equ [r4 + stack_param_offset]\r
  endif\r
endif\r
\r
keys    equ  REG_PARAM_0  ; r1\r
rD      equ  REG_PARAM_1  ; r2\r
rN      equ  r0\r
\r
koffs_x equ  x7\r
koffs_r equ  r7\r
\r
ksize_x equ  x6\r
ksize_r equ  r6\r
\r
keys2   equ  r3\r
\r
state   equ  xmm0\r
key     equ  xmm0\r
key_ymm equ  ymm0\r
key_ymm_n equ   0\r
\r
ifdef x64\r
        ways = 11\r
else\r
        ways = 4\r
endif\r
\r
ways_start_reg equ 1\r
\r
iv      equ     @CatStr(xmm, %(ways_start_reg + ways))\r
iv_ymm  equ     @CatStr(ymm, %(ways_start_reg + ways))\r
\r
\r
WOP macro op, op2\r
    i = 0\r
    rept ways\r
        op      @CatStr(xmm, %(ways_start_reg + i)), op2\r
        i = i + 1\r
    endm\r
endm\r
\r
\r
ifndef ABI_LINUX\r
ifdef x64\r
\r
; we use 32 bytes of home space in stack in WIN64-x64\r
NUM_HOME_MM_REGS   equ (32 / 16)\r
; we preserve xmm registers starting from xmm6 in WIN64-x64\r
MM_START_SAVE_REG  equ 6\r
\r
SAVE_XMM macro num_used_mm_regs:req\r
  num_save_mm_regs = num_used_mm_regs - MM_START_SAVE_REG\r
  if num_save_mm_regs GT 0\r
    num_save_mm_regs2 = num_save_mm_regs - NUM_HOME_MM_REGS\r
    ; RSP is (16*x + 8) after entering the function in WIN64-x64\r
    stack_offset = 16 * num_save_mm_regs2 + (stack_param_offset mod 16)\r
  \r
    i = 0\r
    rept num_save_mm_regs\r
      \r
      if i eq NUM_HOME_MM_REGS\r
        sub  r4, stack_offset\r
      endif\r
      \r
      if i lt NUM_HOME_MM_REGS\r
        movdqa  [r4 + stack_param_offset + i * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i))\r
      else\r
        movdqa  [r4 + (i - NUM_HOME_MM_REGS) * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i))\r
      endif\r
      \r
      i = i + 1\r
    endm\r
  endif\r
endm\r
\r
RESTORE_XMM macro num_used_mm_regs:req\r
  if num_save_mm_regs GT 0\r
    i = 0\r
    if num_save_mm_regs2 GT 0\r
      rept num_save_mm_regs2\r
        movdqa  @CatStr(xmm, %(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i)), [r4 + i * 16]\r
        i = i + 1\r
      endm\r
        add     r4, stack_offset\r
    endif\r
\r
    num_low_regs = num_save_mm_regs - i\r
    i = 0\r
      rept num_low_regs\r
        movdqa  @CatStr(xmm, %(MM_START_SAVE_REG + i)), [r4 + stack_param_offset + i * 16]\r
        i = i + 1\r
      endm\r
  endif\r
endm\r
\r
endif ; x64\r
endif ; ABI_LINUX\r
\r
\r
MY_PROLOG macro num_used_mm_regs:req\r
        ; num_regs_push: must be equal to the number of push operators\r
        ; push    r3\r
        ; push    r5\r
    if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
        push    r6\r
        push    r7\r
    endif\r
\r
        mov     rN, num_param  ; don't move it; num_param can use stack pointer (r4)\r
\r
    if (IS_X64 eq 0)\r
      if (IS_CDECL gt 0)\r
        mov     rD,   [r4 + data_OFFS]\r
        mov     keys, [r4 + aes_OFFS]\r
      endif\r
    elseif (IS_LINUX gt 0)\r
        MY_ABI_LINUX_TO_WIN_2\r
    endif\r
\r
\r
    ifndef ABI_LINUX\r
    ifdef x64\r
        SAVE_XMM num_used_mm_regs\r
    endif\r
    endif\r
   \r
        mov     ksize_x, [keys + 16]\r
        shl     ksize_x, 5\r
endm\r
\r
\r
MY_EPILOG macro\r
    ifndef ABI_LINUX\r
    ifdef x64\r
        RESTORE_XMM num_save_mm_regs\r
    endif\r
    endif\r
    \r
    if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
        pop     r7\r
        pop     r6\r
    endif\r
        ; pop     r5\r
        ; pop     r3\r
    MY_ENDP\r
endm\r
\r
\r
OP_KEY macro op:req, offs:req\r
        op      state, [keys + offs]\r
endm\r
\r
 \r
WOP_KEY macro op:req, offs:req\r
        movdqa  key, [keys + offs]\r
        WOP     op, key\r
endm\r
\r
\r
; ---------- AES-CBC Decode ----------\r
\r
\r
XOR_WITH_DATA macro reg, _ppp_\r
        pxor    reg, [rD + i * 16]\r
endm\r
\r
WRITE_TO_DATA macro reg, _ppp_\r
        movdqa  [rD + i * 16], reg\r
endm\r
\r
\r
; state0    equ  @CatStr(xmm, %(ways_start_reg))\r
\r
key0            equ  @CatStr(xmm, %(ways_start_reg + ways + 1))\r
key0_ymm        equ  @CatStr(ymm, %(ways_start_reg + ways + 1))\r
\r
key_last        equ  @CatStr(xmm, %(ways_start_reg + ways + 2))\r
key_last_ymm    equ  @CatStr(ymm, %(ways_start_reg + ways + 2))\r
key_last_ymm_n  equ                (ways_start_reg + ways + 2)\r
\r
NUM_CBC_REGS    equ  (ways_start_reg + ways + 3)\r
\r
\r
MY_SEG_PROC AesCbc_Decode_HW, 3\r
\r
    AesCbc_Decode_HW_start::\r
        MY_PROLOG NUM_CBC_REGS\r
   \r
    AesCbc_Decode_HW_start_2::\r
        movdqa  iv, [keys]\r
        add     keys, 32\r
\r
        movdqa  key0, [keys + 1 * ksize_r]\r
        movdqa  key_last, [keys]\r
        sub     ksize_x, 16\r
\r
        jmp     check2\r
    align 16\r
    nextBlocks2:\r
        WOP     movdqa, [rD + i * 16]\r
        mov     koffs_x, ksize_x\r
        ; WOP_KEY pxor, ksize_r + 16\r
        WOP     pxor, key0\r
    ; align 16\r
    @@:\r
        WOP_KEY aesdec, 1 * koffs_r\r
        sub     koffs_r, 16\r
        jnz     @B\r
        ; WOP_KEY aesdeclast, 0\r
        WOP     aesdeclast, key_last\r
        \r
        pxor    @CatStr(xmm, %(ways_start_reg)), iv\r
    i = 1\r
    rept ways - 1\r
        pxor    @CatStr(xmm, %(ways_start_reg + i)), [rD + i * 16 - 16]\r
        i = i + 1\r
    endm\r
        movdqa  iv, [rD + ways * 16 - 16]\r
        WOP     WRITE_TO_DATA\r
\r
        add     rD, ways * 16\r
    AesCbc_Decode_HW_start_3::\r
    check2:\r
        sub     rN, ways\r
        jnc     nextBlocks2\r
        add     rN, ways\r
\r
        sub     ksize_x, 16\r
\r
        jmp     check\r
    nextBlock:\r
        movdqa  state, [rD]\r
        mov     koffs_x, ksize_x\r
        ; OP_KEY  pxor, 1 * ksize_r + 32\r
        pxor    state, key0\r
        ; movdqa  state0, [rD]\r
        ; movdqa  state, key0\r
        ; pxor    state, state0\r
    @@:\r
        OP_KEY  aesdec, 1 * koffs_r + 16\r
        OP_KEY  aesdec, 1 * koffs_r\r
        sub     koffs_r, 32\r
        jnz     @B\r
        OP_KEY  aesdec, 16\r
        ; OP_KEY  aesdeclast, 0\r
        aesdeclast state, key_last\r
        \r
        pxor    state, iv\r
        movdqa  iv, [rD]\r
        ; movdqa  iv, state0\r
        movdqa  [rD], state\r
        \r
        add     rD, 16\r
    check:\r
        sub     rN, 1\r
        jnc     nextBlock\r
\r
        movdqa  [keys - 32], iv\r
MY_EPILOG\r
\r
\r
\r
\r
; ---------- AVX ----------\r
\r
\r
AVX__WOP_n macro op\r
    i = 0\r
    rept ways\r
        op      (ways_start_reg + i)\r
        i = i + 1\r
    endm\r
endm\r
\r
AVX__WOP macro op\r
    i = 0\r
    rept ways\r
        op      @CatStr(ymm, %(ways_start_reg + i))\r
        i = i + 1\r
    endm\r
endm\r
\r
\r
AVX__WOP_KEY macro op:req, offs:req\r
        vmovdqa  key_ymm, ymmword ptr [keys2 + offs]\r
        AVX__WOP_n op\r
endm\r
\r
\r
AVX__CBC_START macro reg\r
        ; vpxor   reg, key_ymm, ymmword ptr [rD + 32 * i]\r
        vpxor   reg, key0_ymm, ymmword ptr [rD + 32 * i]\r
endm\r
\r
AVX__CBC_END macro reg\r
    if i eq 0\r
        vpxor   reg, reg, iv_ymm\r
    else\r
        vpxor   reg, reg, ymmword ptr [rD + i * 32 - 16]\r
    endif\r
endm\r
\r
\r
AVX__WRITE_TO_DATA macro reg\r
        vmovdqu ymmword ptr [rD + 32 * i], reg\r
endm\r
\r
AVX__XOR_WITH_DATA macro reg\r
        vpxor   reg, reg, ymmword ptr [rD + 32 * i]\r
endm\r
\r
AVX__CTR_START macro reg\r
        vpaddq  iv_ymm, iv_ymm, one_ymm\r
        ; vpxor   reg, iv_ymm, key_ymm\r
        vpxor   reg, iv_ymm, key0_ymm\r
endm\r
\r
\r
MY_VAES_INSTR_2 macro cmd, dest, a1, a2\r
  db 0c4H\r
  db 2 + 040H + 020h * (1 - (a2) / 8) + 080h * (1 - (dest) / 8)\r
  db 5 + 8 * ((not (a1)) and 15)\r
  db cmd\r
  db 0c0H + 8 * ((dest) and 7) + ((a2) and 7)\r
endm\r
\r
MY_VAES_INSTR macro cmd, dest, a\r
        MY_VAES_INSTR_2  cmd, dest, dest, a\r
endm\r
\r
MY_vaesenc macro dest, a\r
        MY_VAES_INSTR  0dcH, dest, a\r
endm\r
MY_vaesenclast macro dest, a\r
        MY_VAES_INSTR  0ddH, dest, a\r
endm\r
MY_vaesdec macro dest, a\r
        MY_VAES_INSTR  0deH, dest, a\r
endm\r
MY_vaesdeclast macro dest, a\r
        MY_VAES_INSTR  0dfH, dest, a\r
endm\r
\r
\r
AVX__VAES_DEC macro reg\r
        MY_vaesdec reg, key_ymm_n\r
endm\r
\r
AVX__VAES_DEC_LAST_key_last macro reg\r
        ; MY_vaesdeclast reg, key_ymm_n\r
        MY_vaesdeclast reg, key_last_ymm_n\r
endm\r
\r
AVX__VAES_ENC macro reg\r
        MY_vaesenc reg, key_ymm_n\r
endm\r
\r
AVX__VAES_ENC_LAST macro reg\r
        MY_vaesenclast reg, key_ymm_n\r
endm\r
\r
AVX__vinserti128_TO_HIGH macro dest, src\r
        vinserti128  dest, dest, src, 1\r
endm\r
\r
\r
MY_PROC AesCbc_Decode_HW_256, 3\r
  ifdef use_vaes_256\r
        MY_PROLOG NUM_CBC_REGS\r
        \r
        cmp    rN, ways * 2\r
        jb     AesCbc_Decode_HW_start_2\r
\r
        vmovdqa iv, xmmword ptr [keys]\r
        add     keys, 32\r
\r
        vbroadcasti128  key0_ymm, xmmword ptr [keys + 1 * ksize_r]\r
        vbroadcasti128  key_last_ymm, xmmword ptr [keys]\r
        sub     ksize_x, 16\r
        mov     koffs_x, ksize_x\r
        add     ksize_x, ksize_x\r
        \r
        AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 2) * 32)\r
        push    keys2\r
        sub     r4, AVX_STACK_SUB\r
        ; sub     r4, 32\r
        ; sub     r4, ksize_r\r
        ; lea     keys2, [r4 + 32]\r
        mov     keys2, r4\r
        and     keys2, -32\r
    broad:\r
        vbroadcasti128  key_ymm, xmmword ptr [keys + 1 * koffs_r]\r
        vmovdqa         ymmword ptr [keys2 + koffs_r * 2], key_ymm\r
        sub     koffs_r, 16\r
        ; jnc     broad\r
        jnz     broad\r
\r
        sub     rN, ways * 2\r
\r
    align 16\r
    avx_cbcdec_nextBlock2:\r
        mov     koffs_x, ksize_x\r
        ; AVX__WOP_KEY    AVX__CBC_START, 1 * koffs_r + 32\r
        AVX__WOP    AVX__CBC_START\r
    @@:\r
        AVX__WOP_KEY    AVX__VAES_DEC, 1 * koffs_r\r
        sub     koffs_r, 32\r
        jnz     @B\r
        ; AVX__WOP_KEY    AVX__VAES_DEC_LAST, 0\r
        AVX__WOP_n   AVX__VAES_DEC_LAST_key_last\r
\r
        AVX__vinserti128_TO_HIGH  iv_ymm, xmmword ptr [rD]\r
        AVX__WOP        AVX__CBC_END\r
\r
        vmovdqa         iv, xmmword ptr [rD + ways * 32 - 16]\r
        AVX__WOP        AVX__WRITE_TO_DATA\r
        \r
        add     rD, ways * 32\r
        sub     rN, ways * 2\r
        jnc     avx_cbcdec_nextBlock2\r
        add     rN, ways * 2\r
\r
        shr     ksize_x, 1\r
        \r
        ; lea     r4, [r4 + 1 * ksize_r + 32]\r
        add     r4, AVX_STACK_SUB\r
        pop     keys2\r
\r
        vzeroupper\r
        jmp     AesCbc_Decode_HW_start_3\r
  else\r
        jmp     AesCbc_Decode_HW_start\r
  endif\r
MY_ENDP\r
MY_SEG_ENDP\r
\r
\r
\r
    \r
; ---------- AES-CBC Encode ----------\r
\r
e0  equ  xmm1\r
\r
CENC_START_KEY     equ 2\r
CENC_NUM_REG_KEYS  equ (3 * 2)\r
; last_key equ @CatStr(xmm, %(CENC_START_KEY + CENC_NUM_REG_KEYS))\r
\r
MY_SEG_PROC AesCbc_Encode_HW, 3\r
        MY_PROLOG (CENC_START_KEY + CENC_NUM_REG_KEYS + 0)\r
\r
        movdqa  state, [keys]\r
        add     keys, 32\r
        \r
    i = 0\r
    rept CENC_NUM_REG_KEYS\r
        movdqa  @CatStr(xmm, %(CENC_START_KEY + i)), [keys + i * 16]\r
        i = i + 1\r
    endm\r
                                        \r
        add     keys, ksize_r\r
        neg     ksize_r\r
        add     ksize_r, (16 * CENC_NUM_REG_KEYS)\r
        ; movdqa  last_key, [keys]\r
        jmp     check_e\r
\r
    align 16\r
    nextBlock_e:\r
        movdqa  e0, [rD]\r
        mov     koffs_r, ksize_r\r
        pxor    e0, @CatStr(xmm, %(CENC_START_KEY))\r
        pxor    state, e0\r
        \r
    i = 1\r
    rept (CENC_NUM_REG_KEYS - 1)\r
        aesenc  state, @CatStr(xmm, %(CENC_START_KEY + i))\r
        i = i + 1\r
    endm\r
\r
    @@:\r
        OP_KEY  aesenc, 1 * koffs_r\r
        OP_KEY  aesenc, 1 * koffs_r + 16\r
        add     koffs_r, 32\r
        jnz     @B\r
        OP_KEY  aesenclast, 0\r
        ; aesenclast state, last_key\r
        \r
        movdqa  [rD], state\r
        add     rD, 16\r
    check_e:\r
        sub     rN, 1\r
        jnc     nextBlock_e\r
\r
        ; movdqa  [keys - 32], state\r
        movdqa  [keys + 1 * ksize_r - (16 * CENC_NUM_REG_KEYS) - 32], state\r
MY_EPILOG\r
MY_SEG_ENDP\r
\r
\r
    \r
; ---------- AES-CTR ----------\r
\r
ifdef x64\r
        ; ways = 11\r
endif\r
\r
       \r
one             equ  @CatStr(xmm, %(ways_start_reg + ways + 1))\r
one_ymm         equ  @CatStr(ymm, %(ways_start_reg + ways + 1))\r
key0            equ  @CatStr(xmm, %(ways_start_reg + ways + 2))\r
key0_ymm        equ  @CatStr(ymm, %(ways_start_reg + ways + 2))\r
NUM_CTR_REGS    equ  (ways_start_reg + ways + 3)\r
\r
INIT_CTR macro reg, _ppp_\r
        paddq   iv, one\r
        movdqa  reg, iv\r
endm\r
\r
\r
MY_SEG_PROC AesCtr_Code_HW, 3\r
    Ctr_start::\r
        MY_PROLOG NUM_CTR_REGS\r
\r
    Ctr_start_2::\r
        movdqa  iv, [keys]\r
        add     keys, 32\r
        movdqa  key0, [keys]\r
\r
        add     keys, ksize_r\r
        neg     ksize_r\r
        add     ksize_r, 16\r
        \r
    Ctr_start_3::\r
        mov     koffs_x, 1\r
        movd    one, koffs_x\r
        jmp     check2_c\r
\r
    align 16\r
    nextBlocks2_c:\r
        WOP     INIT_CTR, 0\r
        mov     koffs_r, ksize_r\r
        ; WOP_KEY pxor, 1 * koffs_r -16\r
        WOP     pxor, key0\r
    @@:\r
        WOP_KEY aesenc, 1 * koffs_r\r
        add     koffs_r, 16\r
        jnz     @B\r
        WOP_KEY aesenclast, 0\r
        \r
        WOP     XOR_WITH_DATA\r
        WOP     WRITE_TO_DATA\r
        add     rD, ways * 16\r
    check2_c:\r
        sub     rN, ways\r
        jnc     nextBlocks2_c\r
        add     rN, ways\r
\r
        sub     keys, 16\r
        add     ksize_r, 16\r
        \r
        jmp     check_c\r
\r
    ; align 16\r
    nextBlock_c:\r
        paddq   iv, one\r
        ; movdqa  state, [keys + 1 * koffs_r - 16]\r
        movdqa  state, key0\r
        mov     koffs_r, ksize_r\r
        pxor    state, iv\r
        \r
    @@:\r
        OP_KEY  aesenc, 1 * koffs_r\r
        OP_KEY  aesenc, 1 * koffs_r + 16\r
        add     koffs_r, 32\r
        jnz     @B\r
        OP_KEY  aesenc, 0\r
        OP_KEY  aesenclast, 16\r
        \r
        pxor    state, [rD]\r
        movdqa  [rD], state\r
        add     rD, 16\r
    check_c:\r
        sub     rN, 1\r
        jnc     nextBlock_c\r
\r
        ; movdqa  [keys - 32], iv\r
        movdqa  [keys + 1 * ksize_r - 16 - 32], iv\r
MY_EPILOG\r
\r
\r
MY_PROC AesCtr_Code_HW_256, 3\r
  ifdef use_vaes_256\r
        MY_PROLOG NUM_CTR_REGS\r
\r
        cmp    rN, ways * 2\r
        jb     Ctr_start_2\r
\r
        vbroadcasti128  iv_ymm, xmmword ptr [keys]\r
        add     keys, 32\r
        vbroadcasti128  key0_ymm, xmmword ptr [keys]\r
        mov     koffs_x, 1\r
        vmovd           one, koffs_x\r
        vpsubq  iv_ymm, iv_ymm, one_ymm\r
        vpaddq  one, one, one\r
        AVX__vinserti128_TO_HIGH     one_ymm, one\r
        \r
        add     keys, ksize_r\r
        sub     ksize_x, 16\r
        neg     ksize_r\r
        mov     koffs_r, ksize_r\r
        add     ksize_r, ksize_r\r
\r
        AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 1) * 32)\r
        push    keys2\r
        lea     keys2, [r4 - 32]\r
        sub     r4, AVX_STACK_SUB\r
        and     keys2, -32\r
        vbroadcasti128  key_ymm, xmmword ptr [keys]\r
        vmovdqa         ymmword ptr [keys2], key_ymm\r
     @@:\r
        vbroadcasti128  key_ymm, xmmword ptr [keys + 1 * koffs_r]\r
        vmovdqa         ymmword ptr [keys2 + koffs_r * 2], key_ymm\r
        add     koffs_r, 16\r
        jnz     @B\r
\r
        sub     rN, ways * 2\r
        \r
    align 16\r
    avx_ctr_nextBlock2:\r
        mov             koffs_r, ksize_r\r
        AVX__WOP        AVX__CTR_START\r
        ; AVX__WOP_KEY    AVX__CTR_START, 1 * koffs_r - 32\r
    @@:\r
        AVX__WOP_KEY    AVX__VAES_ENC, 1 * koffs_r\r
        add     koffs_r, 32\r
        jnz     @B\r
        AVX__WOP_KEY    AVX__VAES_ENC_LAST, 0\r
       \r
        AVX__WOP        AVX__XOR_WITH_DATA\r
        AVX__WOP        AVX__WRITE_TO_DATA\r
        \r
        add     rD, ways * 32\r
        sub     rN, ways * 2\r
        jnc     avx_ctr_nextBlock2\r
        add     rN, ways * 2\r
        \r
        vextracti128    iv, iv_ymm, 1\r
        sar     ksize_r, 1\r
       \r
        add     r4, AVX_STACK_SUB\r
        pop     keys2\r
        \r
        vzeroupper\r
        jmp     Ctr_start_3\r
  else\r
        jmp     Ctr_start\r
  endif\r
MY_ENDP\r
MY_SEG_ENDP\r
\r
end\r
Commit	Line	Data
9e052883	1	; AesOpt.asm -- AES optimized code for x86 AES hardware instructions\r
	2	; 2021-12-25 : Igor Pavlov : Public domain\r
	3	\r
	4	include 7zAsm.asm\r
	5	\r
	6	ifdef __ASMC__\r
	7	use_vaes_256 equ 1\r
	8	else\r
	9	ifdef ymm0\r
	10	use_vaes_256 equ 1\r
	11	endif\r
	12	endif\r
	13	\r
	14	\r
	15	ifdef use_vaes_256\r
	16	ECHO "++ VAES 256"\r
	17	else\r
	18	ECHO "-- NO VAES 256"\r
	19	endif\r
	20	\r
	21	ifdef x64\r
	22	ECHO "x86-64"\r
	23	else\r
	24	ECHO "x86"\r
	25	if (IS_CDECL gt 0)\r
	26	ECHO "ABI : CDECL"\r
	27	else\r
	28	ECHO "ABI : no CDECL : FASTCALL"\r
	29	endif\r
	30	endif\r
	31	\r
	32	if (IS_LINUX gt 0)\r
	33	ECHO "ABI : LINUX"\r
	34	else\r
	35	ECHO "ABI : WINDOWS"\r
	36	endif\r
	37	\r
	38	MY_ASM_START\r
	39	\r
	40	ifndef x64\r
	41	.686\r
	42	.xmm\r
	43	endif\r
	44	\r
	45	\r
	46	; MY_ALIGN EQU ALIGN(64)\r
	47	MY_ALIGN EQU\r
	48	\r
	49	SEG_ALIGN EQU MY_ALIGN\r
	50	\r
	51	MY_SEG_PROC macro name:req, numParams:req\r
	52	; seg_name equ @CatStr(_TEXT$, name)\r
	53	; seg_name SEGMENT SEG_ALIGN 'CODE'\r
	54	MY_PROC name, numParams\r
	55	endm\r
	56	\r
	57	MY_SEG_ENDP macro\r
	58	; seg_name ENDS\r
	59	endm\r
	60	\r
	61	\r
	62	NUM_AES_KEYS_MAX equ 15\r
	63	\r
	64	; the number of push operators in function PROLOG\r
65	if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
66	num_regs_push equ 2\r
67	stack_param_offset equ (REG_SIZE * (1 + num_regs_push))\r
68	endif\r
69	\r
70	ifdef x64\r
71	num_param equ REG_ABI_PARAM_2\r
72	else\r
73	if (IS_CDECL gt 0)\r
74	; size_t size\r
75	; void * data\r
76	; UInt32 * aes\r
77	; ret-ip <- (r4)\r
78	aes_OFFS equ (stack_param_offset)\r
79	data_OFFS equ (REG_SIZE + aes_OFFS)\r
80	size_OFFS equ (REG_SIZE + data_OFFS)\r
81	num_param equ [r4 + size_OFFS]\r
82	else\r
83	num_param equ [r4 + stack_param_offset]\r
84	endif\r
85	endif\r
86	\r
87	keys equ REG_PARAM_0 ; r1\r
88	rD equ REG_PARAM_1 ; r2\r
89	rN equ r0\r
90	\r
91	koffs_x equ x7\r
92	koffs_r equ r7\r
93	\r
94	ksize_x equ x6\r
95	ksize_r equ r6\r
96	\r
97	keys2 equ r3\r
98	\r
99	state equ xmm0\r
100	key equ xmm0\r
101	key_ymm equ ymm0\r
102	key_ymm_n equ 0\r
103	\r
104	ifdef x64\r
105	ways = 11\r
106	else\r
107	ways = 4\r
108	endif\r
109	\r
110	ways_start_reg equ 1\r
111	\r
112	iv equ @CatStr(xmm, %(ways_start_reg + ways))\r
113	iv_ymm equ @CatStr(ymm, %(ways_start_reg + ways))\r
114	\r
115	\r
116	WOP macro op, op2\r
117	i = 0\r
118	rept ways\r
119	op @CatStr(xmm, %(ways_start_reg + i)), op2\r
120	i = i + 1\r
121	endm\r
122	endm\r
123	\r
124	\r
125	ifndef ABI_LINUX\r
126	ifdef x64\r
127	\r
128	; we use 32 bytes of home space in stack in WIN64-x64\r
129	NUM_HOME_MM_REGS equ (32 / 16)\r
130	; we preserve xmm registers starting from xmm6 in WIN64-x64\r
131	MM_START_SAVE_REG equ 6\r
132	\r
133	SAVE_XMM macro num_used_mm_regs:req\r
134	num_save_mm_regs = num_used_mm_regs - MM_START_SAVE_REG\r
135	if num_save_mm_regs GT 0\r
136	num_save_mm_regs2 = num_save_mm_regs - NUM_HOME_MM_REGS\r
137	; RSP is (16*x + 8) after entering the function in WIN64-x64\r
138	stack_offset = 16 * num_save_mm_regs2 + (stack_param_offset mod 16)\r
139	\r
140	i = 0\r
141	rept num_save_mm_regs\r
142	\r
143	if i eq NUM_HOME_MM_REGS\r
144	sub r4, stack_offset\r
145	endif\r
146	\r
147	if i lt NUM_HOME_MM_REGS\r
148	movdqa [r4 + stack_param_offset + i * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i))\r
149	else\r
150	movdqa [r4 + (i - NUM_HOME_MM_REGS) * 16], @CatStr(xmm, %(MM_START_SAVE_REG + i))\r
151	endif\r
152	\r
153	i = i + 1\r
154	endm\r
155	endif\r
156	endm\r
157	\r
158	RESTORE_XMM macro num_used_mm_regs:req\r
159	if num_save_mm_regs GT 0\r
160	i = 0\r
161	if num_save_mm_regs2 GT 0\r
162	rept num_save_mm_regs2\r
163	movdqa @CatStr(xmm, %(MM_START_SAVE_REG + NUM_HOME_MM_REGS + i)), [r4 + i * 16]\r
164	i = i + 1\r
165	endm\r
166	add r4, stack_offset\r
167	endif\r
168	\r
169	num_low_regs = num_save_mm_regs - i\r
170	i = 0\r
171	rept num_low_regs\r
172	movdqa @CatStr(xmm, %(MM_START_SAVE_REG + i)), [r4 + stack_param_offset + i * 16]\r
173	i = i + 1\r
174	endm\r
175	endif\r
176	endm\r
177	\r
178	endif ; x64\r
179	endif ; ABI_LINUX\r
180	\r
181	\r
182	MY_PROLOG macro num_used_mm_regs:req\r
183	; num_regs_push: must be equal to the number of push operators\r
184	; push r3\r
185	; push r5\r
186	if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
187	push r6\r
188	push r7\r
189	endif\r
190	\r
191	mov rN, num_param ; don't move it; num_param can use stack pointer (r4)\r
192	\r
193	if (IS_X64 eq 0)\r
194	if (IS_CDECL gt 0)\r
195	mov rD, [r4 + data_OFFS]\r
196	mov keys, [r4 + aes_OFFS]\r
197	endif\r
198	elseif (IS_LINUX gt 0)\r
199	MY_ABI_LINUX_TO_WIN_2\r
200	endif\r
201	\r
202	\r
203	ifndef ABI_LINUX\r
204	ifdef x64\r
205	SAVE_XMM num_used_mm_regs\r
206	endif\r
207	endif\r
208	\r
209	mov ksize_x, [keys + 16]\r
210	shl ksize_x, 5\r
211	endm\r
212	\r
213	\r
214	MY_EPILOG macro\r
215	ifndef ABI_LINUX\r
216	ifdef x64\r
217	RESTORE_XMM num_save_mm_regs\r
218	endif\r
219	endif\r
220	\r
221	if (IS_LINUX eq 0) or (IS_X64 eq 0)\r
222	pop r7\r
223	pop r6\r
224	endif\r
225	; pop r5\r
226	; pop r3\r
227	MY_ENDP\r
228	endm\r
229	\r
230	\r
231	OP_KEY macro op:req, offs:req\r
232	op state, [keys + offs]\r
233	endm\r
234	\r
235	\r
236	WOP_KEY macro op:req, offs:req\r
237	movdqa key, [keys + offs]\r
238	WOP op, key\r
239	endm\r
240	\r
241	\r
242	; ---------- AES-CBC Decode ----------\r
243	\r
244	\r
245	XOR_WITH_DATA macro reg, _ppp_\r
246	pxor reg, [rD + i * 16]\r
247	endm\r
248	\r
249	WRITE_TO_DATA macro reg, _ppp_\r
250	movdqa [rD + i * 16], reg\r
251	endm\r
252	\r
253	\r
254	; state0 equ @CatStr(xmm, %(ways_start_reg))\r
255	\r
256	key0 equ @CatStr(xmm, %(ways_start_reg + ways + 1))\r
257	key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1))\r
258	\r
259	key_last equ @CatStr(xmm, %(ways_start_reg + ways + 2))\r
260	key_last_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2))\r
261	key_last_ymm_n equ (ways_start_reg + ways + 2)\r
262	\r
263	NUM_CBC_REGS equ (ways_start_reg + ways + 3)\r
264	\r
265	\r
266	MY_SEG_PROC AesCbc_Decode_HW, 3\r
267	\r
268	AesCbc_Decode_HW_start::\r
269	MY_PROLOG NUM_CBC_REGS\r
270	\r
271	AesCbc_Decode_HW_start_2::\r
272	movdqa iv, [keys]\r
273	add keys, 32\r
274	\r
275	movdqa key0, [keys + 1 * ksize_r]\r
276	movdqa key_last, [keys]\r
277	sub ksize_x, 16\r
278	\r
279	jmp check2\r
280	align 16\r
281	nextBlocks2:\r
282	WOP movdqa, [rD + i * 16]\r
283	mov koffs_x, ksize_x\r
284	; WOP_KEY pxor, ksize_r + 16\r
285	WOP pxor, key0\r
286	; align 16\r
287	@@:\r
288	WOP_KEY aesdec, 1 * koffs_r\r
289	sub koffs_r, 16\r
290	jnz @B\r
291	; WOP_KEY aesdeclast, 0\r
292	WOP aesdeclast, key_last\r
293	\r
294	pxor @CatStr(xmm, %(ways_start_reg)), iv\r
295	i = 1\r
296	rept ways - 1\r
297	pxor @CatStr(xmm, %(ways_start_reg + i)), [rD + i * 16 - 16]\r
298	i = i + 1\r
299	endm\r
300	movdqa iv, [rD + ways * 16 - 16]\r
301	WOP WRITE_TO_DATA\r
302	\r
303	add rD, ways * 16\r
304	AesCbc_Decode_HW_start_3::\r
305	check2:\r
306	sub rN, ways\r
307	jnc nextBlocks2\r
308	add rN, ways\r
309	\r
310	sub ksize_x, 16\r
311	\r
312	jmp check\r
313	nextBlock:\r
314	movdqa state, [rD]\r
315	mov koffs_x, ksize_x\r
316	; OP_KEY pxor, 1 * ksize_r + 32\r
317	pxor state, key0\r
318	; movdqa state0, [rD]\r
319	; movdqa state, key0\r
320	; pxor state, state0\r
321	@@:\r
322	OP_KEY aesdec, 1 * koffs_r + 16\r
323	OP_KEY aesdec, 1 * koffs_r\r
324	sub koffs_r, 32\r
325	jnz @B\r
326	OP_KEY aesdec, 16\r
327	; OP_KEY aesdeclast, 0\r
328	aesdeclast state, key_last\r
329	\r
330	pxor state, iv\r
331	movdqa iv, [rD]\r
332	; movdqa iv, state0\r
333	movdqa [rD], state\r
334	\r
335	add rD, 16\r
336	check:\r
337	sub rN, 1\r
338	jnc nextBlock\r
339	\r
340	movdqa [keys - 32], iv\r
341	MY_EPILOG\r
342	\r
343	\r
344	\r
345	\r
346	; ---------- AVX ----------\r
347	\r
348	\r
349	AVX__WOP_n macro op\r
350	i = 0\r
351	rept ways\r
352	op (ways_start_reg + i)\r
353	i = i + 1\r
354	endm\r
355	endm\r
356	\r
357	AVX__WOP macro op\r
358	i = 0\r
359	rept ways\r
360	op @CatStr(ymm, %(ways_start_reg + i))\r
361	i = i + 1\r
362	endm\r
363	endm\r
364	\r
365	\r
366	AVX__WOP_KEY macro op:req, offs:req\r
367	vmovdqa key_ymm, ymmword ptr [keys2 + offs]\r
368	AVX__WOP_n op\r
369	endm\r
370	\r
371	\r
372	AVX__CBC_START macro reg\r
373	; vpxor reg, key_ymm, ymmword ptr [rD + 32 * i]\r
374	vpxor reg, key0_ymm, ymmword ptr [rD + 32 * i]\r
375	endm\r
376	\r
377	AVX__CBC_END macro reg\r
378	if i eq 0\r
379	vpxor reg, reg, iv_ymm\r
380	else\r
381	vpxor reg, reg, ymmword ptr [rD + i * 32 - 16]\r
382	endif\r
383	endm\r
384	\r
385	\r
386	AVX__WRITE_TO_DATA macro reg\r
387	vmovdqu ymmword ptr [rD + 32 * i], reg\r
388	endm\r
389	\r
390	AVX__XOR_WITH_DATA macro reg\r
391	vpxor reg, reg, ymmword ptr [rD + 32 * i]\r
392	endm\r
393	\r
394	AVX__CTR_START macro reg\r
395	vpaddq iv_ymm, iv_ymm, one_ymm\r
396	; vpxor reg, iv_ymm, key_ymm\r
397	vpxor reg, iv_ymm, key0_ymm\r
398	endm\r
399	\r
400	\r
401	MY_VAES_INSTR_2 macro cmd, dest, a1, a2\r
402	db 0c4H\r
403	db 2 + 040H + 020h * (1 - (a2) / 8) + 080h * (1 - (dest) / 8)\r
404	db 5 + 8 * ((not (a1)) and 15)\r
405	db cmd\r
406	db 0c0H + 8 * ((dest) and 7) + ((a2) and 7)\r
407	endm\r
408	\r
409	MY_VAES_INSTR macro cmd, dest, a\r
410	MY_VAES_INSTR_2 cmd, dest, dest, a\r
411	endm\r
412	\r
413	MY_vaesenc macro dest, a\r
414	MY_VAES_INSTR 0dcH, dest, a\r
415	endm\r
416	MY_vaesenclast macro dest, a\r
417	MY_VAES_INSTR 0ddH, dest, a\r
418	endm\r
419	MY_vaesdec macro dest, a\r
420	MY_VAES_INSTR 0deH, dest, a\r
421	endm\r
422	MY_vaesdeclast macro dest, a\r
423	MY_VAES_INSTR 0dfH, dest, a\r
424	endm\r
425	\r
426	\r
427	AVX__VAES_DEC macro reg\r
428	MY_vaesdec reg, key_ymm_n\r
429	endm\r
430	\r
431	AVX__VAES_DEC_LAST_key_last macro reg\r
432	; MY_vaesdeclast reg, key_ymm_n\r
433	MY_vaesdeclast reg, key_last_ymm_n\r
434	endm\r
435	\r
436	AVX__VAES_ENC macro reg\r
437	MY_vaesenc reg, key_ymm_n\r
438	endm\r
439	\r
440	AVX__VAES_ENC_LAST macro reg\r
441	MY_vaesenclast reg, key_ymm_n\r
442	endm\r
443	\r
444	AVX__vinserti128_TO_HIGH macro dest, src\r
445	vinserti128 dest, dest, src, 1\r
446	endm\r
447	\r
448	\r
449	MY_PROC AesCbc_Decode_HW_256, 3\r
450	ifdef use_vaes_256\r
451	MY_PROLOG NUM_CBC_REGS\r
452	\r
453	cmp rN, ways * 2\r
454	jb AesCbc_Decode_HW_start_2\r
455	\r
456	vmovdqa iv, xmmword ptr [keys]\r
457	add keys, 32\r
458	\r
459	vbroadcasti128 key0_ymm, xmmword ptr [keys + 1 * ksize_r]\r
460	vbroadcasti128 key_last_ymm, xmmword ptr [keys]\r
461	sub ksize_x, 16\r
462	mov koffs_x, ksize_x\r
463	add ksize_x, ksize_x\r
464	\r
465	AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 2) * 32)\r
466	push keys2\r
467	sub r4, AVX_STACK_SUB\r
468	; sub r4, 32\r
469	; sub r4, ksize_r\r
470	; lea keys2, [r4 + 32]\r
471	mov keys2, r4\r
472	and keys2, -32\r
473	broad:\r
474	vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r]\r
475	vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm\r
476	sub koffs_r, 16\r
477	; jnc broad\r
478	jnz broad\r
479	\r
480	sub rN, ways * 2\r
481	\r
482	align 16\r
483	avx_cbcdec_nextBlock2:\r
484	mov koffs_x, ksize_x\r
485	; AVX__WOP_KEY AVX__CBC_START, 1 * koffs_r + 32\r
486	AVX__WOP AVX__CBC_START\r
487	@@:\r
488	AVX__WOP_KEY AVX__VAES_DEC, 1 * koffs_r\r
489	sub koffs_r, 32\r
490	jnz @B\r
491	; AVX__WOP_KEY AVX__VAES_DEC_LAST, 0\r
492	AVX__WOP_n AVX__VAES_DEC_LAST_key_last\r
493	\r
494	AVX__vinserti128_TO_HIGH iv_ymm, xmmword ptr [rD]\r
495	AVX__WOP AVX__CBC_END\r
496	\r
497	vmovdqa iv, xmmword ptr [rD + ways * 32 - 16]\r
498	AVX__WOP AVX__WRITE_TO_DATA\r
499	\r
500	add rD, ways * 32\r
501	sub rN, ways * 2\r
502	jnc avx_cbcdec_nextBlock2\r
503	add rN, ways * 2\r
504	\r
505	shr ksize_x, 1\r
506	\r
507	; lea r4, [r4 + 1 * ksize_r + 32]\r
508	add r4, AVX_STACK_SUB\r
509	pop keys2\r
510	\r
511	vzeroupper\r
512	jmp AesCbc_Decode_HW_start_3\r
513	else\r
514	jmp AesCbc_Decode_HW_start\r
515	endif\r
516	MY_ENDP\r
517	MY_SEG_ENDP\r
518	\r
519	\r
520	\r
521	\r
522	; ---------- AES-CBC Encode ----------\r
523	\r
524	e0 equ xmm1\r
525	\r
526	CENC_START_KEY equ 2\r
527	CENC_NUM_REG_KEYS equ (3 * 2)\r
528	; last_key equ @CatStr(xmm, %(CENC_START_KEY + CENC_NUM_REG_KEYS))\r
529	\r
530	MY_SEG_PROC AesCbc_Encode_HW, 3\r
531	MY_PROLOG (CENC_START_KEY + CENC_NUM_REG_KEYS + 0)\r
532	\r
533	movdqa state, [keys]\r
534	add keys, 32\r
535	\r
536	i = 0\r
537	rept CENC_NUM_REG_KEYS\r
538	movdqa @CatStr(xmm, %(CENC_START_KEY + i)), [keys + i * 16]\r
539	i = i + 1\r
540	endm\r
541	\r
542	add keys, ksize_r\r
543	neg ksize_r\r
544	add ksize_r, (16 * CENC_NUM_REG_KEYS)\r
545	; movdqa last_key, [keys]\r
546	jmp check_e\r
547	\r
548	align 16\r
549	nextBlock_e:\r
550	movdqa e0, [rD]\r
551	mov koffs_r, ksize_r\r
552	pxor e0, @CatStr(xmm, %(CENC_START_KEY))\r
553	pxor state, e0\r
554	\r
555	i = 1\r
556	rept (CENC_NUM_REG_KEYS - 1)\r
557	aesenc state, @CatStr(xmm, %(CENC_START_KEY + i))\r
558	i = i + 1\r
559	endm\r
560	\r
561	@@:\r
562	OP_KEY aesenc, 1 * koffs_r\r
563	OP_KEY aesenc, 1 * koffs_r + 16\r
564	add koffs_r, 32\r
565	jnz @B\r
566	OP_KEY aesenclast, 0\r
567	; aesenclast state, last_key\r
568	\r
569	movdqa [rD], state\r
570	add rD, 16\r
571	check_e:\r
572	sub rN, 1\r
573	jnc nextBlock_e\r
574	\r
575	; movdqa [keys - 32], state\r
576	movdqa [keys + 1 * ksize_r - (16 * CENC_NUM_REG_KEYS) - 32], state\r
577	MY_EPILOG\r
578	MY_SEG_ENDP\r
579	\r
580	\r
581	\r
582	; ---------- AES-CTR ----------\r
583	\r
584	ifdef x64\r
585	; ways = 11\r
586	endif\r
587	\r
588	\r
589	one equ @CatStr(xmm, %(ways_start_reg + ways + 1))\r
590	one_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 1))\r
591	key0 equ @CatStr(xmm, %(ways_start_reg + ways + 2))\r
592	key0_ymm equ @CatStr(ymm, %(ways_start_reg + ways + 2))\r
593	NUM_CTR_REGS equ (ways_start_reg + ways + 3)\r
594	\r
595	INIT_CTR macro reg, _ppp_\r
596	paddq iv, one\r
597	movdqa reg, iv\r
598	endm\r
599	\r
600	\r
601	MY_SEG_PROC AesCtr_Code_HW, 3\r
602	Ctr_start::\r
603	MY_PROLOG NUM_CTR_REGS\r
604	\r
605	Ctr_start_2::\r
606	movdqa iv, [keys]\r
607	add keys, 32\r
608	movdqa key0, [keys]\r
609	\r
610	add keys, ksize_r\r
611	neg ksize_r\r
612	add ksize_r, 16\r
613	\r
614	Ctr_start_3::\r
615	mov koffs_x, 1\r
616	movd one, koffs_x\r
617	jmp check2_c\r
618	\r
619	align 16\r
620	nextBlocks2_c:\r
621	WOP INIT_CTR, 0\r
622	mov koffs_r, ksize_r\r
623	; WOP_KEY pxor, 1 * koffs_r -16\r
624	WOP pxor, key0\r
625	@@:\r
626	WOP_KEY aesenc, 1 * koffs_r\r
627	add koffs_r, 16\r
628	jnz @B\r
629	WOP_KEY aesenclast, 0\r
630	\r
631	WOP XOR_WITH_DATA\r
632	WOP WRITE_TO_DATA\r
633	add rD, ways * 16\r
634	check2_c:\r
635	sub rN, ways\r
636	jnc nextBlocks2_c\r
637	add rN, ways\r
638	\r
639	sub keys, 16\r
640	add ksize_r, 16\r
641	\r
642	jmp check_c\r
643	\r
644	; align 16\r
645	nextBlock_c:\r
646	paddq iv, one\r
647	; movdqa state, [keys + 1 * koffs_r - 16]\r
648	movdqa state, key0\r
649	mov koffs_r, ksize_r\r
650	pxor state, iv\r
651	\r
652	@@:\r
653	OP_KEY aesenc, 1 * koffs_r\r
654	OP_KEY aesenc, 1 * koffs_r + 16\r
655	add koffs_r, 32\r
656	jnz @B\r
657	OP_KEY aesenc, 0\r
658	OP_KEY aesenclast, 16\r
659	\r
660	pxor state, [rD]\r
661	movdqa [rD], state\r
662	add rD, 16\r
663	check_c:\r
664	sub rN, 1\r
665	jnc nextBlock_c\r
666	\r
667	; movdqa [keys - 32], iv\r
668	movdqa [keys + 1 * ksize_r - 16 - 32], iv\r
669	MY_EPILOG\r
670	\r
671	\r
672	MY_PROC AesCtr_Code_HW_256, 3\r
673	ifdef use_vaes_256\r
674	MY_PROLOG NUM_CTR_REGS\r
675	\r
676	cmp rN, ways * 2\r
677	jb Ctr_start_2\r
678	\r
679	vbroadcasti128 iv_ymm, xmmword ptr [keys]\r
680	add keys, 32\r
681	vbroadcasti128 key0_ymm, xmmword ptr [keys]\r
682	mov koffs_x, 1\r
683	vmovd one, koffs_x\r
684	vpsubq iv_ymm, iv_ymm, one_ymm\r
685	vpaddq one, one, one\r
686	AVX__vinserti128_TO_HIGH one_ymm, one\r
687	\r
688	add keys, ksize_r\r
689	sub ksize_x, 16\r
690	neg ksize_r\r
691	mov koffs_r, ksize_r\r
692	add ksize_r, ksize_r\r
693	\r
694	AVX_STACK_SUB = ((NUM_AES_KEYS_MAX + 1 - 1) * 32)\r
695	push keys2\r
696	lea keys2, [r4 - 32]\r
697	sub r4, AVX_STACK_SUB\r
698	and keys2, -32\r
699	vbroadcasti128 key_ymm, xmmword ptr [keys]\r
700	vmovdqa ymmword ptr [keys2], key_ymm\r
701	@@:\r
702	vbroadcasti128 key_ymm, xmmword ptr [keys + 1 * koffs_r]\r
703	vmovdqa ymmword ptr [keys2 + koffs_r * 2], key_ymm\r
704	add koffs_r, 16\r
705	jnz @B\r
706	\r
707	sub rN, ways * 2\r
708	\r
709	align 16\r
710	avx_ctr_nextBlock2:\r
711	mov koffs_r, ksize_r\r
712	AVX__WOP AVX__CTR_START\r
713	; AVX__WOP_KEY AVX__CTR_START, 1 * koffs_r - 32\r
714	@@:\r
715	AVX__WOP_KEY AVX__VAES_ENC, 1 * koffs_r\r
716	add koffs_r, 32\r
717	jnz @B\r
718	AVX__WOP_KEY AVX__VAES_ENC_LAST, 0\r
719	\r
720	AVX__WOP AVX__XOR_WITH_DATA\r
721	AVX__WOP AVX__WRITE_TO_DATA\r
722	\r
723	add rD, ways * 32\r
724	sub rN, ways * 2\r
725	jnc avx_ctr_nextBlock2\r
726	add rN, ways * 2\r
727	\r
728	vextracti128 iv, iv_ymm, 1\r
729	sar ksize_r, 1\r
730	\r
731	add r4, AVX_STACK_SUB\r
732	pop keys2\r
733	\r
734	vzeroupper\r
735	jmp Ctr_start_3\r
736	else\r
737	jmp Ctr_start\r
738	endif\r
739	MY_ENDP\r
740	MY_SEG_ENDP\r
741	\r
742	end\r