From 7d591e888d3bba7e5a2a99ab3ccf225744bf6b8e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 4 Mar 2022 09:02:00 +0000 Subject: [PATCH 001/369] avx: [ZUC] initialize keystream Valgrind was complaining that keystream was unintialized, so it is now set to 0. --- lib/avx/zuc_top_avx.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index ab93c0eb..fb29571e 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -528,6 +528,8 @@ void _zuc_eia3_1_buffer_avx(const void *pKey, uint32_t T = 0; const uint8_t *pIn8 = (const uint8_t *) pBufferIn; + memset(keyStream, 0, sizeof(keyStream)); + asm_ZucInitialization_avx(pKey, pIv, &(zucState)); asm_ZucGenKeystream16B_avx(pZuc, &zucState); @@ -584,7 +586,9 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX_BUFS], 16) = {NULL}; unsigned int allCommonBits; + memset(keyStr, 0, sizeof(keyStr)); memset(T, 0, sizeof(T)); + /* Check if all lengths are equal */ if ((lengthInBits[0] == lengthInBits[1]) && (lengthInBits[0] == lengthInBits[2]) && @@ -778,6 +782,8 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX_BUFS], 16) = {NULL}; unsigned int allCommonBits; + memset(keyStr, 0, sizeof(keyStr)); + /* Check if all lengths are equal */ if ((lengthInBits[0] == lengthInBits[1]) && (lengthInBits[0] == lengthInBits[2]) && @@ -922,6 +928,8 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX_BUFS], 16) = {NULL}; unsigned int allCommonBits; + memset(keyStr, 0, sizeof(keyStr)); + /* Check if all lengths are equal */ if ((lengthInBits[0] == lengthInBits[1]) && (lengthInBits[0] == lengthInBits[2]) && -- GitLab From 6f40e5ffb8f155f29ab99e9b5895d8557e8ec5f4 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 22 Feb 2022 17:46:24 +0000 Subject: [PATCH 002/369] avx: [ZUC] remove unused macro and data --- lib/avx/zuc_x4_avx.asm | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 025578d0..9cfe515f 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -171,10 +171,6 @@ broadcast_word: db 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01 db 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01 -align 16 -all_ffs: -dw 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff - align 16 all_threes: dw 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003 @@ -313,21 +309,6 @@ align 64 vshufps %%r0, %%t0, %%t1, 0x88 ; r0 = {d0 c0 b0 a0} %endmacro -;; -;; make_u31() -;; -%macro make_u31 4 - -%define %%Rt %1 -%define %%Ke %2 -%define %%Ek %3 -%define %%Iv %4 - xor %%Rt, %%Rt - shrd %%Rt, %%Iv, 8 - shrd %%Rt, %%Ek, 15 - shrd %%Rt, %%Ke, 9 -%endmacro - ; ; bits_reorg4() ; -- GitLab From 204f95a07a69a4c21547e1da4e01840c02dc6b86 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 22 Feb 2022 18:05:10 +0000 Subject: [PATCH 003/369] avx: [ZUC] refactor nonlin macro Remove hard-coded registers from bits reorganization macro, and passing the registers through parameters instead. --- lib/avx/zuc_x4_avx.asm | 190 ++++++++++++++++++++--------------------- lib/sse/zuc_x4_sse.asm | 20 ++--- 2 files changed, 100 insertions(+), 110 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 9cfe515f..e1a3e8b2 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -135,18 +135,10 @@ swap_mask: db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c -align 16 -S1_S0_shuf: -db 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F - align 16 S0_S1_shuf: db 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E -align 16 -rev_S1_S0_shuf: -db 0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B, 0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F - align 16 rev_S0_S1_shuf: db 0x08, 0x00, 0x09, 0x01, 0x0A, 0x02, 0x0B, 0x03, 0x0C, 0x04, 0x0D, 0x05, 0x0E, 0x06, 0x0F, 0x07 @@ -366,94 +358,100 @@ align 64 %endmacro ; -; rot_mod32() -; -; uses xmm7 +; Rotate dwords by N_BITS ; -%macro rot_mod32 3 -%if (%3 == 8) - vpshufb %1, %2, [rel rot8_mod32] -%elif (%3 == 16) - vpshufb %1, %2, [rel rot16_mod32] -%elif (%3 == 24) - vpshufb %1, %2, [rel rot24_mod32] +%macro ROT_MOD32 4 +%define %%OUT %1 ; [out] XMM register +%define %%IN %2 ; [in] XMM register +%define %%XTMP %3 ; [clobbered] XMM register +%define %%N_BITS %4 ; [constant] Number of bits + +%if (%%N_BITS == 8) + vpshufb %%OUT, %%IN, [rel rot8_mod32] +%elif (%%N_BITS == 16) + vpshufb %%OUT, %%IN, [rel rot16_mod32] +%elif (%%N_BITS == 24) + vpshufb %%OUT, %%IN, [rel rot24_mod32] %else - vpslld %1, %2, %3 - vpsrld xmm7, %2, (32 - %3) - - vpor %1, xmm7 + vpslld %%OUT, %%IN, %%N_BITS + vpsrld %%XTMP, %%IN, (32 - %%N_BITS) + vpor %%OUT, %%XTMP %endif %endmacro ; -; nonlin_fun4() -; -; return -; W value, updates F_R1[] / F_R2[] +; Updates R1-R2, using X0-X3 and generates W (if needed) ; -%macro nonlin_fun4 1-2 +%macro NONLIN_FUN4 8-9 %define %%STATE %1 ; [in] ZUC state -%define %%W %2 ; [out] XMM register to contain W for all lanes - -%if (%0 == 2) - vmovdqa %%W, [%%STATE + OFS_X0] - vpxor %%W, [%%STATE + OFS_R1] - vpaddd %%W, [%%STATE + OFS_R2] ; W = (BRC_X0 ^ F_R1) + F_R2 +%define %%XTMP1 %2 ; [clobbered] Temporary XMM register +%define %%XTMP2 %3 ; [clobbered] Temporary XMM register +%define %%XTMP3 %4 ; [clobbered] Temporary XMM register +%define %%XTMP4 %5 ; [clobbered] Temporary XMM register +%define %%XTMP5 %6 ; [clobbered] Temporary XMM register +%define %%XTMP6 %7 ; [clobbered] Temporary XMM register +%define %%XTMP7 %8 ; [clobbered] Temporary XMM register +%define %%W %9 ; [out] ZMM register to contain W for all lanes + +%if (%0 == 9) + vmovdqa %%W, [%%STATE + OFS_X0] + vpxor %%W, [%%STATE + OFS_R1] + vpaddd %%W, [%%STATE + OFS_R2] ; W = (BRC_X0 ^ F_R1) + F_R2 %endif - vmovdqa xmm1, [%%STATE + OFS_R1] - vmovdqa xmm2, [%%STATE + OFS_R2] - vpaddd xmm1, [%%STATE + OFS_X1] ; W1 = F_R1 + BRC_X1 - vpxor xmm2, [%%STATE + OFS_X2] ; W2 = F_R2 ^ BRC_X2 - - vpslld xmm3, xmm1, 16 - vpsrld xmm4, xmm1, 16 - vpslld xmm5, xmm2, 16 - vpsrld xmm6, xmm2, 16 - vpor xmm1, xmm3, xmm6 - vpor xmm2, xmm4, xmm5 - - rot_mod32 xmm3, xmm1, 2 - rot_mod32 xmm4, xmm1, 10 - rot_mod32 xmm5, xmm1, 18 - rot_mod32 xmm6, xmm1, 24 - vpxor xmm1, xmm3 - vpxor xmm1, xmm4 - vpxor xmm1, xmm5 - vpxor xmm1, xmm6 ; XMM1 = U = L1(P) - - rot_mod32 xmm3, xmm2, 8 - rot_mod32 xmm4, xmm2, 14 - rot_mod32 xmm5, xmm2, 22 - rot_mod32 xmm6, xmm2, 30 - vpxor xmm2, xmm3 - vpxor xmm2, xmm4 - vpxor xmm2, xmm5 - vpxor xmm2, xmm6 ; XMM2 = V = L2(Q) - - ; Shuffle U and V to have all S0 lookups in XMM1 and all S1 lookups in XMM2 - - ; Compress all S0 and S1 input values in each register - vpshufb xmm1, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 - vpshufb xmm2, [rel S1_S0_shuf] ; S1: Bytes 0-7, S0: Bytes 8-15 - - vshufpd xmm3, xmm1, xmm2, 0x2 ; All S0 input values - vshufpd xmm4, xmm2, xmm1, 0x2 ; All S1 input values - - ; Compute S0 and S1 values - S0_comput_AVX xmm3, xmm1, xmm2 - S1_comput_AVX xmm4, xmm1, xmm2, xmm5 - - ; Need to shuffle back xmm1 & xmm2 before storing output - ; (revert what was done before S0 and S1 computations) - vshufpd xmm1, xmm3, xmm4, 0x2 - vshufpd xmm2, xmm4, xmm3, 0x2 - - vpshufb xmm1, [rel rev_S0_S1_shuf] - vpshufb xmm2, [rel rev_S1_S0_shuf] - - vmovdqa [%%STATE + OFS_R1], xmm1 - vmovdqa [%%STATE + OFS_R2], xmm2 + vmovdqa %%XTMP1, [%%STATE + OFS_R1] + vmovdqa %%XTMP2, [%%STATE + OFS_R2] + vpaddd %%XTMP1, [%%STATE + OFS_X1] ; W1 = F_R1 + BRC_X1 + vpxor %%XTMP2, [%%STATE + OFS_X2] ; W2 = F_R2 ^ BRC_X2 + + vpslld %%XTMP3, %%XTMP1, 16 + vpsrld %%XTMP4, %%XTMP1, 16 + vpslld %%XTMP5, %%XTMP2, 16 + vpsrld %%XTMP6, %%XTMP2, 16 + vpor %%XTMP1, %%XTMP3, %%XTMP6 + vpor %%XTMP2, %%XTMP4, %%XTMP5 + + ROT_MOD32 %%XTMP3, %%XTMP1, %%XTMP7, 2 + ROT_MOD32 %%XTMP4, %%XTMP1, %%XTMP7, 10 + ROT_MOD32 %%XTMP5, %%XTMP1, %%XTMP7, 18 + ROT_MOD32 %%XTMP6, %%XTMP1, %%XTMP7, 24 + vpxor %%XTMP1, %%XTMP3 + vpxor %%XTMP1, %%XTMP4 + vpxor %%XTMP1, %%XTMP5 + vpxor %%XTMP1, %%XTMP6 ; XMM1 = U = L1(P) + + ROT_MOD32 %%XTMP3, %%XTMP2, %%XTMP7, 8 + ROT_MOD32 %%XTMP4, %%XTMP2, %%XTMP7, 14 + ROT_MOD32 %%XTMP5, %%XTMP2, %%XTMP7, 22 + ROT_MOD32 %%XTMP6, %%XTMP2, %%XTMP7, 30 + vpxor %%XTMP2, %%XTMP3 + vpxor %%XTMP2, %%XTMP4 + vpxor %%XTMP2, %%XTMP5 + vpxor %%XTMP2, %%XTMP6 ; XMM2 = V = L2(Q) + + ; Shuffle U and V to have all S0 lookups in XMM1 and all S1 lookups in XMM2 + + ; Compress all S0 and S1 input values in each register + vpshufb %%XTMP1, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 + vpshufb %%XTMP2, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 + + vshufpd %%XTMP3, %%XTMP1, %%XTMP2, 0x0 ; All S0 input values + vshufpd %%XTMP4, %%XTMP2, %%XTMP1, 0x3 ; All S1 input values + + ; Compute S0 and S1 values + S0_comput_AVX %%XTMP3, %%XTMP1, %%XTMP2 + S1_comput_AVX %%XTMP4, %%XTMP1, %%XTMP2, %%XTMP5 + + ; Need to shuffle back %%XTMP1 & %%XTMP2 before storing output + ; (revert what was done before S0 and S1 computations) + vshufpd %%XTMP1, %%XTMP3, %%XTMP4, 0x2 + vshufpd %%XTMP2, %%XTMP3, %%XTMP4, 0x1 + + vpshufb %%XTMP1, [rel rev_S0_S1_shuf] + vpshufb %%XTMP2, [rel rev_S0_S1_shuf] + + vmovdqa [%%STATE + OFS_R1], %%XTMP1 + vmovdqa [%%STATE + OFS_R2], %%XTMP2 %endmacro ; @@ -875,15 +873,15 @@ align 64 %assign N 0 %rep 32 bits_reorg4 rax, N - nonlin_fun4 rax, xmm0 - vpsrld xmm0,1 ; Shift out LSB of W - lfsr_updt4 rax, N, xmm0 ; W (xmm0) used in LFSR update - not set to zero + NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + vpsrld xmm7, 1 ; Shift out LSB of W + lfsr_updt4 rax, N, xmm7 ; W (xmm7) used in LFSR update %assign N N+1 %endrep ; And once more, initial round from keygen phase = 33 times bits_reorg4 rax, 0 - nonlin_fun4 rax + NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 vpxor xmm0, xmm0 lfsr_updt4 rax, 0, xmm0 @@ -968,9 +966,9 @@ asm_Zuc256Initialization_4_avx: %assign N 1 %rep %%NUM_ROUNDS bits_reorg4 rax, N, xmm10 - nonlin_fun4 rax, xmm0 - ; OFS_X3 XOR W (xmm0) and store in stack - vpxor xmm10, xmm0 + NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + ; OFS_X3 XOR W (xmm7) and store in stack + vpxor xmm10, xmm7 vmovdqa [rsp + 4*8 + (N-1)*16], xmm10 vpxor xmm0, xmm0 lfsr_updt4 rax, N, xmm0 @@ -1100,9 +1098,9 @@ asm_ZucGenKeystream4B_4_avx: %assign %%round (%%INITIAL_ROUND + %%N) %rep %%NROUNDS bits_reorg4 rax, %%round, xmm10 - nonlin_fun4 rax, xmm0 - ; OFS_XR XOR W (xmm0) and store in stack - vpxor xmm10, xmm0 + NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 + ; OFS_XR XOR W (xmm7) and store in stack + vpxor xmm10, xmm7 vmovdqa [rsp + _keystr_save + (%%N-1)*16], xmm10 vpxor xmm0, xmm0 lfsr_updt4 rax, %%round, xmm0 diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index 0b924bdb..a424ef00 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -154,18 +154,10 @@ swap_mask: db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c -align 16 -S1_S0_shuf: -db 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F - align 16 S0_S1_shuf: db 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E -align 16 -rev_S1_S0_shuf: -db 0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B, 0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F - align 16 rev_S0_S1_shuf: db 0x08, 0x00, 0x09, 0x01, 0x0A, 0x02, 0x0B, 0x03, 0x0C, 0x04, 0x0D, 0x05, 0x0E, 0x06, 0x0F, 0x07 @@ -534,11 +526,11 @@ mksection .text ; Compress all S0 and S1 input values in each register pshufb %%XTMP1, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 - pshufb %%XTMP2, [rel S1_S0_shuf] ; S1: Bytes 0-7, S0: Bytes 8-15 + pshufb %%XTMP2, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 movdqa %%XTMP3, %%XTMP1 - shufpd %%XTMP1, %%XTMP2, 0x2 ; All S0 input values - shufpd %%XTMP2, %%XTMP3, 0x2 ; All S1 input values + shufpd %%XTMP1, %%XTMP2, 0x0 ; All S0 input values + shufpd %%XTMP2, %%XTMP3, 0x3 ; All S1 input values ; Compute S0 and S1 values S0_comput_SSE %%XTMP1, %%XTMP3, %%XTMP4, USE_GFNI @@ -548,13 +540,13 @@ mksection .text ; (revert what was done before S0 and S1 computations) movdqa %%XTMP3, %%XTMP1 shufpd %%XTMP1, %%XTMP2, 0x2 ; All S0 input values - shufpd %%XTMP2, %%XTMP3, 0x2 ; All S1 input values + shufpd %%XTMP3, %%XTMP2, 0x1 ; All S1 input values pshufb %%XTMP1, [rel rev_S0_S1_shuf] - pshufb %%XTMP2, [rel rev_S1_S0_shuf] + pshufb %%XTMP3, [rel rev_S0_S1_shuf] movdqa [%%STATE + OFS_R1], %%XTMP1 - movdqa [%%STATE + OFS_R2], %%XTMP2 + movdqa [%%STATE + OFS_R2], %%XTMP3 %endmacro ; -- GitLab From 8d5988e672c2fb5a444e3690d68996625d4781e3 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 09:40:09 +0000 Subject: [PATCH 004/369] avx: [ZUC] refactor bits_reorg macro Remove hard-coded registers from bits reorganization macro, and passing the register through parameters instead. --- lib/avx/zuc_x4_avx.asm | 107 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 55 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index e1a3e8b2..6aa6e4ca 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -302,58 +302,51 @@ align 64 %endmacro ; -; bits_reorg4() +; Calculates X0-X3 from LFSR registers ; -; params -; %1 - round number -; %2 - XMM register storing X3 -; rax - LFSR pointer -; uses -; -; return -; -%macro bits_reorg4 2-3 -%define %%STATE %1 ; [in] ZUC state -%define %%ROUND_NUM %2 ; [in] Round number -%define %%X3 %3 ; [out] XMM register containing X3 of all lanes - ; - ; - ; xmm15 = LFSR_S15 - ; xmm14 = LFSR_S14 - ; xmm11 = LFSR_S11 - ; xmm9 = LFSR_S9 - ; xmm7 = LFSR_S7 - ; xmm5 = LFSR_S5 - ; xmm2 = LFSR_S2 - ; xmm0 = LFSR_S0 - ; - vmovdqa xmm15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm14, [%%STATE + ((14 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm11, [%%STATE + ((11 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm9, [%%STATE + (( 9 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm7, [%%STATE + (( 7 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm5, [%%STATE + (( 5 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm2, [%%STATE + (( 2 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16] - - vpxor xmm1, xmm1 - vpslld xmm15, 1 - vpblendw xmm3, xmm14, xmm1, 0xAA - vpblendw xmm15, xmm3, xmm15, 0xAA - - vmovdqa [%%STATE + OFS_X0], xmm15 ; BRC_X0 - vpslld xmm11, 16 - vpsrld xmm9, 15 - vpor xmm11, xmm9 - vmovdqa [%%STATE + OFS_X1], xmm11 ; BRC_X1 - vpslld xmm7, 16 - vpsrld xmm5, 15 - vpor xmm7, xmm5 - vmovdqa [%%STATE + OFS_X2], xmm7 ; BRC_X2 -%if (%0 == 3) - vpslld xmm2, 16 - vpsrld xmm0, 15 - vpor %%X3, xmm2, xmm0 +%macro BITS_REORG4 12-13 +%define %%STATE %1 ; [in] ZUC state +%define %%ROUND_NUM %2 ; [in] Round number +%define %%LFSR_0 %3 ; [clobbered] LFSR_0 +%define %%LFSR_2 %4 ; [clobbered] LFSR_2 +%define %%LFSR_5 %5 ; [clobbered] LFSR_5 +%define %%LFSR_7 %6 ; [clobbered] LFSR_7 +%define %%LFSR_9 %7 ; [clobbered] LFSR_9 +%define %%LFSR_11 %8 ; [clobbered] LFSR_11 +%define %%LFSR_14 %9 ; [clobbered] LFSR_14 +%define %%LFSR_15 %10 ; [clobbered] LFSR_15 +%define %%XTMP1 %11 ; [clobbered] Temporary XMM register +%define %%XTMP2 %12 ; [clobbered] Temporary XMM register +%define %%X3 %13 ; [out] XMM register containing X3 of all lanes (only for work mode) + vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_14, [%%STATE + ((14 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_11, [%%STATE + ((11 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_9, [%%STATE + (( 9 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_7, [%%STATE + (( 7 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_5, [%%STATE + (( 5 + %%ROUND_NUM) % 16)*16] +%if (%0 == 13) ;Only needed when generating X3 (for "working" mode) + vmovdqa %%LFSR_2, [%%STATE + (( 2 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16] +%endif + + vpxor %%XTMP1, %%XTMP1 + vpslld %%LFSR_15, 1 + vpblendw %%XTMP2, %%LFSR_14, %%XTMP1, 0xAA + vpblendw %%LFSR_15, %%LFSR_15, %%XTMP2, 0x55 + + vmovdqa [%%STATE + OFS_X0], %%LFSR_15 ; BRC_X0 + vpslld %%LFSR_11, 16 + vpsrld %%LFSR_9, 15 + vpor %%LFSR_11, %%LFSR_9 + vmovdqa [%%STATE + OFS_X1], %%LFSR_11 ; BRC_X1 + vpslld %%LFSR_7, 16 + vpsrld %%LFSR_5, 15 + vpor %%LFSR_7, %%LFSR_5 + vmovdqa [%%STATE + OFS_X2], %%LFSR_7 ; BRC_X2 +%if (%0 == 13) + vpslld %%LFSR_2, 16 + vpsrld %%LFSR_0, 15 + vpor %%X3, %%LFSR_2, %%LFSR_0 %endif %endmacro @@ -872,7 +865,8 @@ align 64 ; Shift LFSR 32-times, update state variables %assign N 0 %rep 32 - bits_reorg4 rax, N + BITS_REORG4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + xmm6, xmm7, xmm8, xmm9 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 vpsrld xmm7, 1 ; Shift out LSB of W lfsr_updt4 rax, N, xmm7 ; W (xmm7) used in LFSR update @@ -880,7 +874,8 @@ align 64 %endrep ; And once more, initial round from keygen phase = 33 times - bits_reorg4 rax, 0 + BITS_REORG4 rax, 0, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + xmm6, xmm7, xmm8, xmm9 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 vpxor xmm0, xmm0 lfsr_updt4 rax, 0, xmm0 @@ -965,7 +960,8 @@ asm_Zuc256Initialization_4_avx: ; Generate N*4B of keystream in N rounds %assign N 1 %rep %%NUM_ROUNDS - bits_reorg4 rax, N, xmm10 + BITS_REORG4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + xmm6, xmm7, xmm8, xmm9, xmm10 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 ; OFS_X3 XOR W (xmm7) and store in stack vpxor xmm10, xmm7 @@ -1097,7 +1093,8 @@ asm_ZucGenKeystream4B_4_avx: %assign %%N 1 %assign %%round (%%INITIAL_ROUND + %%N) %rep %%NROUNDS - bits_reorg4 rax, %%round, xmm10 + BITS_REORG4 rax, %%round, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + xmm6, xmm7, xmm8, xmm9, xmm10 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 ; OFS_XR XOR W (xmm7) and store in stack vpxor xmm10, xmm7 -- GitLab From e9c192b1a9e5abe29176d92d84836a288a170e34 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 10:01:45 +0000 Subject: [PATCH 005/369] avx: [ZUC] refactor lfsr_updt macro Remove hard-coded registers from LFSR update function macro, and passing the registers through parameters instead. --- lib/avx/zuc_x4_avx.asm | 159 ++++++++++++++++++++++------------------- lib/sse/zuc_x4_sse.asm | 10 +-- 2 files changed, 90 insertions(+), 79 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 6aa6e4ca..42a26260 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -211,8 +211,6 @@ endstruc mksection .text align 64 -%define MASK31 xmm12 - %define OFS_R1 (16*16) %define OFS_R2 (OFS_R1 + 16) %define OFS_X0 (OFS_R2 + 16) @@ -494,77 +492,82 @@ align 64 %endmacro ; -; add_mod31() -; add two 32-bit args and reduce mod (2^31-1) -; params -; %1 - arg1/res -; %2 - arg2 -; uses -; xmm2 -; return -; %1 -%macro add_mod31 2 - vpaddd %1, %2 - vpsrld xmm2, %1, 31 - vpand %1, MASK31 - vpaddd %1, xmm2 +; Add two 32-bit args and reduce mod (2^31-1) +; +%macro ADD_MOD31 4 +%define %%IN_OUT %1 ; [in/out] XMM register with first input and output +%define %%IN2 %2 ; [in] XMM register with second input +%define %%XTMP %3 ; [clobbered] Temporary XMM register +%define %%MASK31 %4 ; [in] XMM register containing 0x7FFFFFFF's in all dwords + vpaddd %%IN_OUT, %%IN2 + vpsrld %%XTMP, %%IN_OUT, 31 + vpand %%IN_OUT, %%MASK31 + vpaddd %%IN_OUT, %%XTMP %endmacro ; -; rot_mod31() -; rotate (mult by pow of 2) 32-bit arg and reduce mod (2^31-1) -; params -; %1 - arg -; %2 - # of bits -; uses -; xmm2 -; return -; %1 -%macro rot_mod31 2 +; Rotate (mult by pow of 2) 32-bit arg and reduce mod (2^31-1) +; +%macro ROT_MOD31 4 +%define %%IN_OUT %1 ; [in/out] XMM register with input and output +%define %%XTMP %2 ; [clobbered] Temporary XMM register +%define %%MASK31 %3 ; [in] XMM register containing 0x7FFFFFFF's in all dwords +%define %%N_BITS %4 ; [immediate] Number of bits to rotate for each dword - vpslld xmm2, %1, %2 - vpsrld %1, %1, (31 - %2) + vpslld %%XTMP, %%IN_OUT, %%N_BITS + vpsrld %%IN_OUT, (31 - %%N_BITS) - vpor %1, xmm2 - vpand %1, MASK31 + vpor %%IN_OUT, %%XTMP + vpand %%IN_OUT, %%MASK31 %endmacro ; -; lfsr_updt4() +; Update LFSR registers, calculating S_16 +; +; S_16 = [ 2^15*S_15 + 2^17*S_13 + 2^21*S_10 + 2^20*S_4 + (1 + 2^8)*S_0 ] mod (2^31 - 1) +; If init mode, add W to the calculation above. +; S_16 -> S_15 for next round ; -%macro lfsr_updt4 3 -%define %%STATE %1 ; [in] ZUC state -%define %%ROUND_NUM %2 ; [in] Round number -%define %%W %3 ; [in/clobbered] XMM register to contain W for all lanes - ; - ; xmm1 = LFSR_S0 - ; xmm4 = LFSR_S4 - ; xmm10 = LFSR_S10 - ; xmm13 = LFSR_S13 - ; xmm15 = LFSR_S15 - ; - vmovdqa xmm1, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*16] - vmovdqa xmm15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*16] - - ; Calculate LFSR feedback - add_mod31 %%W, xmm1 - rot_mod31 xmm1, 8 - add_mod31 %%W, xmm1 - rot_mod31 xmm4, 20 - add_mod31 %%W, xmm4 - rot_mod31 xmm10, 21 - add_mod31 %%W, xmm10 - rot_mod31 xmm13, 17 - add_mod31 %%W, xmm13 - rot_mod31 xmm15, 15 - add_mod31 %%W, xmm15 - - vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16], %%W - - ; LFSR_S16 = (LFSR_S15++) = eax +%macro LFSR_UPDT4 11 +%define %%STATE %1 ; [in] ZUC state +%define %%ROUND_NUM %2 ; [in] Round number +%define %%LFSR_0 %3 ; [clobbered] LFSR_0 (XMM) +%define %%LFSR_4 %4 ; [clobbered] LFSR_4 (XMM) +%define %%LFSR_10 %5 ; [clobbered] LFSR_10 (XMM) +%define %%LFSR_13 %6 ; [clobbered] LFSR_13 (XMM) +%define %%LFSR_15 %7 ; [clobbered] LFSR_15 (XMM) +%define %%XTMP %8 ; [clobbered] Temporary XMM register +%define %%MASK_31 %9 ; [in] Mask_31 +%define %%W %10 ; [in/clobbered] In init mode, contains W for all 4 lanes +%define %%MODE %11 ; [constant] "init" / "work" mode + + vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*16] + vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*16] + + ; Calculate LFSR feedback (S_16) + + ; In Init mode, W is added to the S_16 calculation +%ifidn %%MODE, init + ADD_MOD31 %%W, %%LFSR_0, %%XTMP, %%MASK_31 +%else + vmovdqa %%W, %%LFSR_0 +%endif + ROT_MOD31 %%LFSR_0, %%XTMP, %%MASK_31, 8 + ADD_MOD31 %%W, %%LFSR_0, %%XTMP, %%MASK_31 + ROT_MOD31 %%LFSR_4, %%XTMP, %%MASK_31, 20 + ADD_MOD31 %%W, %%LFSR_4, %%XTMP, %%MASK_31 + ROT_MOD31 %%LFSR_10, %%XTMP, %%MASK_31, 21 + ADD_MOD31 %%W, %%LFSR_10, %%XTMP, %%MASK_31 + ROT_MOD31 %%LFSR_13, %%XTMP, %%MASK_31, 17 + ADD_MOD31 %%W, %%LFSR_13, %%XTMP, %%MASK_31 + ROT_MOD31 %%LFSR_15, %%XTMP, %%MASK_31, 15 + ADD_MOD31 %%W, %%LFSR_15, %%XTMP, %%MASK_31 + + ; Store LFSR_S16 + vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16], %%W %endmacro ; @@ -747,6 +750,8 @@ align 64 %define tag_sz r9 ; Only used in ZUC-256 %endif +%define %%MASK_31 xmm15 + FUNC_SAVE mov rax, pState @@ -860,7 +865,7 @@ align 64 %endif ;; %%KEY_SIZE == 256 ; Load read-only registers - vmovdqa xmm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] ; Shift LFSR 32-times, update state variables %assign N 0 @@ -869,7 +874,8 @@ align 64 xmm6, xmm7, xmm8, xmm9 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 vpsrld xmm7, 1 ; Shift out LSB of W - lfsr_updt4 rax, N, xmm7 ; W (xmm7) used in LFSR update + LFSR_UPDT4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + %%MASK_31, xmm7, init ; W (xmm7) used in LFSR update %assign N N+1 %endrep @@ -877,8 +883,8 @@ align 64 BITS_REORG4 rax, 0, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ xmm6, xmm7, xmm8, xmm9 NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - vpxor xmm0, xmm0 - lfsr_updt4 rax, 0, xmm0 + LFSR_UPDT4 rax, 0, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + %%MASK_31, xmm7, work FUNC_RESTORE @@ -936,6 +942,8 @@ asm_Zuc256Initialization_4_avx: %define pKS rdx %endif +%define %%MASK_31 xmm15 + FUNC_SAVE ; Store 4 keystream pointers on the stack @@ -955,7 +963,7 @@ asm_Zuc256Initialization_4_avx: mov rax, pState ; Load read-only registers - vmovdqa xmm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds %assign N 1 @@ -966,8 +974,8 @@ asm_Zuc256Initialization_4_avx: ; OFS_X3 XOR W (xmm7) and store in stack vpxor xmm10, xmm7 vmovdqa [rsp + 4*8 + (N-1)*16], xmm10 - vpxor xmm0, xmm0 - lfsr_updt4 rax, N, xmm0 + LFSR_UPDT4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + %%MASK_31, xmm7, work %assign N N+1 %endrep @@ -1086,8 +1094,11 @@ asm_ZucGenKeystream4B_4_avx: %define %%TMP1 rdi %define %%TMP2 rsi %endif + +%define %%MASK_31 xmm15 + ; Load read-only registers - vmovdqa xmm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds %assign %%N 1 @@ -1099,8 +1110,8 @@ asm_ZucGenKeystream4B_4_avx: ; OFS_XR XOR W (xmm7) and store in stack vpxor xmm10, xmm7 vmovdqa [rsp + _keystr_save + (%%N-1)*16], xmm10 - vpxor xmm0, xmm0 - lfsr_updt4 rax, %%round, xmm0 + LFSR_UPDT4 rax, %%round, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ + %%MASK_31, xmm7, work %assign %%N (%%N + 1) %assign %%round (%%round + 1) %endrep diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index a424ef00..1a2bf816 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -628,11 +628,11 @@ mksection .text %define %%STATE %1 ; [in] ZUC state %define %%ROUND_NUM %2 ; [in] Round number %define %%TMP %3 ; [clobbered] Temporary GP register (used when ROUND_NUM is a register) -%define %%LFSR_0 %4 ; [clobbered] LFSR_0 -%define %%LFSR_4 %5 ; [clobbered] LFSR_2 -%define %%LFSR_10 %6 ; [clobbered] LFSR_5 -%define %%LFSR_13 %7 ; [clobbered] LFSR_7 -%define %%LFSR_15 %8 ; [clobbered] LFSR_9 +%define %%LFSR_0 %4 ; [clobbered] LFSR_0 (XMM) +%define %%LFSR_4 %5 ; [clobbered] LFSR_4 (XMM) +%define %%LFSR_10 %6 ; [clobbered] LFSR_10 (XMM) +%define %%LFSR_13 %7 ; [clobbered] LFSR_13 (XMM) +%define %%LFSR_15 %8 ; [clobbered] LFSR_15 (XMM) %define %%XTMP %9 ; [clobbered] Temporary XMM register %define %%MASK_31 %10 ; [in] Mask_31 %define %%W %11 ; [in/clobbered] In init mode, contains W for all 4 lanes -- GitLab From 1dc7954cd54e0727ebf4de6fdefc6599d60d8d9d Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 10:21:41 +0000 Subject: [PATCH 006/369] avx: [ZUC] refactor main auxiliary functions Refactored init, keygen and cipher assembly functions, defining register names and fixing leading spaces. --- lib/avx/zuc_x4_avx.asm | 687 +++++++++++++++++++++-------------------- lib/sse/zuc_x4_sse.asm | 6 +- 2 files changed, 358 insertions(+), 335 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 42a26260..8de3358f 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -591,11 +591,11 @@ align 64 %define %%LFSR %6 ;; [out] XMM register to contain initialized LFSR regs %define %%XTMP %7 ;; [clobbered] XMM temporary register - vpshufb %%LFSR, %%KEY, %%SHUF_KEY - vpsrld %%LFSR, 1 - vpshufb %%XTMP, %%IV, %%SHUF_IV - vpor %%LFSR, %%XTMP - vpor %%LFSR, %%EKD_MASK + vpshufb %%LFSR, %%KEY, %%SHUF_KEY + vpsrld %%LFSR, 1 + vpshufb %%XTMP, %%IV, %%SHUF_IV + vpor %%LFSR, %%XTMP + vpor %%LFSR, %%EKD_MASK %endmacro @@ -613,282 +613,287 @@ align 64 %define %%TMP %8 ;; [clobbered] GP temporary register %define %%CONSTANTS %9 ;; [in] Address to constants - ; s0 - s3 - vpxor %%LFSR0_3, %%LFSR0_3 - vpinsrb %%LFSR0_3, [%%KEY], 3 ; s0 - vpinsrb %%LFSR0_3, [%%KEY + 1], 7 ; s1 - vpinsrb %%LFSR0_3, [%%KEY + 2], 11 ; s2 - vpinsrb %%LFSR0_3, [%%KEY + 3], 15 ; s3 + ; s0 - s3 + vpxor %%LFSR0_3, %%LFSR0_3 + vpinsrb %%LFSR0_3, [%%KEY], 3 ; s0 + vpinsrb %%LFSR0_3, [%%KEY + 1], 7 ; s1 + vpinsrb %%LFSR0_3, [%%KEY + 2], 11 ; s2 + vpinsrb %%LFSR0_3, [%%KEY + 3], 15 ; s3 - vpsrld %%LFSR0_3, 1 + vpsrld %%LFSR0_3, 1 - vpor %%LFSR0_3, [%%CONSTANTS] ; s0 - s3 + vpor %%LFSR0_3, [%%CONSTANTS] ; s0 - s3 - vpinsrb %%LFSR0_3, [%%KEY + 21], 1 ; s0 - vpinsrb %%LFSR0_3, [%%KEY + 16], 0 ; s0 + vpinsrb %%LFSR0_3, [%%KEY + 21], 1 ; s0 + vpinsrb %%LFSR0_3, [%%KEY + 16], 0 ; s0 - vpinsrb %%LFSR0_3, [%%KEY + 22], 5 ; s1 - vpinsrb %%LFSR0_3, [%%KEY + 17], 4 ; s1 + vpinsrb %%LFSR0_3, [%%KEY + 22], 5 ; s1 + vpinsrb %%LFSR0_3, [%%KEY + 17], 4 ; s1 - vpinsrb %%LFSR0_3, [%%KEY + 23], 9 ; s2 - vpinsrb %%LFSR0_3, [%%KEY + 18], 8 ; s2 + vpinsrb %%LFSR0_3, [%%KEY + 23], 9 ; s2 + vpinsrb %%LFSR0_3, [%%KEY + 18], 8 ; s2 - vpinsrb %%LFSR0_3, [%%KEY + 24], 13 ; s3 - vpinsrb %%LFSR0_3, [%%KEY + 19], 12 ; s3 + vpinsrb %%LFSR0_3, [%%KEY + 24], 13 ; s3 + vpinsrb %%LFSR0_3, [%%KEY + 19], 12 ; s3 - ; s4 - s7 - vpxor %%LFSR4_7, %%LFSR4_7 - vpinsrb %%LFSR4_7, [%%KEY + 4], 3 ; s4 - vpinsrb %%LFSR4_7, [%%IV], 7 ; s5 - vpinsrb %%LFSR4_7, [%%IV + 1], 11 ; s6 - vpinsrb %%LFSR4_7, [%%IV + 10], 15 ; s7 + ; s4 - s7 + vpxor %%LFSR4_7, %%LFSR4_7 + vpinsrb %%LFSR4_7, [%%KEY + 4], 3 ; s4 + vpinsrb %%LFSR4_7, [%%IV], 7 ; s5 + vpinsrb %%LFSR4_7, [%%IV + 1], 11 ; s6 + vpinsrb %%LFSR4_7, [%%IV + 10], 15 ; s7 - vpsrld %%LFSR4_7, 1 + vpsrld %%LFSR4_7, 1 - vpinsrb %%LFSR4_7, [%%KEY + 25], 1 ; s4 - vpinsrb %%LFSR4_7, [%%KEY + 20], 0 ; s4 + vpinsrb %%LFSR4_7, [%%KEY + 25], 1 ; s4 + vpinsrb %%LFSR4_7, [%%KEY + 20], 0 ; s4 - vpinsrb %%LFSR4_7, [%%KEY + 5], 5 ; s5 - vpinsrb %%LFSR4_7, [%%KEY + 26], 4 ; s5 + vpinsrb %%LFSR4_7, [%%KEY + 5], 5 ; s5 + vpinsrb %%LFSR4_7, [%%KEY + 26], 4 ; s5 - vpinsrb %%LFSR4_7, [%%KEY + 6], 9 ; s6 - vpinsrb %%LFSR4_7, [%%KEY + 27], 8 ; s6 + vpinsrb %%LFSR4_7, [%%KEY + 6], 9 ; s6 + vpinsrb %%LFSR4_7, [%%KEY + 27], 8 ; s6 - vpinsrb %%LFSR4_7, [%%KEY + 7], 13 ; s7 - vpinsrb %%LFSR4_7, [%%IV + 2], 12 ; s7 + vpinsrb %%LFSR4_7, [%%KEY + 7], 13 ; s7 + vpinsrb %%LFSR4_7, [%%IV + 2], 12 ; s7 - vpor %%LFSR4_7, [%%CONSTANTS + 16] ; s4 - s7 + vpor %%LFSR4_7, [%%CONSTANTS + 16] ; s4 - s7 - vmovd %%XTMP, [%%IV + 17] - vpshufb %%XTMP, [rel shuf_mask_iv_17_19] - vpand %%XTMP, [rel clear_iv_mask] + vmovd %%XTMP, [%%IV + 17] + vpshufb %%XTMP, [rel shuf_mask_iv_17_19] + vpand %%XTMP, [rel clear_iv_mask] - vpor %%LFSR4_7, %%XTMP + vpor %%LFSR4_7, %%XTMP - ; s8 - s11 - vpxor %%LFSR8_11, %%LFSR8_11 - vpinsrb %%LFSR8_11, [%%KEY + 8], 3 ; s8 - vpinsrb %%LFSR8_11, [%%KEY + 9], 7 ; s9 - vpinsrb %%LFSR8_11, [%%IV + 5], 11 ; s10 - vpinsrb %%LFSR8_11, [%%KEY + 11], 15 ; s11 + ; s8 - s11 + vpxor %%LFSR8_11, %%LFSR8_11 + vpinsrb %%LFSR8_11, [%%KEY + 8], 3 ; s8 + vpinsrb %%LFSR8_11, [%%KEY + 9], 7 ; s9 + vpinsrb %%LFSR8_11, [%%IV + 5], 11 ; s10 + vpinsrb %%LFSR8_11, [%%KEY + 11], 15 ; s11 - vpsrld %%LFSR8_11, 1 + vpsrld %%LFSR8_11, 1 - vpinsrb %%LFSR8_11, [%%IV + 3], 1 ; s8 - vpinsrb %%LFSR8_11, [%%IV + 11], 0 ; s8 + vpinsrb %%LFSR8_11, [%%IV + 3], 1 ; s8 + vpinsrb %%LFSR8_11, [%%IV + 11], 0 ; s8 - vpinsrb %%LFSR8_11, [%%IV + 12], 5 ; s9 - vpinsrb %%LFSR8_11, [%%IV + 4], 4 ; s9 + vpinsrb %%LFSR8_11, [%%IV + 12], 5 ; s9 + vpinsrb %%LFSR8_11, [%%IV + 4], 4 ; s9 - vpinsrb %%LFSR8_11, [%%KEY + 10], 9 ; s10 - vpinsrb %%LFSR8_11, [%%KEY + 28], 8 ; s10 + vpinsrb %%LFSR8_11, [%%KEY + 10], 9 ; s10 + vpinsrb %%LFSR8_11, [%%KEY + 28], 8 ; s10 - vpinsrb %%LFSR8_11, [%%IV + 6], 13 ; s11 - vpinsrb %%LFSR8_11, [%%IV + 13], 12 ; s11 + vpinsrb %%LFSR8_11, [%%IV + 6], 13 ; s11 + vpinsrb %%LFSR8_11, [%%IV + 13], 12 ; s11 - vpor %%LFSR8_11, [%%CONSTANTS + 32] ; s8 - s11 + vpor %%LFSR8_11, [%%CONSTANTS + 32] ; s8 - s11 - vmovd %%XTMP, [%%IV + 20] - vpshufb %%XTMP, [rel shuf_mask_iv_20_23] - vpand %%XTMP, [rel clear_iv_mask] + vmovd %%XTMP, [%%IV + 20] + vpshufb %%XTMP, [rel shuf_mask_iv_20_23] + vpand %%XTMP, [rel clear_iv_mask] - vpor %%LFSR8_11, %%XTMP + vpor %%LFSR8_11, %%XTMP - ; s12 - s15 - vpxor %%LFSR12_15, %%LFSR12_15 - vpinsrb %%LFSR12_15, [%%KEY + 12], 3 ; s12 - vpinsrb %%LFSR12_15, [%%KEY + 13], 7 ; s13 - vpinsrb %%LFSR12_15, [%%KEY + 14], 11 ; s14 - vpinsrb %%LFSR12_15, [%%KEY + 15], 15 ; s15 + ; s12 - s15 + vpxor %%LFSR12_15, %%LFSR12_15 + vpinsrb %%LFSR12_15, [%%KEY + 12], 3 ; s12 + vpinsrb %%LFSR12_15, [%%KEY + 13], 7 ; s13 + vpinsrb %%LFSR12_15, [%%KEY + 14], 11 ; s14 + vpinsrb %%LFSR12_15, [%%KEY + 15], 15 ; s15 - vpsrld %%LFSR12_15, 1 + vpsrld %%LFSR12_15, 1 - vpinsrb %%LFSR12_15, [%%IV + 7], 1 ; s12 - vpinsrb %%LFSR12_15, [%%IV + 14], 0 ; s12 + vpinsrb %%LFSR12_15, [%%IV + 7], 1 ; s12 + vpinsrb %%LFSR12_15, [%%IV + 14], 0 ; s12 - vpinsrb %%LFSR12_15, [%%IV + 15], 5 ; s13 - vpinsrb %%LFSR12_15, [%%IV + 8], 4 ; s13 + vpinsrb %%LFSR12_15, [%%IV + 15], 5 ; s13 + vpinsrb %%LFSR12_15, [%%IV + 8], 4 ; s13 - vpinsrb %%LFSR12_15, [%%IV + 16], 9 ; s14 - vpinsrb %%LFSR12_15, [%%IV + 9], 8 ; s14 + vpinsrb %%LFSR12_15, [%%IV + 16], 9 ; s14 + vpinsrb %%LFSR12_15, [%%IV + 9], 8 ; s14 - vpinsrb %%LFSR12_15, [%%KEY + 30], 13 ; s15 - vpinsrb %%LFSR12_15, [%%KEY + 29], 12 ; s15 + vpinsrb %%LFSR12_15, [%%KEY + 30], 13 ; s15 + vpinsrb %%LFSR12_15, [%%KEY + 29], 12 ; s15 - vpor %%LFSR12_15, [%%CONSTANTS + 48] ; s12 - s15 + vpor %%LFSR12_15, [%%CONSTANTS + 48] ; s12 - s15 - movzx DWORD(%%TMP), byte [%%IV + 24] - and DWORD(%%TMP), 0x0000003f - shl DWORD(%%TMP), 16 - vmovd %%XTMP, DWORD(%%TMP) + movzx DWORD(%%TMP), byte [%%IV + 24] + and DWORD(%%TMP), 0x0000003f + shl DWORD(%%TMP), 16 + vmovd %%XTMP, DWORD(%%TMP) - movzx DWORD(%%TMP), byte [%%KEY + 31] - shl DWORD(%%TMP), 12 - and DWORD(%%TMP), 0x000f0000 ; high nibble of K_31 - vpinsrd %%XTMP, DWORD(%%TMP), 2 + movzx DWORD(%%TMP), byte [%%KEY + 31] + shl DWORD(%%TMP), 12 + and DWORD(%%TMP), 0x000f0000 ; high nibble of K_31 + vpinsrd %%XTMP, DWORD(%%TMP), 2 - movzx DWORD(%%TMP), byte [%%KEY + 31] - shl DWORD(%%TMP), 16 - and DWORD(%%TMP), 0x000f0000 ; low nibble of K_31 - vpinsrd %%XTMP, DWORD(%%TMP), 3 + movzx DWORD(%%TMP), byte [%%KEY + 31] + shl DWORD(%%TMP), 16 + and DWORD(%%TMP), 0x000f0000 ; low nibble of K_31 + vpinsrd %%XTMP, DWORD(%%TMP), 3 - vpor %%LFSR12_15, %%XTMP + vpor %%LFSR12_15, %%XTMP %endmacro %macro ZUC_INIT_4 1 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) -%ifdef LINUX - %define pKe rdi - %define pIv rsi - %define pState rdx - %define tag_sz rcx ; Only used in ZUC-256 -%else - %define pKe rcx - %define pIv rdx - %define pState r8 - %define tag_sz r9 ; Only used in ZUC-256 -%endif - -%define %%MASK_31 xmm15 - - FUNC_SAVE +%define pKe arg1 +%define pIv arg2 +%define pState arg3 +%define tag_sz arg4 ; Only used in ZUC-256 + +%define %%XTMP1 xmm0 +%define %%XTMP2 xmm1 +%define %%XTMP3 xmm2 +%define %%XTMP4 xmm3 +%define %%XTMP5 xmm4 +%define %%XTMP6 xmm5 +%define %%XTMP7 xmm6 +%define %%XTMP8 xmm7 +%define %%XTMP9 xmm8 +%define %%XTMP10 xmm9 +%define %%XTMP11 xmm10 +%define %%XTMP12 xmm11 +%define %%XTMP13 xmm12 +%define %%XTMP14 xmm13 +%define %%XTMP15 xmm14 +%define %%XTMP16 xmm15 + +%define %%W %%XTMP10 +%define %%X3 %%XTMP11 +%define %%MASK_31 %%XTMP16 - mov rax, pState + FUNC_SAVE - ; Zero out R1-R2 (only lower 128 bits) - vpxor xmm0, xmm0 -%assign I 0 -%rep 2 - vmovdqa [pState + OFS_R1 + I*16], xmm0 -%assign I (I + 1) -%endrep + ; Zero out R1-R2 + vpxor %%XTMP1, %%XTMP1 + vmovdqa [pState + OFS_R1], %%XTMP1 + vmovdqa [pState + OFS_R2], %%XTMP1 %if %%KEY_SIZE == 128 - ;; Load key and IVs -%assign off 0 -%assign i 4 -%assign j 8 + ;; Load key and IVs +%assign %%OFF 0 +%assign %%I 1 +%assign %%J 5 %rep 4 - mov r9, [pKe + off] - vmovdqu APPEND(xmm,i), [r9] - ; Read 16 bytes of IV - vmovdqa APPEND(xmm,j), [pIv + off*4] -%assign off (off + 8) -%assign i (i + 1) -%assign j (j + 1) + mov r15, [pKe + %%OFF] + vmovdqu APPEND(%%XTMP, %%I), [r15] + ; Read 16 bytes of IV + vmovdqa APPEND(%%XTMP, %%J), [pIv + %%OFF*4] +%assign %%OFF (%%OFF + 8) +%assign %%I (%%I + 1) +%assign %%J (%%J + 1) %endrep - ;;; Initialize all LFSR registers in four steps: - ;;; first, registers 0-3, then registers 4-7, 8-11, 12-15 -%assign off 0 + ;;; Initialize all LFSR registers in four steps: + ;;; first, registers 0-3, then registers 4-7, 8-11, 12-15 +%assign %%OFF 0 %rep 4 - ; Set read-only registers for shuffle masks for key, IV and Ek_d for 8 registers - vmovdqa xmm13, [rel shuf_mask_key + off] - vmovdqa xmm14, [rel shuf_mask_iv + off] - vmovdqa xmm15, [rel Ek_d + off] - - ; Set 4xLFSR registers for all packets -%assign idx 0 -%assign i 4 -%assign j 8 + ; Set read-only registers for shuffle masks for key, IV and Ek_d for 8 registers + vmovdqa %%XTMP13, [rel shuf_mask_key + %%OFF] + vmovdqa %%XTMP14, [rel shuf_mask_iv + %%OFF] + vmovdqa %%XTMP15, [rel Ek_d + %%OFF] + + ; Set 4xLFSR registers for all packets +%assign %%IDX 9 +%assign %%I 1 +%assign %%J 5 %rep 4 - INIT_LFSR_128 APPEND(xmm,i), APPEND(xmm,j), xmm13, xmm14, xmm15, APPEND(xmm, idx), xmm12 -%assign idx (idx + 1) -%assign i (i + 1) -%assign j (j + 1) + INIT_LFSR_128 APPEND(%%XTMP,%%I), APPEND(%%XTMP,%%J), %%XTMP13, %%XTMP14, \ + %%XTMP15, APPEND(%%XTMP, %%IDX), %%XTMP16 +%assign %%IDX (%%IDX + 1) +%assign %%I (%%I + 1) +%assign %%J (%%J + 1) %endrep - ; Store 4xLFSR registers in memory (reordering first, - ; so all SX registers are together) - TRANSPOSE4_U32 xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 + ; Store 4xLFSR registers in memory (reordering first, + ; so all SX registers are together) + TRANSPOSE4_U32 %%XTMP9, %%XTMP10, %%XTMP11, %%XTMP12, %%XTMP13, %%XTMP14 -%assign i 0 -%rep 4 - vmovdqa [pState + 4*off + 16*i], APPEND(xmm, i) -%assign i (i+1) -%endrep + vmovdqa [pState + 4*%%OFF], %%XTMP9 + vmovdqa [pState + 4*%%OFF + 16], %%XTMP10 + vmovdqa [pState + 4*%%OFF + 16*2], %%XTMP11 + vmovdqa [pState + 4*%%OFF + 16*3], %%XTMP12 -%assign off (off + 16) +%assign %%OFF (%%OFF + 16) %endrep %else ;; %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea r13, [rel EK256_d64] - bsf DWORD(tag_sz), DWORD(tag_sz) - dec DWORD(tag_sz) - shl DWORD(tag_sz), 6 - add r13, tag_sz + ; Get pointer to constants (depending on tag size, this will point at + ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) + lea r13, [rel EK256_d64] + bsf DWORD(tag_sz), DWORD(tag_sz) + dec DWORD(tag_sz) + shl DWORD(tag_sz), 6 + add r13, tag_sz ;;; Initialize all LFSR registers -%assign off 0 -%rep 4 - ;; Load key and IV for each packet - mov r12, [pKe + off] - lea r10, [pIv + off*4] - - ; Initialize S0-15 for each packet - INIT_LFSR_256 r12, r10, xmm0, xmm1, xmm2, xmm3, xmm4, r11, r13 - -%assign i 0 +%assign %%OFF 0 %rep 4 - vmovdqa [pState + 64*i + 2*off], APPEND(xmm, i) -%assign i (i+1) -%endrep - -%assign off (off + 8) + ;; Load key and IV for each packet + mov r15, [pKe + %%OFF] + lea r10, [pIv + %%OFF*4] + + ; Initialize S0-15 for each packet + INIT_LFSR_256 r15, r10, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, r11, r13 + + vmovdqa [pState + 2*%%OFF], %%XTMP1 + vmovdqa [pState + 2*%%OFF + 64], %%XTMP2 + vmovdqa [pState + 2*%%OFF + 64*2], %%XTMP3 + vmovdqa [pState + 2*%%OFF + 64*3], %%XTMP4 +%assign %%OFF (%%OFF + 8) %endrep - ; Read, transpose and store, so all S_X from the 4 packets are in the same register -%assign off 0 -%rep 4 - -%assign i 0 + ; Read, transpose and store, so all S_X from the 4 packets are + ; in the same register +%assign %%OFF 0 %rep 4 - vmovdqa APPEND(xmm, i), [pState + 16*i + off] -%assign i (i+1) -%endrep + vmovdqa %%XTMP1, [pState + %%OFF] + vmovdqa %%XTMP2, [pState + %%OFF + 16] + vmovdqa %%XTMP3, [pState + %%OFF + 16*2] + vmovdqa %%XTMP4, [pState + %%OFF + 16*3] - TRANSPOSE4_U32 xmm0, xmm1, xmm2, xmm3, xmm14, xmm15 + TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 -%assign i 0 -%rep 4 - vmovdqa [pState + 16*i + off], APPEND(xmm, i) -%assign i (i+1) -%endrep + vmovdqa [pState + %%OFF], %%XTMP1 + vmovdqa [pState + %%OFF + 16], %%XTMP2 + vmovdqa [pState + %%OFF + 16*2], %%XTMP3 + vmovdqa [pState + %%OFF + 16*3], %%XTMP4 -%assign off (off + 64) +%assign %%OFF (%%OFF + 64) %endrep %endif ;; %%KEY_SIZE == 256 - ; Load read-only registers - vmovdqa %%MASK_31, [rel mask31] + ; Load read-only registers + vmovdqa %%MASK_31, [rel mask31] ; Shift LFSR 32-times, update state variables %assign N 0 %rep 32 - BITS_REORG4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - xmm6, xmm7, xmm8, xmm9 - NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - vpsrld xmm7, 1 ; Shift out LSB of W - LFSR_UPDT4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - %%MASK_31, xmm7, init ; W (xmm7) used in LFSR update + BITS_REORG4 pState, N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10 + NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ + %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W + vpsrld %%W, 1 ; Shift out LSB of W + LFSR_UPDT4 pState, N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + %%MASK_31, %%W, init ; W used in LFSR update %assign N N+1 %endrep - ; And once more, initial round from keygen phase = 33 times - BITS_REORG4 rax, 0, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - xmm6, xmm7, xmm8, xmm9 - NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - LFSR_UPDT4 rax, 0, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - %%MASK_31, xmm7, work + ; And once more, initial round from keygen phase = 33 times + BITS_REORG4 pState, 0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10 + NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ + %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W + LFSR_UPDT4 pState, 0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + %%MASK_31, %%XTMP8, work - FUNC_RESTORE + FUNC_RESTORE - ret + ret %endmacro MKGLOBAL(asm_ZucInitialization_4_avx,function,internal) @@ -912,14 +917,14 @@ asm_Zuc256Initialization_4_avx: %if %%NUM_ROUNDS != 16 %assign %%i 0 %rep 16 - vmovdqa APPEND(xmm,%%i), [%%STATE + 16*%%i] + vmovdqa APPEND(xmm,%%i), [%%STATE + 16*%%i] %assign %%i (%%i+1) %endrep %assign %%i 0 %assign %%j %%NUM_ROUNDS %rep 16 - vmovdqa [%%STATE + 16*%%i], APPEND(xmm,%%j) + vmovdqa [%%STATE + 16*%%i], APPEND(xmm,%%j) %assign %%i (%%i+1) %assign %%j ((%%j+1) % 16) %endrep @@ -934,89 +939,99 @@ asm_Zuc256Initialization_4_avx: %macro KEYGEN_4_AVX 1 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds -%ifdef LINUX - %define pState rdi - %define pKS rsi -%else - %define pState rcx - %define pKS rdx -%endif +%define pState arg1 +%define pKS arg2 + +%define %%XTMP1 xmm0 +%define %%XTMP2 xmm1 +%define %%XTMP3 xmm2 +%define %%XTMP4 xmm3 +%define %%XTMP5 xmm4 +%define %%XTMP6 xmm5 +%define %%XTMP7 xmm6 +%define %%XTMP8 xmm7 +%define %%XTMP9 xmm8 +%define %%XTMP10 xmm9 +%define %%XTMP11 xmm10 +%define %%XTMP12 xmm11 +%define %%XTMP13 xmm12 +%define %%XTMP14 xmm13 +%define %%XTMP15 xmm14 +%define %%XTMP16 xmm15 + +%define %%W %%XTMP10 +%define %%X3 %%XTMP11 +%define %%MASK_31 %%XTMP16 -%define %%MASK_31 xmm15 - - FUNC_SAVE + FUNC_SAVE - ; Store 4 keystream pointers on the stack - ; and reserve memory for storing keystreams for all 4 buffers - mov r10, rsp - sub rsp, (4*8 + %%NUM_ROUNDS * 16) - and rsp, -16 + ; Store 4 keystream pointers on the stack + ; and reserve memory for storing keystreams for all 4 buffers + mov r10, rsp + sub rsp, (4*8 + %%NUM_ROUNDS * 16) + and rsp, -16 -%assign i 0 -%rep 2 - vmovdqa xmm0, [pKS + 16*i] - vmovdqa [rsp + 16*i], xmm0 -%assign i (i+1) -%endrep + vmovdqa %%XTMP1, [pKS] + vmovdqa %%XTMP2, [pKS + 16] + vmovdqa [rsp], %%XTMP1 + vmovdqa [rsp + 8*2], %%XTMP2 - ; Load state pointer in RAX - mov rax, pState + ; Load read-only registers + vmovdqa %%MASK_31, [rel mask31] - ; Load read-only registers - vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds -%assign N 1 +%assign %%N 1 %rep %%NUM_ROUNDS - BITS_REORG4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - xmm6, xmm7, xmm8, xmm9, xmm10 - NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - ; OFS_X3 XOR W (xmm7) and store in stack - vpxor xmm10, xmm7 - vmovdqa [rsp + 4*8 + (N-1)*16], xmm10 - LFSR_UPDT4 rax, N, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - %%MASK_31, xmm7, work -%assign N N+1 + BITS_REORG4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, %%X3 + NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ + %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor %%X3, %%W + vmovdqa [rsp + 4*8 + (%%N-1)*16], %%X3 + LFSR_UPDT4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + %%MASK_31, %%XTMP8, work +%assign %%N (%%N + 1) %endrep %if (%%NUM_ROUNDS == 4) - ;; Load all OFS_X3 -%assign i 0 -%rep 4 - vmovdqa APPEND(xmm,i), [rsp + 4*8 + i*16] -%assign i (i+1) -%endrep + ;; Load all OFS_X3 + vmovdqa %%XTMP1, [rsp + 4*8] + vmovdqa %%XTMP2, [rsp + 4*8 + 16] + vmovdqa %%XTMP3, [rsp + 4*8 + 16*2] + vmovdqa %%XTMP4, [rsp + 4*8 + 16*3] - TRANSPOSE4_U32 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5 + TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 - store16B_kstr4 xmm0, xmm1, xmm2, xmm3 + store16B_kstr4 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4 %else ;; NUM_ROUNDS != 4 -%assign idx 0 +%assign %%IDX 1 %rep %%NUM_ROUNDS - vmovdqa APPEND(xmm, idx), [rsp + 4*8 + idx*16] - store4B_kstr4 APPEND(xmm, idx) -%assign idx (idx + 1) + vmovdqa APPEND(%%XTMP, %%IDX), [rsp + 4*8 + (%%IDX-1)*16] + store4B_kstr4 APPEND(%%XTMP, %%IDX) +%assign %%IDX (%%IDX + 1) %endrep %endif ;; NUM_ROUNDS == 4 ;; Clear stack frame containing keystream information %ifdef SAFE_DATA - vpxor xmm0, xmm0 -%assign i 0 -%rep (2+%%NUM_ROUNDS) - vmovdqa [rsp + i*16], xmm0 -%assign i (i+1) + vpxor %%XTMP1, %%XTMP1 +%assign %%I 0 +%rep (2 + %%NUM_ROUNDS) + vmovdqa [rsp + %%I*16], %%XTMP1 +%assign %%I (%%I + 1) %endrep %endif - ;; Reorder memory for LFSR registers, as not all 16 rounds - ;; will be completed (can be 4 or 2) - REORDER_LFSR rax, %%NUM_ROUNDS + ;; Reorder memory for LFSR registers, as not all 16 rounds + ;; will be completed (can be 4 or 2) + REORDER_LFSR pState, %%NUM_ROUNDS - ;; Restore rsp pointer to value before pushing keystreams - mov rsp, r10 + ;; Restore rsp pointer to value before pushing keystreams + mov rsp, r10 - FUNC_RESTORE + FUNC_RESTORE %endmacro @@ -1095,7 +1110,26 @@ asm_ZucGenKeystream4B_4_avx: %define %%TMP2 rsi %endif -%define %%MASK_31 xmm15 +%define %%XTMP1 xmm0 +%define %%XTMP2 xmm1 +%define %%XTMP3 xmm2 +%define %%XTMP4 xmm3 +%define %%XTMP5 xmm4 +%define %%XTMP6 xmm5 +%define %%XTMP7 xmm6 +%define %%XTMP8 xmm7 +%define %%XTMP9 xmm8 +%define %%XTMP10 xmm9 +%define %%XTMP11 xmm10 +%define %%XTMP12 xmm11 +%define %%XTMP13 xmm12 +%define %%XTMP14 xmm13 +%define %%XTMP15 xmm14 +%define %%XTMP16 xmm15 + +%define %%W %%XTMP10 +%define %%X3 %%XTMP11 +%define %%MASK_31 %%XTMP16 ; Load read-only registers vmovdqa %%MASK_31, [rel mask31] @@ -1104,29 +1138,28 @@ asm_ZucGenKeystream4B_4_avx: %assign %%N 1 %assign %%round (%%INITIAL_ROUND + %%N) %rep %%NROUNDS - BITS_REORG4 rax, %%round, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - xmm6, xmm7, xmm8, xmm9, xmm10 - NONLIN_FUN4 rax, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7 - ; OFS_XR XOR W (xmm7) and store in stack - vpxor xmm10, xmm7 - vmovdqa [rsp + _keystr_save + (%%N-1)*16], xmm10 - LFSR_UPDT4 rax, %%round, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, \ - %%MASK_31, xmm7, work + BITS_REORG4 pState, %%round, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, %%X3 + NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ + %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor %%X3, %%W + vmovdqa [rsp + _keystr_save + (%%N-1)*16], %%X3 + LFSR_UPDT4 pState, %%round, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + %%MASK_31, %%XTMP8, work %assign %%N (%%N + 1) %assign %%round (%%round + 1) %endrep -%assign %%N 0 -%assign %%idx 4 +%assign %%N 1 %rep %%NROUNDS - vmovdqa APPEND(xmm, %%idx), [rsp + _keystr_save + %%N*16] + vmovdqa APPEND(%%XTMP, %%N), [rsp + _keystr_save + (%%N-1)*16] %assign %%N (%%N + 1) -%assign %%idx (%%idx+1) %endrep - TRANSPOSE4_U32 xmm4, xmm5, xmm6, xmm7, xmm8, xmm9 + TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 - vmovdqa xmm15, [rel swap_mask] + vmovdqa %%XTMP15, [rel swap_mask] ;; XOR Input buffer with keystream in rounds of 16B mov r12, [pIn] @@ -1140,29 +1173,29 @@ asm_ZucGenKeystream4B_4_avx: ;; Read in r10 the word containing the number of final bytes to read for each lane movzx r10d, word [rsp + _rem_bytes_save] - simd_load_avx_16_1 xmm0, r12 + %%OFFSET, r10 + simd_load_avx_16_1 %%XTMP5, r12 + %%OFFSET, r10 movzx r10d, word [rsp + _rem_bytes_save + 2] - simd_load_avx_16_1 xmm1, r13 + %%OFFSET, r10 + simd_load_avx_16_1 %%XTMP6, r13 + %%OFFSET, r10 movzx r10d, word [rsp + _rem_bytes_save + 4] - simd_load_avx_16_1 xmm2, r14 + %%OFFSET, r10 + simd_load_avx_16_1 %%XTMP7, r14 + %%OFFSET, r10 movzx r10d, word [rsp + _rem_bytes_save + 6] - simd_load_avx_16_1 xmm3, r15 + %%OFFSET, r10 + simd_load_avx_16_1 %%XTMP8, r15 + %%OFFSET, r10 %else - vmovdqu xmm0, [r12 + %%OFFSET] - vmovdqu xmm1, [r13 + %%OFFSET] - vmovdqu xmm2, [r14 + %%OFFSET] - vmovdqu xmm3, [r15 + %%OFFSET] + vmovdqu %%XTMP5, [r12 + %%OFFSET] + vmovdqu %%XTMP6, [r13 + %%OFFSET] + vmovdqu %%XTMP7, [r14 + %%OFFSET] + vmovdqu %%XTMP8, [r15 + %%OFFSET] %endif - vpshufb xmm4, xmm15 - vpshufb xmm5, xmm15 - vpshufb xmm6, xmm15 - vpshufb xmm7, xmm15 + vpshufb %%XTMP1, %%XTMP15 + vpshufb %%XTMP2, %%XTMP15 + vpshufb %%XTMP3, %%XTMP15 + vpshufb %%XTMP4, %%XTMP15 - vpxor xmm4, xmm0 - vpxor xmm5, xmm1 - vpxor xmm6, xmm2 - vpxor xmm7, xmm3 + vpxor %%XTMP1, %%XTMP5 + vpxor %%XTMP2, %%XTMP6 + vpxor %%XTMP3, %%XTMP7 + vpxor %%XTMP4, %%XTMP8 mov r12, [pOut] mov r13, [pOut + 8] @@ -1171,22 +1204,22 @@ asm_ZucGenKeystream4B_4_avx: %if (%%LAST_CALL == 1) movzx r10d, word [rsp + _rem_bytes_save] - simd_store_avx r12, xmm4, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r12, %%XTMP1, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 2] - simd_store_avx r13, xmm5, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r13, %%XTMP2, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 4] - simd_store_avx r14, xmm6, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r14, %%XTMP3, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 6] - simd_store_avx r15, xmm7, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r15, %%XTMP4, r10, %%TMP1, %%TMP2, %%OFFSET ; Restore registers mov %%TMP1, [rsp + _gpr_save] mov %%TMP2, [rsp + _gpr_save + 8] %else - vmovdqu [r12 + %%OFFSET], xmm4 - vmovdqu [r13 + %%OFFSET], xmm5 - vmovdqu [r14 + %%OFFSET], xmm6 - vmovdqu [r15 + %%OFFSET], xmm7 + vmovdqu [r12 + %%OFFSET], %%XTMP1 + vmovdqu [r13 + %%OFFSET], %%XTMP2 + vmovdqu [r14 + %%OFFSET], %%XTMP3 + vmovdqu [r15 + %%OFFSET], %%XTMP4 %endif %endmacro @@ -1211,22 +1244,15 @@ asm_ZucGenKeystream4B_4_avx: MKGLOBAL(asm_ZucCipher_4_avx,function,internal) asm_ZucCipher_4_avx: -%ifdef LINUX - %define pState rdi - %define pIn rsi - %define pOut rdx - %define lengths rcx - %define arg5 r8 +%define pState arg1 +%define pIn arg2 +%define pOut arg3 +%define lengths arg4 - %define nrounds r8 +%ifdef LINUX + %define nrounds r8 %else - %define pState rcx - %define pIn rdx - %define pOut r8 - %define lengths r9 - %define arg5 [rsp + 40] - - %define nrounds rdi + %define nrounds rdi %endif %define min_length r10 @@ -1280,9 +1306,6 @@ asm_ZucCipher_4_avx: vmovq [rsp + _rem_bytes_save], xmm1 mov [rsp + _rsp_save], rax - ; Load state pointer in RAX - mov rax, pState - loop_cipher64: cmp min_length, 64 jl exit_loop_cipher64 @@ -1348,7 +1371,7 @@ _final_rounds_is_1_3: %rep 4 APPEND(_num_final_rounds_is_,I): CIPHERNx4B_4 I, 0, buf_idx, 1 - REORDER_LFSR rax, I + REORDER_LFSR pState, I add buf_idx, (I*4) jmp exit_final_rounds %assign I (I + 1) @@ -1361,7 +1384,7 @@ APPEND(_num_final_rounds_is_,I): add buf_idx, 16 CIPHERNx4B_4 (I-4), 4, buf_idx, 1 add buf_idx, ((I-4)*4) - REORDER_LFSR rax, I + REORDER_LFSR pState, I jmp exit_final_rounds %assign I (I + 1) %endrep @@ -1375,7 +1398,7 @@ APPEND(_num_final_rounds_is_,I): add buf_idx, 16 CIPHERNx4B_4 (I-8), 8, buf_idx, 1 add buf_idx, ((I-8)*4) - REORDER_LFSR rax, I + REORDER_LFSR pState, I jmp exit_final_rounds %assign I (I + 1) %endrep @@ -1391,23 +1414,23 @@ APPEND(_num_final_rounds_is_,I): add buf_idx, 16 CIPHERNx4B_4 (I-12), 12, buf_idx, 1 add buf_idx, ((I-12)*4) - REORDER_LFSR rax, I + REORDER_LFSR pState, I jmp exit_final_rounds %assign I (I + 1) %endrep exit_final_rounds: ;; update in/out pointers - vmovq xmm0, buf_idx - vpshufd xmm0, xmm0, 0x44 - vpaddq xmm1, xmm0, [pIn] - vpaddq xmm2, xmm0, [pIn + 16] - vmovdqa [pIn], xmm1 - vmovdqa [pIn + 16], xmm2 - vpaddq xmm1, xmm0, [pOut] - vpaddq xmm2, xmm0, [pOut + 16] - vmovdqa [pOut], xmm1 - vmovdqa [pOut + 16], xmm2 + vmovq xmm0, buf_idx + vpshufd xmm0, xmm0, 0x44 + vpaddq xmm1, xmm0, [pIn] + vpaddq xmm2, xmm0, [pIn + 16] + vmovdqa [pIn], xmm1 + vmovdqa [pIn + 16], xmm2 + vpaddq xmm1, xmm0, [pOut] + vpaddq xmm2, xmm0, [pOut + 16] + vmovdqa [pOut], xmm1 + vmovdqa [pOut + 16], xmm2 ;; Clear stack frame containing keystream information %ifdef SAFE_DATA diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index 1a2bf816..f76e795b 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -926,9 +926,9 @@ mksection .text FUNC_SAVE ; Zero out R1-R2 - pxor xmm0, xmm0 - movdqa [pState + OFS_R1], xmm0 - movdqa [pState + OFS_R1 + 16], xmm0 + pxor %%XTMP1, %%XTMP1 + movdqa [pState + OFS_R1], %%XTMP1 + movdqa [pState + OFS_R2], %%XTMP1 %if %%KEY_SIZE == 128 ;; Load key and IVs -- GitLab From af5314cdd47f9ef2bd79517743f0b88346e67f1e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 14:30:53 +0000 Subject: [PATCH 007/369] avx: [ZUC] refactor store keystream functions Refactored 4-byte and 16-byte keystream storing functions. Also, no need to store the keystream pointers in the stack. --- lib/avx/zuc_x4_avx.asm | 103 +++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 56 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 8de3358f..958496d6 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -446,49 +446,39 @@ align 64 %endmacro ; -; store16B_kstr4() +; Stores 16 bytes of keystream for 4 lanes ; -%macro store16B_kstr4 4 -%define %%DATA16B_L0 %1 ; [in] 16 bytes of keystream for lane 0 -%define %%DATA16B_L1 %2 ; [in] 16 bytes of keystream for lane 1 -%define %%DATA16B_L2 %3 ; [in] 16 bytes of keystream for lane 2 -%define %%DATA16B_L3 %4 ; [in] 16 bytes of keystream for lane 3 - - mov rcx, [rsp] - mov rdx, [rsp + 8] - mov r8, [rsp + 16] - mov r9, [rsp + 24] - vmovdqu [rcx], %%DATA16B_L0 - vmovdqu [rdx], %%DATA16B_L1 - vmovdqu [r8], %%DATA16B_L2 - vmovdqu [r9], %%DATA16B_L3 +%macro STORE16B_KSTR4 8 +%define %%DATA16B_L0 %1 ; [in] 16 bytes of keystream for lane 0 +%define %%DATA16B_L1 %2 ; [in] 16 bytes of keystream for lane 1 +%define %%DATA16B_L2 %3 ; [in] 16 bytes of keystream for lane 2 +%define %%DATA16B_L3 %4 ; [in] 16 bytes of keystream for lane 3 +%define %%KS_PTR0 %5 ; [in] Pointer to keystream for lane 0 +%define %%KS_PTR1 %6 ; [in] Pointer to keystream for lane 1 +%define %%KS_PTR2 %7 ; [in] Pointer to keystream for lane 2 +%define %%KS_PTR3 %8 ; [in] Pointer to keystream for lane 3 + + vmovdqa [%%KS_PTR0], %%DATA16B_L0 + vmovdqa [%%KS_PTR1], %%DATA16B_L1 + vmovdqa [%%KS_PTR2], %%DATA16B_L2 + vmovdqa [%%KS_PTR3], %%DATA16B_L3 %endmacro ; -; store4B_kstr4() +; Stores 4 bytes of keystream for 4 lanes ; -; params -; -; %1 - XMM register with OFS_X3 -; return -; -%macro store4B_kstr4 1 - mov rcx, [rsp] - mov rdx, [rsp + 8] - mov r8, [rsp + 16] - mov r9, [rsp + 24] - vpextrd [r9], %1, 3 - vpextrd [r8], %1, 2 - vpextrd [rdx], %1, 1 - vmovd [rcx], %1 - add rcx, 4 - add rdx, 4 - add r8, 4 - add r9, 4 - mov [rsp], rcx - mov [rsp + 8], rdx - mov [rsp + 16], r8 - mov [rsp + 24], r9 +%macro STORE4B_KSTR4 6 +%define %%DATA4B_L03 %1 ; [in] 4 bytes of keystream for lanes 0-3 +%define %%KS_PTR0 %2 ; [in] Pointer to keystream for lane 0 +%define %%KS_PTR1 %3 ; [in] Pointer to keystream for lane 1 +%define %%KS_PTR2 %4 ; [in] Pointer to keystream for lane 2 +%define %%KS_PTR3 %5 ; [in] Pointer to keystream for lane 3 +%define %%OFFSET %6 ; [in] Offset into keystream + + vmovd [%%KS_PTR0 + %%OFFSET], %%DATA4B_L03 + vpextrd [%%KS_PTR1 + %%OFFSET], %%DATA4B_L03, 1 + vpextrd [%%KS_PTR2 + %%OFFSET], %%DATA4B_L03, 2 + vpextrd [%%KS_PTR3 + %%OFFSET], %%DATA4B_L03, 3 %endmacro ; @@ -965,17 +955,11 @@ asm_Zuc256Initialization_4_avx: FUNC_SAVE - ; Store 4 keystream pointers on the stack - ; and reserve memory for storing keystreams for all 4 buffers + ; Reserve memory for storing keystreams for all 4 buffers mov r10, rsp - sub rsp, (4*8 + %%NUM_ROUNDS * 16) + sub rsp, (%%NUM_ROUNDS * 16) and rsp, -16 - vmovdqa %%XTMP1, [pKS] - vmovdqa %%XTMP2, [pKS + 16] - vmovdqa [rsp], %%XTMP1 - vmovdqa [rsp + 8*2], %%XTMP2 - ; Load read-only registers vmovdqa %%MASK_31, [rel mask31] @@ -988,29 +972,36 @@ asm_Zuc256Initialization_4_avx: NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W ; OFS_X3 XOR W and store in stack - vpxor %%X3, %%W - vmovdqa [rsp + 4*8 + (%%N-1)*16], %%X3 + vpxor %%X3, %%W + vmovdqa [rsp + (%%N-1)*16], %%X3 LFSR_UPDT4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ %%MASK_31, %%XTMP8, work %assign %%N (%%N + 1) %endrep + ; Read keystream pointers + mov r12, [pKS] + mov r13, [pKS + 8] + mov r14, [pKS + 16] + mov r15, [pKS + 24] %if (%%NUM_ROUNDS == 4) ;; Load all OFS_X3 - vmovdqa %%XTMP1, [rsp + 4*8] - vmovdqa %%XTMP2, [rsp + 4*8 + 16] - vmovdqa %%XTMP3, [rsp + 4*8 + 16*2] - vmovdqa %%XTMP4, [rsp + 4*8 + 16*3] + vmovdqa %%XTMP1, [rsp] + vmovdqa %%XTMP2, [rsp + 16] + vmovdqa %%XTMP3, [rsp + 16*2] + vmovdqa %%XTMP4, [rsp + 16*3] TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 - store16B_kstr4 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4 + STORE16B_KSTR4 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, r12, r13, r14, r15 %else ;; NUM_ROUNDS != 4 %assign %%IDX 1 +%assign %%OFFSET 0 %rep %%NUM_ROUNDS - vmovdqa APPEND(%%XTMP, %%IDX), [rsp + 4*8 + (%%IDX-1)*16] - store4B_kstr4 APPEND(%%XTMP, %%IDX) + vmovdqa APPEND(%%XTMP, %%IDX), [rsp + (%%IDX-1)*16] + STORE4B_KSTR4 APPEND(%%XTMP, %%IDX), r12, r13, r14, r15, %%OFFSET %assign %%IDX (%%IDX + 1) +%assign %%OFFSET (%%OFFSET + 4) %endrep %endif ;; NUM_ROUNDS == 4 @@ -1018,7 +1009,7 @@ asm_Zuc256Initialization_4_avx: %ifdef SAFE_DATA vpxor %%XTMP1, %%XTMP1 %assign %%I 0 -%rep (2 + %%NUM_ROUNDS) +%rep %%NUM_ROUNDS vmovdqa [rsp + %%I*16], %%XTMP1 %assign %%I (%%I + 1) %endrep -- GitLab From 96e0bd07f2f8718dfbdff9af6e94a714fcfb86cb Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 18:48:49 +0000 Subject: [PATCH 008/369] avx: [ZUC] do not store keystream in stack Now that there are enough free XMM registers, keystream does not have to be stored in stack. --- lib/avx/zuc_x4_avx.asm | 110 ++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 74 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 958496d6..45156fca 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -202,7 +202,6 @@ db 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff ; Stack frame for ZucCipher function struc STACK -_keystr_save resq 2*4 ; Space for 4 keystreams _rsp_save: resq 1 ; Space for rsp pointer _gpr_save: resq 2 ; Space for GP registers _rem_bytes_save resq 1 ; Space for number of remaining bytes @@ -949,17 +948,15 @@ asm_Zuc256Initialization_4_avx: %define %%XTMP15 xmm14 %define %%XTMP16 xmm15 -%define %%W %%XTMP10 -%define %%X3 %%XTMP11 +%define %%W %%XTMP11 +%define %%KSTR1 %%XTMP12 +%define %%KSTR2 %%XTMP13 +%define %%KSTR3 %%XTMP14 +%define %%KSTR4 %%XTMP15 %define %%MASK_31 %%XTMP16 FUNC_SAVE - ; Reserve memory for storing keystreams for all 4 buffers - mov r10, rsp - sub rsp, (%%NUM_ROUNDS * 16) - and rsp, -16 - ; Load read-only registers vmovdqa %%MASK_31, [rel mask31] @@ -968,60 +965,38 @@ asm_Zuc256Initialization_4_avx: %assign %%N 1 %rep %%NUM_ROUNDS BITS_REORG4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ - %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, %%X3 + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, APPEND(%%KSTR, %%N) NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W ; OFS_X3 XOR W and store in stack - vpxor %%X3, %%W - vmovdqa [rsp + (%%N-1)*16], %%X3 + vpxor APPEND(%%KSTR, %%N), %%W LFSR_UPDT4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ %%MASK_31, %%XTMP8, work %assign %%N (%%N + 1) %endrep - ; Read keystream pointers + ; Read keystream pointers and store the keystreams mov r12, [pKS] mov r13, [pKS + 8] mov r14, [pKS + 16] mov r15, [pKS + 24] %if (%%NUM_ROUNDS == 4) - ;; Load all OFS_X3 - vmovdqa %%XTMP1, [rsp] - vmovdqa %%XTMP2, [rsp + 16] - vmovdqa %%XTMP3, [rsp + 16*2] - vmovdqa %%XTMP4, [rsp + 16*3] - - TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 - - STORE16B_KSTR4 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, r12, r13, r14, r15 + TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 + STORE16B_KSTR4 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, r12, r13, r14, r15 %else ;; NUM_ROUNDS != 4 %assign %%IDX 1 %assign %%OFFSET 0 %rep %%NUM_ROUNDS - vmovdqa APPEND(%%XTMP, %%IDX), [rsp + (%%IDX-1)*16] - STORE4B_KSTR4 APPEND(%%XTMP, %%IDX), r12, r13, r14, r15, %%OFFSET + STORE4B_KSTR4 APPEND(%%KSTR, %%IDX), r12, r13, r14, r15, %%OFFSET %assign %%IDX (%%IDX + 1) %assign %%OFFSET (%%OFFSET + 4) %endrep %endif ;; NUM_ROUNDS == 4 - ;; Clear stack frame containing keystream information -%ifdef SAFE_DATA - vpxor %%XTMP1, %%XTMP1 -%assign %%I 0 -%rep %%NUM_ROUNDS - vmovdqa [rsp + %%I*16], %%XTMP1 -%assign %%I (%%I + 1) -%endrep -%endif - ;; Reorder memory for LFSR registers, as not all 16 rounds - ;; will be completed (can be 4 or 2) + ;; will be completed REORDER_LFSR pState, %%NUM_ROUNDS - ;; Restore rsp pointer to value before pushing keystreams - mov rsp, r10 - FUNC_RESTORE %endmacro @@ -1119,7 +1094,10 @@ asm_ZucGenKeystream4B_4_avx: %define %%XTMP16 xmm15 %define %%W %%XTMP10 -%define %%X3 %%XTMP11 +%define %%KSTR1 %%XTMP12 +%define %%KSTR2 %%XTMP13 +%define %%KSTR3 %%XTMP14 +%define %%KSTR4 %%XTMP15 %define %%MASK_31 %%XTMP16 ; Load read-only registers @@ -1130,27 +1108,20 @@ asm_ZucGenKeystream4B_4_avx: %assign %%round (%%INITIAL_ROUND + %%N) %rep %%NROUNDS BITS_REORG4 pState, %%round, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ - %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, %%X3 + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, APPEND(%%KSTR, %%N) NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W ; OFS_X3 XOR W and store in stack - vpxor %%X3, %%W - vmovdqa [rsp + _keystr_save + (%%N-1)*16], %%X3 + vpxor APPEND(%%KSTR, %%N), %%W LFSR_UPDT4 pState, %%round, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ %%MASK_31, %%XTMP8, work %assign %%N (%%N + 1) %assign %%round (%%round + 1) %endrep -%assign %%N 1 -%rep %%NROUNDS - vmovdqa APPEND(%%XTMP, %%N), [rsp + _keystr_save + (%%N-1)*16] -%assign %%N (%%N + 1) -%endrep - - TRANSPOSE4_U32 %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 - vmovdqa %%XTMP15, [rel swap_mask] + vmovdqa %%XTMP16, [rel swap_mask] ;; XOR Input buffer with keystream in rounds of 16B mov r12, [pIn] @@ -1178,15 +1149,15 @@ asm_ZucGenKeystream4B_4_avx: vmovdqu %%XTMP8, [r15 + %%OFFSET] %endif - vpshufb %%XTMP1, %%XTMP15 - vpshufb %%XTMP2, %%XTMP15 - vpshufb %%XTMP3, %%XTMP15 - vpshufb %%XTMP4, %%XTMP15 + vpshufb %%KSTR1, %%XTMP16 + vpshufb %%KSTR2, %%XTMP16 + vpshufb %%KSTR3, %%XTMP16 + vpshufb %%KSTR4, %%XTMP16 - vpxor %%XTMP1, %%XTMP5 - vpxor %%XTMP2, %%XTMP6 - vpxor %%XTMP3, %%XTMP7 - vpxor %%XTMP4, %%XTMP8 + vpxor %%KSTR1, %%XTMP5 + vpxor %%KSTR2, %%XTMP6 + vpxor %%KSTR3, %%XTMP7 + vpxor %%KSTR4, %%XTMP8 mov r12, [pOut] mov r13, [pOut + 8] @@ -1195,22 +1166,22 @@ asm_ZucGenKeystream4B_4_avx: %if (%%LAST_CALL == 1) movzx r10d, word [rsp + _rem_bytes_save] - simd_store_avx r12, %%XTMP1, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r12, %%KSTR1, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 2] - simd_store_avx r13, %%XTMP2, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r13, %%KSTR2, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 4] - simd_store_avx r14, %%XTMP3, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r14, %%KSTR3, r10, %%TMP1, %%TMP2, %%OFFSET movzx r10d, word [rsp + _rem_bytes_save + 6] - simd_store_avx r15, %%XTMP4, r10, %%TMP1, %%TMP2, %%OFFSET + simd_store_avx r15, %%KSTR4, r10, %%TMP1, %%TMP2, %%OFFSET ; Restore registers mov %%TMP1, [rsp + _gpr_save] mov %%TMP2, [rsp + _gpr_save + 8] %else - vmovdqu [r12 + %%OFFSET], %%XTMP1 - vmovdqu [r13 + %%OFFSET], %%XTMP2 - vmovdqu [r14 + %%OFFSET], %%XTMP3 - vmovdqu [r15 + %%OFFSET], %%XTMP4 + vmovdqu [r12 + %%OFFSET], %%KSTR1 + vmovdqu [r13 + %%OFFSET], %%KSTR2 + vmovdqu [r14 + %%OFFSET], %%KSTR3 + vmovdqu [r15 + %%OFFSET], %%KSTR4 %endif %endmacro @@ -1423,15 +1394,6 @@ exit_final_rounds: vmovdqa [pOut], xmm1 vmovdqa [pOut + 16], xmm2 - ;; Clear stack frame containing keystream information -%ifdef SAFE_DATA - vpxor xmm0, xmm0 -%assign i 0 -%rep 4 - vmovdqa [rsp + _keystr_save + i*16], xmm0 -%assign i (i+1) -%endrep -%endif ; Restore rsp mov rsp, [rsp + _rsp_save] -- GitLab From b8836a3ef97e9660b705a8007bbc8340240810e3 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 25 Feb 2022 18:53:36 +0000 Subject: [PATCH 009/369] avx: [ZUC] remove unused variables --- lib/avx/zuc_top_avx.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index fb29571e..8333d806 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -639,9 +639,6 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], /* Process each packet separately for the remaining bits */ for (i = 0; i < NUM_AVX_BUFS; i++) { - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; @@ -673,7 +670,6 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 16 bytes */ if (!remainBits) @@ -838,9 +834,6 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], if (job_in_lane[i] == NULL) continue; - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; @@ -872,7 +865,6 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 16 bytes */ if (!remainBits) -- GitLab From aa0cf18bdc7a38f265e232674c8dc1d63ba6fec9 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Mon, 7 Mar 2022 14:15:48 +0000 Subject: [PATCH 010/369] CI: Add clang to github CI --- .github/workflows/clang_aesni_make.yml | 20 ++++++++++++++++++++ .github/workflows/clang_make.yml | 20 ++++++++++++++++++++ .github/workflows/clang_static_make.yml | 20 ++++++++++++++++++++ .github/workflows/linux_static_make.yml | 20 ++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 .github/workflows/clang_aesni_make.yml create mode 100644 .github/workflows/clang_make.yml create mode 100644 .github/workflows/clang_static_make.yml create mode 100644 .github/workflows/linux_static_make.yml diff --git a/.github/workflows/clang_aesni_make.yml b/.github/workflows/clang_aesni_make.yml new file mode 100644 index 00000000..de5292e3 --- /dev/null +++ b/.github/workflows/clang_aesni_make.yml @@ -0,0 +1,20 @@ +name: Clang aesni make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - run: sudo apt install -y nasm clang + - uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: make aesni + run: make CC=clang AESNI_EMU=y -j diff --git a/.github/workflows/clang_make.yml b/.github/workflows/clang_make.yml new file mode 100644 index 00000000..afa58dbc --- /dev/null +++ b/.github/workflows/clang_make.yml @@ -0,0 +1,20 @@ +name: Clang make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - run: sudo apt install -y nasm clang + - uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: make + run: make CC=clang -j diff --git a/.github/workflows/clang_static_make.yml b/.github/workflows/clang_static_make.yml new file mode 100644 index 00000000..78c27115 --- /dev/null +++ b/.github/workflows/clang_static_make.yml @@ -0,0 +1,20 @@ +name: Clang static make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - run: sudo apt install -y nasm clang + - uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: make static + run: make CC=clang SHARED=n -j diff --git a/.github/workflows/linux_static_make.yml b/.github/workflows/linux_static_make.yml new file mode 100644 index 00000000..7655ab15 --- /dev/null +++ b/.github/workflows/linux_static_make.yml @@ -0,0 +1,20 @@ +name: Linux static make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - run: sudo apt install -y nasm + - uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: make static + run: make SHARED=n -j -- GitLab From a7b55cb5fc290910ae5fb84ab6d2b27505d14bb3 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 1 Mar 2022 14:36:45 +0000 Subject: [PATCH 011/369] avx2: [ZUC] remove unused macro and data --- lib/avx2/zuc_top_avx2.c | 10 ---------- lib/avx2/zuc_x8_avx2.asm | 19 ------------------- 2 files changed, 29 deletions(-) diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index e11c41ff..5276d199 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -488,16 +488,6 @@ void zuc_eea3_n_buffer_avx2(const void * const pKey[], const void * const pIv[], #endif } -static inline uint64_t rotate_left(uint64_t u, size_t r) -{ - return (((u) << (r)) | ((u) >> (64 - (r)))); -} - -static inline uint64_t load_uint64(const void *ptr) -{ - return *((const uint64_t *)ptr); -} - static inline void _zuc_eia3_1_buffer_avx2(const void *pKey, const void *pIv, diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index a9598f01..777d4bdb 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -150,10 +150,6 @@ broadcast_word: db 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01 db 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01 -align 16 -all_ffs: -dw 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff - align 16 all_threes: dw 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003, 0x0003 @@ -272,21 +268,6 @@ align 64 %endmacro -;; -;; make_u31() -;; -%macro make_u31 4 - -%define %%Rt %1 -%define %%Ke %2 -%define %%Ek %3 -%define %%Iv %4 - xor %%Rt, %%Rt - shrd %%Rt, %%Iv, 8 - shrd %%Rt, %%Ek, 15 - shrd %%Rt, %%Ke, 9 -%endmacro - ; ; bits_reorg8() ; -- GitLab From c388c4cdbb67516a99dd8e930af0ada7a172728e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 1 Mar 2022 15:44:05 +0000 Subject: [PATCH 012/369] avx2: [ZUC] refactor bits_reorg macro Remove hard-coded registers from bits reorganization macro, and passing the register through parameters instead. --- lib/avx2/zuc_x8_avx2.asm | 99 +++++++++++++++++++++------------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 777d4bdb..e53d41c7 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -269,52 +269,53 @@ align 64 %endmacro ; -; bits_reorg8() +; Calculates X0-X3 from LFSR registers ; -%macro bits_reorg8 2-3 -%define %%STATE %1 ; [in] ZUC state -%define %%ROUND_NUM %2 ; [in] Round number -%define %%X3 %3 ; [out] YMM register containing X3 of all lanes - ; - ; ymm15 = LFSR_S15 - ; ymm14 = LFSR_S14 - ; ymm11 = LFSR_S11 - ; ymm9 = LFSR_S9 - ; ymm7 = LFSR_S7 - ; ymm5 = LFSR_S5 - ; ymm2 = LFSR_S2 - ; ymm0 = LFSR_S0 - ; - vmovdqa ymm15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm14, [%%STATE + ((14 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm11, [%%STATE + ((11 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm9, [%%STATE + (( 9 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm7, [%%STATE + (( 7 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm5, [%%STATE + (( 5 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm2, [%%STATE + (( 2 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] - - vpxor ymm1, ymm1 - vpslld ymm15, 1 - vpblendw ymm3, ymm14, ymm1, 0xAA - vpblendw ymm15, ymm3, ymm15, 0xAA - - vmovdqa [%%STATE + OFS_X0], ymm15 ; BRC_X0 - vpslld ymm11, 16 - vpsrld ymm9, 15 - vpor ymm11, ymm9 - vmovdqa [%%STATE + OFS_X1], ymm11 ; BRC_X1 - vpslld ymm7, 16 - vpsrld ymm5, 15 - vpor ymm7, ymm5 - vmovdqa [%%STATE + OFS_X2], ymm7 ; BRC_X2 -%if (%0 == 3) - vpslld ymm2, 16 - vpsrld ymm0, 15 - vpor %%X3, ymm2, ymm0 ; Store BRC_X3 in YMM register +%macro BITS_REORG8 12-13 +%define %%STATE %1 ; [in] ZUC state +%define %%ROUND_NUM %2 ; [in] Round number +%define %%LFSR_0 %3 ; [clobbered] LFSR_0 +%define %%LFSR_2 %4 ; [clobbered] LFSR_2 +%define %%LFSR_5 %5 ; [clobbered] LFSR_5 +%define %%LFSR_7 %6 ; [clobbered] LFSR_7 +%define %%LFSR_9 %7 ; [clobbered] LFSR_9 +%define %%LFSR_11 %8 ; [clobbered] LFSR_11 +%define %%LFSR_14 %9 ; [clobbered] LFSR_14 +%define %%LFSR_15 %10 ; [clobbered] LFSR_15 +%define %%YTMP1 %11 ; [clobbered] Temporary YMM register +%define %%YTMP2 %12 ; [clobbered] Temporary YMM register +%define %%X3 %13 ; [out] YMM register containing X3 of all lanes (only for work mode) + vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_14, [%%STATE + ((14 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_11, [%%STATE + ((11 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_9, [%%STATE + (( 9 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_7, [%%STATE + (( 7 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_5, [%%STATE + (( 5 + %%ROUND_NUM) % 16)*32] +%if (%0 == 13) ;Only needed when generating X3 (for "working" mode) + vmovdqa %%LFSR_2, [%%STATE + (( 2 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] %endif -%endmacro + vpxor %%YTMP1, %%YTMP1 + vpslld %%LFSR_15, 1 + vpblendw %%YTMP2, %%LFSR_14, %%YTMP1, 0xAA + vpblendw %%LFSR_15, %%LFSR_15, %%YTMP2, 0x55 + + vmovdqa [%%STATE + OFS_X0], %%LFSR_15 ; BRC_X0 + vpslld %%LFSR_11, 16 + vpsrld %%LFSR_9, 15 + vpor %%LFSR_11, %%LFSR_9 + vmovdqa [%%STATE + OFS_X1], %%LFSR_11 ; BRC_X1 + vpslld %%LFSR_7, 16 + vpsrld %%LFSR_5, 15 + vpor %%LFSR_7, %%LFSR_5 + vmovdqa [%%STATE + OFS_X2], %%LFSR_7 ; BRC_X2 +%if (%0 == 13) + vpslld %%LFSR_2, 16 + vpsrld %%LFSR_0, 15 + vpor %%X3, %%LFSR_2, %%LFSR_0 +%endif +%endmacro ; ; rot_mod32() ; @@ -843,7 +844,8 @@ align 64 ; Shift LFSR 32-times, update state variables %assign N 0 %rep 32 - bits_reorg8 rax, N + BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + ymm6, ymm7, ymm8, ymm9 nonlin_fun8 rax, ymm0 vpsrld ymm0,1 ; Shift out LSB of W lfsr_updt8 rax, N, ymm0 ; W (ymm0) used in LFSR update - not set to zero @@ -851,7 +853,8 @@ align 64 %endrep ; And once more, initial round from keygen phase = 33 times - bits_reorg8 rax, 0 + BITS_REORG8 rax, 0, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + ymm6, ymm7, ymm8, ymm9 nonlin_fun8 rax vpxor ymm0, ymm0 @@ -911,7 +914,8 @@ asm_Zuc256Initialization_8_avx2: ; Generate N*4B of keystream in N rounds %assign N 1 %rep %%NUM_ROUNDS - bits_reorg8 rax, N, ymm10 + BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + ymm6, ymm7, ymm8, ymm9, ymm10 nonlin_fun8 rax, ymm0 ; OFS_X3 XOR W (ymm0) and store in stack vpxor ymm10, ymm0 @@ -1054,7 +1058,8 @@ asm_ZucGenKeystream4B_8_avx2: %assign N 1 %assign round (%%INITIAL_ROUND + N) %rep %%NROUNDS - bits_reorg8 rax, round, ymm10 + BITS_REORG8 rax, round, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + ymm6, ymm7, ymm8, ymm9, ymm10 nonlin_fun8 rax, ymm0 ; OFS_XR XOR W (ymm0) vpxor ymm10, ymm0 -- GitLab From 25f067872c063385252c2be8fe10104b92142e13 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 1 Mar 2022 16:06:08 +0000 Subject: [PATCH 013/369] avx2: [ZUC] refactor nonlin macro Remove hard-coded registers from bits reorganization macro, and passing the registers through parameters instead. --- lib/avx2/zuc_x8_avx2.asm | 193 +++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 98 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index e53d41c7..6cd72460 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -104,21 +104,11 @@ db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c -align 32 -S1_S0_shuf: -db 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F -db 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F - align 32 S0_S1_shuf: db 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, db 0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, -align 32 -rev_S1_S0_shuf: -db 0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B, 0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F -db 0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B, 0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F - align 32 rev_S0_S1_shuf: db 0x08, 0x00, 0x09, 0x01, 0x0A, 0x02, 0x0B, 0x03, 0x0C, 0x04, 0x0D, 0x05, 0x0E, 0x06, 0x0F, 0x07 @@ -316,95 +306,102 @@ align 64 vpor %%X3, %%LFSR_2, %%LFSR_0 %endif %endmacro + ; -; rot_mod32() -; -; uses ymm7 +; Rotate dwords by N_BITS ; -%macro rot_mod32 3 -%if (%3 == 8) - vpshufb %1, %2, [rel rot8_mod32] -%elif (%3 == 16) - vpshufb %1, %2, [rel rot16_mod32] -%elif (%3 == 24) - vpshufb %1, %2, [rel rot24_mod32] +%macro ROT_MOD32 4 +%define %%OUT %1 ; [out] YMM register +%define %%IN %2 ; [in] YMM register +%define %%YTMP %3 ; [clobbered] YMM register +%define %%N_BITS %4 ; [constant] Number of bits + +%if (%%N_BITS == 8) + vpshufb %%OUT, %%IN, [rel rot8_mod32] +%elif (%%N_BITS == 16) + vpshufb %%OUT, %%IN, [rel rot16_mod32] +%elif (%%N_BITS == 24) + vpshufb %%OUT, %%IN, [rel rot24_mod32] %else - vpslld %1, %2, %3 - vpsrld ymm7, %2, (32 - %3) - - vpor %1, ymm7 + vpslld %%OUT, %%IN, %%N_BITS + vpsrld %%YTMP, %%IN, (32 - %%N_BITS) + vpor %%OUT, %%YTMP %endif %endmacro ; -; nonlin_fun8() +; Updates R1-R2, using X0-X3 and generates W (if needed) ; -; return -; W value, updates F_R1[] / F_R2[] -; -%macro nonlin_fun8 1-2 +%macro NONLIN_FUN8 8-9 %define %%STATE %1 ; [in] ZUC state -%define %%W %2 ; [out] YMM register to contain W for all lanes - -%if (%0 == 2) - vmovdqa %%W, [%%STATE + OFS_X0] - vpxor %%W, [%%STATE + OFS_R1] - vpaddd %%W, [%%STATE + OFS_R2] ; W = (BRC_X0 ^ F_R1) + F_R2 +%define %%YTMP1 %2 ; [clobbered] Temporary YMM register +%define %%YTMP2 %3 ; [clobbered] Temporary YMM register +%define %%YTMP3 %4 ; [clobbered] Temporary YMM register +%define %%YTMP4 %5 ; [clobbered] Temporary YMM register +%define %%YTMP5 %6 ; [clobbered] Temporary YMM register +%define %%YTMP6 %7 ; [clobbered] Temporary YMM register +%define %%YTMP7 %8 ; [clobbered] Temporary YMM register +%define %%W %9 ; [out] ZMM register to contain W for all lanes + +%if (%0 == 9) + vmovdqa %%W, [%%STATE + OFS_X0] + vpxor %%W, [%%STATE + OFS_R1] + vpaddd %%W, [%%STATE + OFS_R2] ; W = (BRC_X0 ^ F_R1) + F_R2 %endif - vmovdqa ymm1, [%%STATE + OFS_R1] - vmovdqa ymm2, [%%STATE + OFS_R2] - vpaddd ymm1, [%%STATE + OFS_X1] ; W1 = F_R1 + BRC_X1 - vpxor ymm2, [%%STATE + OFS_X2] ; W2 = F_R2 ^ BRC_X2 - - vpslld ymm3, ymm1, 16 - vpsrld ymm4, ymm1, 16 - vpslld ymm5, ymm2, 16 - vpsrld ymm6, ymm2, 16 - vpor ymm1, ymm3, ymm6 - vpor ymm2, ymm4, ymm5 - - rot_mod32 ymm3, ymm1, 2 - rot_mod32 ymm4, ymm1, 10 - rot_mod32 ymm5, ymm1, 18 - rot_mod32 ymm6, ymm1, 24 - vpxor ymm1, ymm3 - vpxor ymm1, ymm4 - vpxor ymm1, ymm5 - vpxor ymm1, ymm6 ; XMM1 = U = L1(P) - - rot_mod32 ymm3, ymm2, 8 - rot_mod32 ymm4, ymm2, 14 - rot_mod32 ymm5, ymm2, 22 - rot_mod32 ymm6, ymm2, 30 - vpxor ymm2, ymm3 - vpxor ymm2, ymm4 - vpxor ymm2, ymm5 - vpxor ymm2, ymm6 ; XMM2 = V = L2(Q) - - ; Shuffle U and V to have all S0 lookups in XMM1 and all S1 lookups in XMM2 - - ; Compress all S0 and S1 input values in each register - vpshufb ymm1, [rel S0_S1_shuf] ; S0: Bytes 0-7,16-23 S1: Bytes 8-15,24-31 - vpshufb ymm2, [rel S1_S0_shuf] ; S1: Bytes 0-7,16-23 S0: Bytes 8-15,24-31 - - vshufpd ymm3, ymm1, ymm2, 0xA ; All S0 input values - vshufpd ymm4, ymm2, ymm1, 0xA ; All S1 input values - - ; Compute S0 and S1 values - S0_comput_AVX2 ymm3, ymm1, ymm2 - S1_comput_AVX2 ymm4, ymm1, ymm2, ymm5 - - ; Need to shuffle back ymm1 & ymm2 before storing output - ; (revert what was done before S0 and S1 computations) - vshufpd ymm1, ymm3, ymm4, 0xA - vshufpd ymm2, ymm4, ymm3, 0xA - - vpshufb ymm1, [rel rev_S0_S1_shuf] - vpshufb ymm2, [rel rev_S1_S0_shuf] - - vmovdqa [%%STATE + OFS_R1], ymm1 - vmovdqa [%%STATE + OFS_R2], ymm2 + vmovdqa %%YTMP1, [%%STATE + OFS_R1] + vmovdqa %%YTMP2, [%%STATE + OFS_R2] + vpaddd %%YTMP1, [%%STATE + OFS_X1] ; W1 = F_R1 + BRC_X1 + vpxor %%YTMP2, [%%STATE + OFS_X2] ; W2 = F_R2 ^ BRC_X2 + + vpslld %%YTMP3, %%YTMP1, 16 + vpsrld %%YTMP4, %%YTMP1, 16 + vpslld %%YTMP5, %%YTMP2, 16 + vpsrld %%YTMP6, %%YTMP2, 16 + vpor %%YTMP1, %%YTMP3, %%YTMP6 + vpor %%YTMP2, %%YTMP4, %%YTMP5 + + ROT_MOD32 %%YTMP3, %%YTMP1, %%YTMP7, 2 + ROT_MOD32 %%YTMP4, %%YTMP1, %%YTMP7, 10 + ROT_MOD32 %%YTMP5, %%YTMP1, %%YTMP7, 18 + ROT_MOD32 %%YTMP6, %%YTMP1, %%YTMP7, 24 + vpxor %%YTMP1, %%YTMP3 + vpxor %%YTMP1, %%YTMP4 + vpxor %%YTMP1, %%YTMP5 + vpxor %%YTMP1, %%YTMP6 ; XMM1 = U = L1(P) + + ROT_MOD32 %%YTMP3, %%YTMP2, %%YTMP7, 8 + ROT_MOD32 %%YTMP4, %%YTMP2, %%YTMP7, 14 + ROT_MOD32 %%YTMP5, %%YTMP2, %%YTMP7, 22 + ROT_MOD32 %%YTMP6, %%YTMP2, %%YTMP7, 30 + vpxor %%YTMP2, %%YTMP3 + vpxor %%YTMP2, %%YTMP4 + vpxor %%YTMP2, %%YTMP5 + vpxor %%YTMP2, %%YTMP6 ; XMM2 = V = L2(Q) + + ; Shuffle U and V to have all S0 lookups in %%YTMP1 and all S1 lookups in %%YTMP2 + + ; Compress all S0 and S1 input values in each register + vpshufb %%YTMP1, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 + vpshufb %%YTMP2, [rel S0_S1_shuf] ; S0: Bytes 0-7, S1: Bytes 8-15 + + vshufpd %%YTMP3, %%YTMP1, %%YTMP2, 0x00 ; All S0 input values + vshufpd %%YTMP4, %%YTMP2, %%YTMP1, 0xFF ; All S1 input values + + ; Compute S0 and S1 values + S0_comput_AVX2 %%YTMP3, %%YTMP1, %%YTMP2 + S1_comput_AVX2 %%YTMP4, %%YTMP1, %%YTMP2, %%YTMP5 + + ; Need to shuffle back %%YTMP1 & %%YTMP2 before storing output + ; (revert what was done before S0 and S1 computations) + vshufpd %%YTMP1, %%YTMP3, %%YTMP4, 0xAA + vshufpd %%YTMP2, %%YTMP3, %%YTMP4, 0x55 + + vpshufb %%YTMP1, [rel rev_S0_S1_shuf] + vpshufb %%YTMP2, [rel rev_S0_S1_shuf] + + vmovdqa [%%STATE + OFS_R1], %%YTMP1 + vmovdqa [%%STATE + OFS_R2], %%YTMP2 %endmacro ; @@ -846,16 +843,16 @@ align 64 %rep 32 BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ ymm6, ymm7, ymm8, ymm9 - nonlin_fun8 rax, ymm0 - vpsrld ymm0,1 ; Shift out LSB of W - lfsr_updt8 rax, N, ymm0 ; W (ymm0) used in LFSR update - not set to zero + NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 + vpsrld ymm7, 1 ; Shift out LSB of W + lfsr_updt8 rax, N, ymm7 ; W (ymm7) used in LFSR update %assign N N+1 %endrep ; And once more, initial round from keygen phase = 33 times BITS_REORG8 rax, 0, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ ymm6, ymm7, ymm8, ymm9 - nonlin_fun8 rax + NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 vpxor ymm0, ymm0 lfsr_updt8 rax, 0, ymm0 @@ -916,9 +913,9 @@ asm_Zuc256Initialization_8_avx2: %rep %%NUM_ROUNDS BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ ymm6, ymm7, ymm8, ymm9, ymm10 - nonlin_fun8 rax, ymm0 - ; OFS_X3 XOR W (ymm0) and store in stack - vpxor ymm10, ymm0 + NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 + ; OFS_X3 XOR W (ymm7) and store in stack + vpxor ymm10, ymm7 vmovdqa [rsp + 64 + (N-1)*32], ymm10 vpxor ymm0, ymm0 lfsr_updt8 rax, N, ymm0 @@ -1060,9 +1057,9 @@ asm_ZucGenKeystream4B_8_avx2: %rep %%NROUNDS BITS_REORG8 rax, round, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ ymm6, ymm7, ymm8, ymm9, ymm10 - nonlin_fun8 rax, ymm0 - ; OFS_XR XOR W (ymm0) - vpxor ymm10, ymm0 + NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 + ; OFS_X3 XOR W (ymm7) + vpxor ymm10, ymm7 vmovdqa [rsp + (N-1)*32], ymm10 vpxor ymm0, ymm0 lfsr_updt8 rax, round, ymm0 -- GitLab From 828df9f6842dc3ecd3fe32ad658a4cc8172f4ff7 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 1 Mar 2022 18:15:58 +0000 Subject: [PATCH 014/369] avx2: [ZUC] refactor lfsr_updt macro Remove hard-coded registers from LFSR update function macro, and passing the registers through parameters instead. --- lib/avx2/zuc_x8_avx2.asm | 158 +++++++++++++++++++++------------------ 1 file changed, 84 insertions(+), 74 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 6cd72460..3a5ffe31 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -159,8 +159,6 @@ dw 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020, 0x0020 mksection .text align 64 -%define MASK31 ymm12 - %define OFS_R1 (16*(2*16)) %define OFS_R2 (OFS_R1 + (2*16)) %define OFS_X0 (OFS_R2 + (2*16)) @@ -484,78 +482,82 @@ align 64 %endmacro ; -; add_mod31() -; add two 32-bit args and reduce mod (2^31-1) -; params -; %1 - arg1/res -; %2 - arg2 -; uses -; ymm2 -; return -; %1 -%macro add_mod31 2 - vpaddd %1, %2 - vpsrld ymm2, %1, 31 - vpand %1, MASK31 - vpaddd %1, ymm2 +; Add two 32-bit args and reduce mod (2^31-1) +; +%macro ADD_MOD31 4 +%define %%IN_OUT %1 ; [in/out] YMM register with first input and output +%define %%IN2 %2 ; [in] YMM register with second input +%define %%YTMP %3 ; [clobbered] Temporary YMM register +%define %%MASK31 %4 ; [in] YMM register containing 0x7FFFFFFF's in all dwords + vpaddd %%IN_OUT, %%IN2 + vpsrld %%YTMP, %%IN_OUT, 31 + vpand %%IN_OUT, %%MASK31 + vpaddd %%IN_OUT, %%YTMP %endmacro ; -; rot_mod31() -; rotate (mult by pow of 2) 32-bit arg and reduce mod (2^31-1) -; params -; %1 - arg -; %2 - # of bits -; uses -; ymm2 -; return -; %1 -%macro rot_mod31 2 +; Rotate (mult by pow of 2) 32-bit arg and reduce mod (2^31-1) +; +%macro ROT_MOD31 4 +%define %%IN_OUT %1 ; [in/out] YMM register with input and output +%define %%YTMP %2 ; [clobbered] Temporary YMM register +%define %%MASK31 %3 ; [in] YMM register containing 0x7FFFFFFF's in all dwords +%define %%N_BITS %4 ; [immediate] Number of bits to rotate for each dword - vpslld ymm2, %1, %2 - vpsrld %1, %1, (31 - %2) + vpslld %%YTMP, %%IN_OUT, %%N_BITS + vpsrld %%IN_OUT, (31 - %%N_BITS) - vpor %1, ymm2 - vpand %1, MASK31 + vpor %%IN_OUT, %%YTMP + vpand %%IN_OUT, %%MASK31 %endmacro ; -; lfsr_updt8() +; Update LFSR registers, calculating S_16 ; +; S_16 = [ 2^15*S_15 + 2^17*S_13 + 2^21*S_10 + 2^20*S_4 + (1 + 2^8)*S_0 ] mod (2^31 - 1) +; If init mode, add W to the calculation above. +; S_16 -> S_15 for next round ; -%macro lfsr_updt8 3 -%define %%STATE %1 ; [in] ZUC state -%define %%ROUND_NUM %2 ; [in] Round number -%define %%W %3 ; [in/clobbered] YMM register to contain W for all lanes - ; - ; ymm1 = LFSR_S0 - ; ymm4 = LFSR_S4 - ; ymm10 = LFSR_S10 - ; ymm13 = LFSR_S13 - ; ymm15 = LFSR_S15 - ; - vmovdqa ymm1, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*32] - vmovdqa ymm15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] - - ; Calculate LFSR feedback - add_mod31 %%W, ymm1 - rot_mod31 ymm1, 8 - add_mod31 %%W, ymm1 - rot_mod31 ymm4, 20 - add_mod31 %%W, ymm4 - rot_mod31 ymm10, 21 - add_mod31 %%W, ymm10 - rot_mod31 ymm13, 17 - add_mod31 %%W, ymm13 - rot_mod31 ymm15, 15 - add_mod31 %%W, ymm15 - - vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32], %%W - - ; LFSR_S16 = (LFSR_S15++) = eax +%macro LFSR_UPDT8 11 +%define %%STATE %1 ; [in] ZUC state +%define %%ROUND_NUM %2 ; [in] Round number +%define %%LFSR_0 %3 ; [clobbered] LFSR_0 (YMM) +%define %%LFSR_4 %4 ; [clobbered] LFSR_4 (YMM) +%define %%LFSR_10 %5 ; [clobbered] LFSR_10 (YMM) +%define %%LFSR_13 %6 ; [clobbered] LFSR_13 (YMM) +%define %%LFSR_15 %7 ; [clobbered] LFSR_15 (YMM) +%define %%YTMP %8 ; [clobbered] Temporary YMM register +%define %%MASK_31 %9 ; [in] Mask_31 +%define %%W %10 ; [in/clobbered] In init mode, contains W for all 4 lanes +%define %%MODE %11 ; [constant] "init" / "work" mode + + vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] + + ; Calculate LFSR feedback (S_16) + + ; In Init mode, W is added to the S_16 calculation +%ifidn %%MODE, init + ADD_MOD31 %%W, %%LFSR_0, %%YTMP, %%MASK_31 +%else + vmovdqa %%W, %%LFSR_0 +%endif + ROT_MOD31 %%LFSR_0, %%YTMP, %%MASK_31, 8 + ADD_MOD31 %%W, %%LFSR_0, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_4, %%YTMP, %%MASK_31, 20 + ADD_MOD31 %%W, %%LFSR_4, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_10, %%YTMP, %%MASK_31, 21 + ADD_MOD31 %%W, %%LFSR_10, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_13, %%YTMP, %%MASK_31, 17 + ADD_MOD31 %%W, %%LFSR_13, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_15, %%YTMP, %%MASK_31, 15 + ADD_MOD31 %%W, %%LFSR_15, %%YTMP, %%MASK_31 + + ; Store LFSR_S16 + vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32], %%W %endmacro ; @@ -740,6 +742,8 @@ align 64 %define tag_sz r9 ; Only used in ZUC-256 %endif +%define %%MASK_31 ymm15 + FUNC_SAVE ; Zero out R1/R2 (only lower half is used) @@ -834,7 +838,7 @@ align 64 %endif ;; %%KEY_SIZE == 256 ; Load read-only registers - vmovdqa ymm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] mov rax, pState @@ -845,7 +849,8 @@ align 64 ymm6, ymm7, ymm8, ymm9 NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 vpsrld ymm7, 1 ; Shift out LSB of W - lfsr_updt8 rax, N, ymm7 ; W (ymm7) used in LFSR update + LFSR_UPDT8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + %%MASK_31, ymm7, init ; W (ymm7) used in LFSR update %assign N N+1 %endrep @@ -854,8 +859,8 @@ align 64 ymm6, ymm7, ymm8, ymm9 NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 - vpxor ymm0, ymm0 - lfsr_updt8 rax, 0, ymm0 + LFSR_UPDT8 rax, 0, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + %%MASK_31, ymm7, work FUNC_RESTORE @@ -887,6 +892,8 @@ asm_Zuc256Initialization_8_avx2: %define pKS rdx %endif +%define %%MASK_31 ymm15 + FUNC_SAVE ; Store 8 keystream pointers on the stack @@ -906,7 +913,7 @@ asm_Zuc256Initialization_8_avx2: mov rax, pState ; Load read-only registers - vmovdqa ymm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds %assign N 1 @@ -917,8 +924,8 @@ asm_Zuc256Initialization_8_avx2: ; OFS_X3 XOR W (ymm7) and store in stack vpxor ymm10, ymm7 vmovdqa [rsp + 64 + (N-1)*32], ymm10 - vpxor ymm0, ymm0 - lfsr_updt8 rax, N, ymm0 + LFSR_UPDT8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + %%MASK_31, ymm7, work %assign N N+1 %endrep @@ -1048,8 +1055,11 @@ asm_ZucGenKeystream4B_8_avx2: %define %%TMP1 rdi %define %%TMP2 rsi %endif + +%define %%MASK_31 ymm15 + ; Load read-only registers - vmovdqa ymm12, [rel mask31] + vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds %assign N 1 @@ -1061,8 +1071,8 @@ asm_ZucGenKeystream4B_8_avx2: ; OFS_X3 XOR W (ymm7) vpxor ymm10, ymm7 vmovdqa [rsp + (N-1)*32], ymm10 - vpxor ymm0, ymm0 - lfsr_updt8 rax, round, ymm0 + LFSR_UPDT8 rax, round, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ + %%MASK_31, ymm7, work %assign N N+1 %assign round (round + 1) %endrep -- GitLab From 92bd056200a4c446401a9239cc3aa66053316bf3 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 2 Mar 2022 09:34:34 +0000 Subject: [PATCH 015/369] avx2: [ZUC] refactor main auxiliary functions Refactored init, keygen and cipher assembly functions, defining register names and fixing leading spaces. --- lib/avx2/zuc_x8_avx2.asm | 823 ++++++++++++++++++++------------------- 1 file changed, 432 insertions(+), 391 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 3a5ffe31..322c3051 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -32,6 +32,21 @@ %include "include/memcpy.asm" %include "include/mb_mgr_datastruct.asm" %include "include/cet.inc" + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%define arg5 r8 +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%define arg5 qword [rsp + 40] +%endif + %define APPEND(a,b) a %+ b mksection .rodata @@ -241,14 +256,14 @@ align 64 %if %%NUM_ROUNDS != 16 %assign i 0 %rep 16 - vmovdqa APPEND(ymm,i), [%%STATE + 32*i] + vmovdqa APPEND(ymm,i), [%%STATE + 32*i] %assign i (i+1) %endrep %assign i 0 %assign j %%NUM_ROUNDS %rep 16 - vmovdqa [%%STATE + 32*i], APPEND(ymm,j) + vmovdqa [%%STATE + 32*i], APPEND(ymm,j) %assign i (i+1) %assign j ((j+1) % 16) %endrep @@ -415,20 +430,20 @@ align 64 %define %%DATA32B_L6 %7 ; [in] 32 bytes of keystream for lane 6 %define %%DATA32B_L7 %8 ; [in] 32 bytes of keystream for lane 7 - mov rcx, [rsp] + mov r12, [rsp] mov rdx, [rsp + 8] mov r8, [rsp + 16] mov r9, [rsp + 24] - vmovdqu [rcx], %%DATA32B_L0 + vmovdqu [r12], %%DATA32B_L0 vmovdqu [rdx], %%DATA32B_L1 vmovdqu [r8], %%DATA32B_L2 vmovdqu [r9], %%DATA32B_L3 - mov rcx, [rsp + 32] + mov r12, [rsp + 32] mov rdx, [rsp + 40] mov r8, [rsp + 48] mov r9, [rsp + 56] - vmovdqu [rcx], %%DATA32B_L4 + vmovdqu [r12], %%DATA32B_L4 vmovdqu [rdx], %%DATA32B_L5 vmovdqu [r8], %%DATA32B_L6 vmovdqu [r9], %%DATA32B_L7 @@ -444,40 +459,40 @@ align 64 ; return ; %macro store4B_kstr8 1 - mov rcx, [rsp] - mov rdx, [rsp + 8] - mov r8, [rsp + 16] - mov r9, [rsp + 24] - vpextrd [r9], XWORD(%1), 3 - vpextrd [r8], XWORD(%1), 2 - vpextrd [rdx], XWORD(%1), 1 - vmovd [rcx], XWORD(%1) - add rcx, 4 - add rdx, 4 - add r8, 4 - add r9, 4 - mov [rsp], rcx - mov [rsp + 8], rdx - mov [rsp + 16], r8 - mov [rsp + 24], r9 + mov r12, [rsp] + mov r13, [rsp + 8] + mov r14, [rsp + 16] + mov r15, [rsp + 24] + vpextrd [r15], XWORD(%1), 3 + vpextrd [r14], XWORD(%1), 2 + vpextrd [r13], XWORD(%1), 1 + vmovd [r12], XWORD(%1) + add r12, 4 + add r13, 4 + add r14, 4 + add r15, 4 + mov [rsp], r12 + mov [rsp + 8], r13 + mov [rsp + 16], r14 + mov [rsp + 24], r15 vextracti128 XWORD(%1), %1, 1 - mov rcx, [rsp + 32] - mov rdx, [rsp + 40] - mov r8, [rsp + 48] - mov r9, [rsp + 56] - vpextrd [r9], XWORD(%1), 3 - vpextrd [r8], XWORD(%1), 2 - vpextrd [rdx], XWORD(%1), 1 - vmovd [rcx], XWORD(%1) - add rcx, 4 - add rdx, 4 - add r8, 4 - add r9, 4 - mov [rsp + 32], rcx - mov [rsp + 40], rdx - mov [rsp + 48], r8 - mov [rsp + 56], r9 + mov r12, [rsp + 32] + mov r13, [rsp + 40] + mov r14, [rsp + 48] + mov r15, [rsp + 56] + vpextrd [r15], XWORD(%1), 3 + vpextrd [r14], XWORD(%1), 2 + vpextrd [r13], XWORD(%1), 1 + vmovd [r12], XWORD(%1) + add r12, 4 + add r13, 4 + add r14, 4 + add r15, 4 + mov [rsp + 32], r12 + mov [rsp + 40], r13 + mov [rsp + 48], r14 + mov [rsp + 56], r15 %endmacro @@ -531,30 +546,30 @@ align 64 %define %%W %10 ; [in/clobbered] In init mode, contains W for all 4 lanes %define %%MODE %11 ; [constant] "init" / "work" mode - vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] - vmovdqa %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*32] - vmovdqa %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*32] - vmovdqa %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*32] - vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*32] + vmovdqa %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*32] - ; Calculate LFSR feedback (S_16) + ; Calculate LFSR feedback (S_16) - ; In Init mode, W is added to the S_16 calculation + ; In Init mode, W is added to the S_16 calculation %ifidn %%MODE, init ADD_MOD31 %%W, %%LFSR_0, %%YTMP, %%MASK_31 %else vmovdqa %%W, %%LFSR_0 %endif - ROT_MOD31 %%LFSR_0, %%YTMP, %%MASK_31, 8 - ADD_MOD31 %%W, %%LFSR_0, %%YTMP, %%MASK_31 - ROT_MOD31 %%LFSR_4, %%YTMP, %%MASK_31, 20 - ADD_MOD31 %%W, %%LFSR_4, %%YTMP, %%MASK_31 - ROT_MOD31 %%LFSR_10, %%YTMP, %%MASK_31, 21 - ADD_MOD31 %%W, %%LFSR_10, %%YTMP, %%MASK_31 - ROT_MOD31 %%LFSR_13, %%YTMP, %%MASK_31, 17 - ADD_MOD31 %%W, %%LFSR_13, %%YTMP, %%MASK_31 - ROT_MOD31 %%LFSR_15, %%YTMP, %%MASK_31, 15 - ADD_MOD31 %%W, %%LFSR_15, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_0, %%YTMP, %%MASK_31, 8 + ADD_MOD31 %%W, %%LFSR_0, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_4, %%YTMP, %%MASK_31, 20 + ADD_MOD31 %%W, %%LFSR_4, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_10, %%YTMP, %%MASK_31, 21 + ADD_MOD31 %%W, %%LFSR_10, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_13, %%YTMP, %%MASK_31, 17 + ADD_MOD31 %%W, %%LFSR_13, %%YTMP, %%MASK_31 + ROT_MOD31 %%LFSR_15, %%YTMP, %%MASK_31, 15 + ADD_MOD31 %%W, %%LFSR_15, %%YTMP, %%MASK_31 ; Store LFSR_S16 vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*32], %%W @@ -581,13 +596,13 @@ align 64 %define %%LFSR %6 ;; [out] YMM register to contain initialized LFSR regs %define %%YTMP %7 ;; [clobbered] YMM temporary register - vbroadcastf128 %%LFSR, [%%KEY] - vbroadcastf128 %%YTMP, [%%IV] - vpshufb %%LFSR, %%SHUF_KEY - vpsrld %%LFSR, 1 - vpshufb %%YTMP, %%SHUF_IV - vpor %%LFSR, %%YTMP - vpor %%LFSR, %%EKD_MASK + vbroadcastf128 %%LFSR, [%%KEY] + vbroadcastf128 %%YTMP, [%%IV] + vpshufb %%LFSR, %%SHUF_KEY + vpsrld %%LFSR, 1 + vpshufb %%YTMP, %%SHUF_IV + vpor %%LFSR, %%YTMP + vpor %%LFSR, %%EKD_MASK %endmacro @@ -604,267 +619,274 @@ align 64 %define %%TMP %7 ;; [clobbered] GP temporary register %define %%CONSTANTS %8 ;; [in] Address to constants - ; s0 - s7 - vpxor %%LFSR0_7, %%LFSR0_7 - vpinsrb XWORD(%%LFSR0_7), [%%KEY], 3 ; s0 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 1], 7 ; s1 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 2], 11 ; s2 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 3], 15 ; s3 + ; s0 - s7 + vpxor %%LFSR0_7, %%LFSR0_7 + vpinsrb XWORD(%%LFSR0_7), [%%KEY], 3 ; s0 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 1], 7 ; s1 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 2], 11 ; s2 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 3], 15 ; s3 - vpsrld XWORD(%%LFSR0_7), 1 + vpsrld XWORD(%%LFSR0_7), 1 - vpor XWORD(%%LFSR0_7), [%%CONSTANTS] ; s0 - s3 + vpor XWORD(%%LFSR0_7), [%%CONSTANTS] ; s0 - s3 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 21], 1 ; s0 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 16], 0 ; s0 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 21], 1 ; s0 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 16], 0 ; s0 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 22], 5 ; s1 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 17], 4 ; s1 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 22], 5 ; s1 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 17], 4 ; s1 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 23], 9 ; s2 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 18], 8 ; s2 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 23], 9 ; s2 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 18], 8 ; s2 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 24], 13 ; s3 - vpinsrb XWORD(%%LFSR0_7), [%%KEY + 19], 12 ; s3 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 24], 13 ; s3 + vpinsrb XWORD(%%LFSR0_7), [%%KEY + 19], 12 ; s3 - vpxor %%XTMP, %%XTMP - vpinsrb %%XTMP, [%%KEY + 4], 3 ; s4 - vpinsrb %%XTMP, [%%IV], 7 ; s5 - vpinsrb %%XTMP, [%%IV + 1], 11 ; s6 - vpinsrb %%XTMP, [%%IV + 10], 15 ; s7 + vpxor %%XTMP, %%XTMP + vpinsrb %%XTMP, [%%KEY + 4], 3 ; s4 + vpinsrb %%XTMP, [%%IV], 7 ; s5 + vpinsrb %%XTMP, [%%IV + 1], 11 ; s6 + vpinsrb %%XTMP, [%%IV + 10], 15 ; s7 - vpsrld %%XTMP, 1 + vpsrld %%XTMP, 1 - vpinsrb %%XTMP, [%%KEY + 25], 1 ; s4 - vpinsrb %%XTMP, [%%KEY + 20], 0 ; s4 + vpinsrb %%XTMP, [%%KEY + 25], 1 ; s4 + vpinsrb %%XTMP, [%%KEY + 20], 0 ; s4 - vpinsrb %%XTMP, [%%KEY + 5], 5 ; s5 - vpinsrb %%XTMP, [%%KEY + 26], 4 ; s5 + vpinsrb %%XTMP, [%%KEY + 5], 5 ; s5 + vpinsrb %%XTMP, [%%KEY + 26], 4 ; s5 - vpinsrb %%XTMP, [%%KEY + 6], 9 ; s6 - vpinsrb %%XTMP, [%%KEY + 27], 8 ; s6 + vpinsrb %%XTMP, [%%KEY + 6], 9 ; s6 + vpinsrb %%XTMP, [%%KEY + 27], 8 ; s6 - vpinsrb %%XTMP, [%%KEY + 7], 13 ; s7 - vpinsrb %%XTMP, [%%IV + 2], 12 ; s7 + vpinsrb %%XTMP, [%%KEY + 7], 13 ; s7 + vpinsrb %%XTMP, [%%IV + 2], 12 ; s7 - vpor %%XTMP, [%%CONSTANTS + 16] ; s4 - s7 + vpor %%XTMP, [%%CONSTANTS + 16] ; s4 - s7 - vmovd %%XTMP2, [%%IV + 17] - vpshufb %%XTMP2, [rel shuf_mask_iv_17_19] - vpand %%XTMP2, [rel clear_iv_mask] + vmovd %%XTMP2, [%%IV + 17] + vpshufb %%XTMP2, [rel shuf_mask_iv_17_19] + vpand %%XTMP2, [rel clear_iv_mask] - vpor %%XTMP, %%XTMP2 + vpor %%XTMP, %%XTMP2 - vinserti128 %%LFSR0_7, %%XTMP, 1 + vinserti128 %%LFSR0_7, %%XTMP, 1 - ; s8 - s15 - vpxor %%LFSR8_15, %%LFSR8_15 - vpinsrb XWORD(%%LFSR8_15), [%%KEY + 8], 3 ; s8 - vpinsrb XWORD(%%LFSR8_15), [%%KEY + 9], 7 ; s9 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 5], 11 ; s10 - vpinsrb XWORD(%%LFSR8_15), [%%KEY + 11], 15 ; s11 + ; s8 - s15 + vpxor %%LFSR8_15, %%LFSR8_15 + vpinsrb XWORD(%%LFSR8_15), [%%KEY + 8], 3 ; s8 + vpinsrb XWORD(%%LFSR8_15), [%%KEY + 9], 7 ; s9 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 5], 11 ; s10 + vpinsrb XWORD(%%LFSR8_15), [%%KEY + 11], 15 ; s11 - vpsrld XWORD(%%LFSR8_15), 1 + vpsrld XWORD(%%LFSR8_15), 1 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 3], 1 ; s8 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 11], 0 ; s8 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 3], 1 ; s8 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 11], 0 ; s8 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 12], 5 ; s9 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 4], 4 ; s9 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 12], 5 ; s9 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 4], 4 ; s9 - vpinsrb XWORD(%%LFSR8_15), [%%KEY + 10], 9 ; s10 - vpinsrb XWORD(%%LFSR8_15), [%%KEY + 28], 8 ; s10 + vpinsrb XWORD(%%LFSR8_15), [%%KEY + 10], 9 ; s10 + vpinsrb XWORD(%%LFSR8_15), [%%KEY + 28], 8 ; s10 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 6], 13 ; s11 - vpinsrb XWORD(%%LFSR8_15), [%%IV + 13], 12 ; s11 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 6], 13 ; s11 + vpinsrb XWORD(%%LFSR8_15), [%%IV + 13], 12 ; s11 - vpor XWORD(%%LFSR8_15), [%%CONSTANTS + 32] ; s8 - s11 + vpor XWORD(%%LFSR8_15), [%%CONSTANTS + 32] ; s8 - s11 - vmovd %%XTMP, [%%IV + 20] - vpshufb %%XTMP, [rel shuf_mask_iv_20_23] - vpand %%XTMP, [rel clear_iv_mask] + vmovd %%XTMP, [%%IV + 20] + vpshufb %%XTMP, [rel shuf_mask_iv_20_23] + vpand %%XTMP, [rel clear_iv_mask] - vpor XWORD(%%LFSR8_15), %%XTMP + vpor XWORD(%%LFSR8_15), %%XTMP - vpxor %%XTMP, %%XTMP - vpinsrb %%XTMP, [%%KEY + 12], 3 ; s12 - vpinsrb %%XTMP, [%%KEY + 13], 7 ; s13 - vpinsrb %%XTMP, [%%KEY + 14], 11 ; s14 - vpinsrb %%XTMP, [%%KEY + 15], 15 ; s15 + vpxor %%XTMP, %%XTMP + vpinsrb %%XTMP, [%%KEY + 12], 3 ; s12 + vpinsrb %%XTMP, [%%KEY + 13], 7 ; s13 + vpinsrb %%XTMP, [%%KEY + 14], 11 ; s14 + vpinsrb %%XTMP, [%%KEY + 15], 15 ; s15 - vpsrld %%XTMP, 1 + vpsrld %%XTMP, 1 - vpinsrb %%XTMP, [%%IV + 7], 1 ; s12 - vpinsrb %%XTMP, [%%IV + 14], 0 ; s12 + vpinsrb %%XTMP, [%%IV + 7], 1 ; s12 + vpinsrb %%XTMP, [%%IV + 14], 0 ; s12 - vpinsrb %%XTMP, [%%IV + 15], 5 ; s13 - vpinsrb %%XTMP, [%%IV + 8], 4 ; s13 + vpinsrb %%XTMP, [%%IV + 15], 5 ; s13 + vpinsrb %%XTMP, [%%IV + 8], 4 ; s13 - vpinsrb %%XTMP, [%%IV + 16], 9 ; s14 - vpinsrb %%XTMP, [%%IV + 9], 8 ; s14 + vpinsrb %%XTMP, [%%IV + 16], 9 ; s14 + vpinsrb %%XTMP, [%%IV + 9], 8 ; s14 - vpinsrb %%XTMP, [%%KEY + 30], 13 ; s15 - vpinsrb %%XTMP, [%%KEY + 29], 12 ; s15 + vpinsrb %%XTMP, [%%KEY + 30], 13 ; s15 + vpinsrb %%XTMP, [%%KEY + 29], 12 ; s15 - vpor %%XTMP, [%%CONSTANTS + 48] ; s12 - s15 + vpor %%XTMP, [%%CONSTANTS + 48] ; s12 - s15 - movzx DWORD(%%TMP), byte [%%IV + 24] - and DWORD(%%TMP), 0x0000003f - shl DWORD(%%TMP), 16 - vmovd %%XTMP2, DWORD(%%TMP) + movzx DWORD(%%TMP), byte [%%IV + 24] + and DWORD(%%TMP), 0x0000003f + shl DWORD(%%TMP), 16 + vmovd %%XTMP2, DWORD(%%TMP) - movzx DWORD(%%TMP), byte [%%KEY + 31] - shl DWORD(%%TMP), 12 - and DWORD(%%TMP), 0x000f0000 ; high nibble of K_31 - vpinsrd %%XTMP2, DWORD(%%TMP), 2 + movzx DWORD(%%TMP), byte [%%KEY + 31] + shl DWORD(%%TMP), 12 + and DWORD(%%TMP), 0x000f0000 ; high nibble of K_31 + vpinsrd %%XTMP2, DWORD(%%TMP), 2 - movzx DWORD(%%TMP), byte [%%KEY + 31] - shl DWORD(%%TMP), 16 - and DWORD(%%TMP), 0x000f0000 ; low nibble of K_31 - vpinsrd %%XTMP2, DWORD(%%TMP), 3 + movzx DWORD(%%TMP), byte [%%KEY + 31] + shl DWORD(%%TMP), 16 + and DWORD(%%TMP), 0x000f0000 ; low nibble of K_31 + vpinsrd %%XTMP2, DWORD(%%TMP), 3 - vpor %%XTMP, %%XTMP2 - vinserti128 %%LFSR8_15, %%XTMP, 1 + vpor %%XTMP, %%XTMP2 + vinserti128 %%LFSR8_15, %%XTMP, 1 %endmacro %macro ZUC_INIT_8 1 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) -%ifdef LINUX - %define pKe rdi - %define pIv rsi - %define pState rdx - %define tag_sz rcx ; Only used in ZUC-256 -%else - %define pKe rcx - %define pIv rdx - %define pState r8 - %define tag_sz r9 ; Only used in ZUC-256 -%endif - -%define %%MASK_31 ymm15 +%define pKe arg1 +%define pIv arg2 +%define pState arg3 +%define tag_sz arg4 ; Only used in ZUC-256 + +%define %%YTMP1 ymm0 +%define %%YTMP2 ymm1 +%define %%YTMP3 ymm2 +%define %%YTMP4 ymm3 +%define %%YTMP5 ymm4 +%define %%YTMP6 ymm5 +%define %%YTMP7 ymm6 +%define %%YTMP8 ymm7 +%define %%YTMP9 ymm8 +%define %%YTMP10 ymm9 +%define %%YTMP11 ymm10 +%define %%YTMP12 ymm11 +%define %%YTMP13 ymm12 +%define %%YTMP14 ymm13 +%define %%YTMP15 ymm14 +%define %%YTMP16 ymm15 + +%define %%W %%YTMP10 +%define %%X3 %%YTMP11 +%define %%MASK_31 %%YTMP16 - FUNC_SAVE + FUNC_SAVE - ; Zero out R1/R2 (only lower half is used) - vpxor ymm0, ymm0 -%assign I 0 -%rep 2 - vmovdqa [pState + OFS_R1 + I*32], ymm0 -%assign I (I + 1) -%endrep + ; Zero out R1/R2 + vpxor %%YTMP1, %%YTMP1 + vmovdqa [pState + OFS_R1], %%YTMP1 + vmovdqa [pState + OFS_R2], %%YTMP1 - ;;; Initialize all LFSR registers in two steps: - ;;; first, registers 0-7, then registers 8-15 + ;;; Initialize all LFSR registers in two steps: + ;;; first, registers 0-7, then registers 8-15 %if %%KEY_SIZE == 128 -%assign off 0 +%assign %%OFF 0 %rep 2 - ; Set read-only registers for shuffle masks for key, IV and Ek_d for 8 registers - vmovdqa ymm13, [rel shuf_mask_key + off] - vmovdqa ymm14, [rel shuf_mask_iv + off] - vmovdqa ymm15, [rel Ek_d + off] - - ; Set 8xLFSR registers for all packets -%assign idx 0 + ; Set read-only registers for shuffle masks for key, IV and Ek_d for 8 registers + vmovdqa %%YTMP13, [rel shuf_mask_key + %%OFF] + vmovdqa %%YTMP14, [rel shuf_mask_iv + %%OFF] + vmovdqa %%YTMP15, [rel Ek_d + %%OFF] + + ; Set 8xLFSR registers for all packets +%assign %%I 1 +%assign %%OFF_PTR 0 %rep 8 - mov r9, [pKe + 8*idx] ; Load Key N pointer - lea r10, [pIv + 32*idx] ; Load IV N pointer - INIT_LFSR_128 r9, r10, ymm13, ymm14, ymm15, APPEND(ymm, idx), ymm12 -%assign idx (idx + 1) + mov r9, [pKe + %%OFF_PTR] ; Load Key N pointer + lea r10, [pIv + 4*%%OFF_PTR] ; Load IV N pointer + INIT_LFSR_128 r9, r10, %%YTMP13, %%YTMP14, %%YTMP15, APPEND(%%YTMP, %%I), %%YTMP12 +%assign %%I (%%I + 1) +%assign %%OFF_PTR (%%OFF_PTR + 8) %endrep - ; Store 8xLFSR registers in memory (reordering first, - ; so all SX registers are together) - TRANSPOSE8_U32 ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, ymm8, ymm9 + ; Store 8xLFSR registers in memory (reordering first, + ; so all SX registers are together) + TRANSPOSE8_U32 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 -%assign i 0 +%assign %%I 1 %rep 8 - vmovdqa [pState + 8*off + 32*i], APPEND(ymm, i) -%assign i (i+1) + vmovdqa [pState + 8*%%OFF + 32*(%%I-1)], APPEND(%%YTMP, %%I) +%assign %%I (%%I+1) %endrep -%assign off (off + 32) +%assign %%OFF (%%OFF + 32) %endrep %else ;; %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea r13, [rel EK256_d64] - bsf DWORD(tag_sz), DWORD(tag_sz) - dec DWORD(tag_sz) - shl DWORD(tag_sz), 6 - add r13, tag_sz + ; Get pointer to constants (depending on tag size, this will point at + ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) + lea r13, [rel EK256_d64] + bsf DWORD(tag_sz), DWORD(tag_sz) + dec DWORD(tag_sz) + shl DWORD(tag_sz), 6 + add r13, tag_sz ;;; Initialize all LFSR registers -%assign off 0 +%assign %%OFF 0 %rep 8 - ;; Load key and IV for each packet - mov r12, [pKe + off] - lea r10, [pIv + 4*off] ; Load IV N pointer + ;; Load key and IV for each packet + mov r15, [pKe + %%OFF] + lea r10, [pIv + 4*%%OFF] ; Load IV N pointer - ; Initialize S0-15 for each packet - INIT_LFSR_256 r12, r10, ymm0, ymm1, xmm2, xmm3, r11, r13 + ; Initialize S0-15 for each packet + INIT_LFSR_256 r15, r10, %%YTMP1, %%YTMP2, XWORD(%%YTMP3), XWORD(%%YTMP4), r11, r13 -%assign i 0 -%rep 2 - vmovdqa [pState + 256*i + 4*off], APPEND(ymm, i) -%assign i (i+1) -%endrep + vmovdqa [pState + 4*%%OFF], %%YTMP1 + vmovdqa [pState + 256 + 4*%%OFF], %%YTMP2 -%assign off (off + 8) +%assign %%OFF (%%OFF + 8) %endrep ; Read, transpose and store, so all S_X from the 8 packets are in the same register -%assign off 0 +%assign %%OFF 0 %rep 2 -%assign i 0 +%assign %%I 1 %rep 8 - vmovdqa APPEND(ymm, i), [pState + 32*i + off] -%assign i (i+1) + vmovdqa APPEND(%%YTMP, %%I), [pState + 32*(%%I-1) + %%OFF] +%assign %%I (%%I+1) %endrep - TRANSPOSE8_U32 ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, ymm8, ymm9 + TRANSPOSE8_U32 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 -%assign i 0 +%assign %%I 1 %rep 8 - vmovdqa [pState + 32*i + off], APPEND(ymm, i) -%assign i (i+1) + vmovdqa [pState + 32*(%%I-1) + %%OFF], APPEND(%%YTMP, %%I) +%assign %%I (%%I+1) %endrep -%assign off (off + 256) +%assign %%OFF (%%OFF + 256) %endrep %endif ;; %%KEY_SIZE == 256 - ; Load read-only registers - vmovdqa %%MASK_31, [rel mask31] - - mov rax, pState + ; Load read-only registers + vmovdqa %%MASK_31, [rel mask31] - ; Shift LFSR 32-times, update state variables + ; Shift LFSR 32-times, update state variables %assign N 0 %rep 32 - BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - ymm6, ymm7, ymm8, ymm9 - NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 - vpsrld ymm7, 1 ; Shift out LSB of W - LFSR_UPDT8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - %%MASK_31, ymm7, init ; W (ymm7) used in LFSR update + BITS_REORG8 pState, N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 + NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ + %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W + vpsrld %%W, 1 ; Shift out LSB of W + LFSR_UPDT8 pState, N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%MASK_31, %%W, init ; W used in LFSR update %assign N N+1 %endrep - ; And once more, initial round from keygen phase = 33 times - BITS_REORG8 rax, 0, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - ymm6, ymm7, ymm8, ymm9 - NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 - - LFSR_UPDT8 rax, 0, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - %%MASK_31, ymm7, work + ; And once more, initial round from keygen phase = 33 times + BITS_REORG8 pState, 0, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 + NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ + %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W + LFSR_UPDT8 pState, 0, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%MASK_31, %%YTMP8, work - FUNC_RESTORE + FUNC_RESTORE - ret + ret %endmacro MKGLOBAL(asm_ZucInitialization_8_avx2,function,internal) @@ -884,101 +906,111 @@ asm_Zuc256Initialization_8_avx2: %macro KEYGEN_8_AVX2 1 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds -%ifdef LINUX - %define pState rdi - %define pKS rsi -%else - %define pState rcx - %define pKS rdx -%endif +%define pState arg1 +%define pKS arg2 + +%define %%YTMP1 ymm0 +%define %%YTMP2 ymm1 +%define %%YTMP3 ymm2 +%define %%YTMP4 ymm3 +%define %%YTMP5 ymm4 +%define %%YTMP6 ymm5 +%define %%YTMP7 ymm6 +%define %%YTMP8 ymm7 +%define %%YTMP9 ymm8 +%define %%YTMP10 ymm9 +%define %%YTMP11 ymm10 +%define %%YTMP12 ymm11 +%define %%YTMP13 ymm12 +%define %%YTMP14 ymm13 +%define %%YTMP15 ymm14 +%define %%YTMP16 ymm15 + +%define %%W %%YTMP10 +%define %%X3 %%YTMP11 +%define %%MASK_31 %%YTMP16 -%define %%MASK_31 ymm15 - - FUNC_SAVE - - ; Store 8 keystream pointers on the stack - ; and reserve memory for storing keystreams for all 8 buffers - mov r10, rsp - sub rsp, (8*8 + %%NUM_ROUNDS * 32) - and rsp, -32 + FUNC_SAVE -%assign i 0 -%rep 2 - vmovdqa ymm0, [pKS + 32*i] - vmovdqa [rsp + 32*i], ymm0 -%assign i (i+1) -%endrep + ; Store 8 keystream pointers on the stack + ; and reserve memory for storing keystreams for all 8 buffers + mov r10, rsp + sub rsp, (8*8 + %%NUM_ROUNDS * 32) + and rsp, -32 - ; Load state pointer in RAX - mov rax, pState + vmovdqa ymm0, [pKS] + vmovdqa [rsp], ymm0 + vmovdqa ymm0, [pKS + 32] + vmovdqa [rsp + 32], ymm0 - ; Load read-only registers - vmovdqa %%MASK_31, [rel mask31] + ; Load read-only registers + vmovdqa %%MASK_31, [rel mask31] - ; Generate N*4B of keystream in N rounds -%assign N 1 + ; Generate N*4B of keystream in N rounds +%assign %%N 1 %rep %%NUM_ROUNDS - BITS_REORG8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - ymm6, ymm7, ymm8, ymm9, ymm10 - NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 - ; OFS_X3 XOR W (ymm7) and store in stack - vpxor ymm10, ymm7 - vmovdqa [rsp + 64 + (N-1)*32], ymm10 - LFSR_UPDT8 rax, N, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - %%MASK_31, ymm7, work -%assign N N+1 + BITS_REORG8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10, %%X3 + NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ + %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor %%X3, %%W + vmovdqa [rsp + 8*8 + (%%N-1)*32], %%X3 + LFSR_UPDT8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%MASK_31, %%YTMP8, work +%assign %%N %%N+1 %endrep %if (%%NUM_ROUNDS == 8) - ;; Load all OFS_X3 - vmovdqa xmm0,[rsp + 64] - vmovdqa xmm1,[rsp + 64 + 32*1] - vmovdqa xmm2,[rsp + 64 + 32*2] - vmovdqa xmm3,[rsp + 64 + 32*3] - vmovdqa xmm4,[rsp + 64 + 16] - vmovdqa xmm5,[rsp + 64 + 32*1 + 16] - vmovdqa xmm6,[rsp + 64 + 32*2 + 16] - vmovdqa xmm7,[rsp + 64 + 32*3 + 16] - - vinserti128 ymm0, ymm0, [rsp + 64 + 32*4], 0x01 - vinserti128 ymm1, ymm1, [rsp + 64 + 32*5], 0x01 - vinserti128 ymm2, ymm2, [rsp + 64 + 32*6], 0x01 - vinserti128 ymm3, ymm3, [rsp + 64 + 32*7], 0x01 - vinserti128 ymm4, ymm4, [rsp + 64 + 32*4 + 16], 0x01 - vinserti128 ymm5, ymm5, [rsp + 64 + 32*5 + 16], 0x01 - vinserti128 ymm6, ymm6, [rsp + 64 + 32*6 + 16], 0x01 - vinserti128 ymm7, ymm7, [rsp + 64 + 32*7 + 16], 0x01 - - TRANSPOSE8_U32_PRELOADED ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, ymm8, ymm9 - - store32B_kstr8 ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 + ;; Load all OFS_X3 + vmovdqa XWORD(%%YTMP1), [rsp + 8*8] + vmovdqa XWORD(%%YTMP2), [rsp + 8*8 + 32*1] + vmovdqa XWORD(%%YTMP3), [rsp + 8*8 + 32*2] + vmovdqa XWORD(%%YTMP4), [rsp + 8*8 + 32*3] + vmovdqa XWORD(%%YTMP5), [rsp + 8*8 + 16] + vmovdqa XWORD(%%YTMP6), [rsp + 8*8 + 32*1 + 16] + vmovdqa XWORD(%%YTMP7), [rsp + 8*8 + 32*2 + 16] + vmovdqa XWORD(%%YTMP8), [rsp + 8*8 + 32*3 + 16] + + vinserti128 %%YTMP1, %%YTMP1, [rsp + 8*8 + 32*4], 0x01 + vinserti128 %%YTMP2, %%YTMP2, [rsp + 8*8 + 32*5], 0x01 + vinserti128 %%YTMP3, %%YTMP3, [rsp + 8*8 + 32*6], 0x01 + vinserti128 %%YTMP4, %%YTMP4, [rsp + 8*8 + 32*7], 0x01 + vinserti128 %%YTMP5, %%YTMP5, [rsp + 8*8 + 32*4 + 16], 0x01 + vinserti128 %%YTMP6, %%YTMP6, [rsp + 8*8 + 32*5 + 16], 0x01 + vinserti128 %%YTMP7, %%YTMP7, [rsp + 8*8 + 32*6 + 16], 0x01 + vinserti128 %%YTMP8, %%YTMP8, [rsp + 8*8 + 32*7 + 16], 0x01 + + TRANSPOSE8_U32_PRELOADED %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 + + store32B_kstr8 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8 %else ;; NUM_ROUNDS == 8 -%assign idx 0 +%assign %%I 1 %rep %%NUM_ROUNDS - vmovdqa APPEND(ymm, idx), [rsp + 64 + idx*32] - store4B_kstr8 APPEND(ymm, idx) -%assign idx (idx + 1) + vmovdqa APPEND(%%YTMP, %%I), [rsp + 8*8 + (%%I-1)*32] + store4B_kstr8 APPEND(%%YTMP, %%I) +%assign %%I (%%I + 1) %endrep %endif ;; NUM_ROUNDS == 8 - ;; Reorder LFSR registers, as not all 16 rounds have been completed - REORDER_LFSR rax, %%NUM_ROUNDS + ;; Reorder LFSR registers, as not all 16 rounds have been completed + REORDER_LFSR pState, %%NUM_ROUNDS ;; Clear stack frame containing keystream information %ifdef SAFE_DATA - vpxor ymm0, ymm0 -%assign i 0 + vpxor %%YTMP1, %%YTMP1 +%assign %%I 0 %rep (2+%%NUM_ROUNDS) - vmovdqa [rsp + i*32], ymm0 -%assign i (i+1) + vmovdqa [rsp + %%I*32], %%YTMP1 +%assign %%I (%%I+1) %endrep %endif - ;; Restore rsp pointer - mov rsp, r10 + ;; Restore rsp pointer + mov rsp, r10 - FUNC_RESTORE + FUNC_RESTORE %endmacro @@ -995,10 +1027,10 @@ asm_Zuc256Initialization_8_avx2: ;; MKGLOBAL(asm_ZucGenKeystream32B_8_avx2,function,internal) asm_ZucGenKeystream32B_8_avx2: - endbranch64 - KEYGEN_8_AVX2 8 - vzeroupper - ret + endbranch64 + KEYGEN_8_AVX2 8 + vzeroupper + ret ;; ;; void asm_ZucGenKeystream8B_8_avx2(state8_t *pSta, u32* pKeyStr[8]) @@ -1013,10 +1045,10 @@ asm_ZucGenKeystream32B_8_avx2: ;; MKGLOBAL(asm_ZucGenKeystream8B_8_avx2,function,internal) asm_ZucGenKeystream8B_8_avx2: - endbranch64 - KEYGEN_8_AVX2 2 - vzeroupper - ret + endbranch64 + KEYGEN_8_AVX2 2 + vzeroupper + ret ;; ;; void asm_ZucGenKeystream4B_8_avx2(state8_t *pSta, u32* pKeyStr[8]) @@ -1031,10 +1063,10 @@ asm_ZucGenKeystream8B_8_avx2: ;; MKGLOBAL(asm_ZucGenKeystream4B_8_avx2,function,internal) asm_ZucGenKeystream4B_8_avx2: - endbranch64 - KEYGEN_8_AVX2 1 - vzeroupper - ret + endbranch64 + KEYGEN_8_AVX2 1 + vzeroupper + ret ;; ;; Encrypt N*4B bytes on all 8 buffers @@ -1056,37 +1088,55 @@ asm_ZucGenKeystream4B_8_avx2: %define %%TMP2 rsi %endif -%define %%MASK_31 ymm15 +%define %%YTMP1 ymm0 +%define %%YTMP2 ymm1 +%define %%YTMP3 ymm2 +%define %%YTMP4 ymm3 +%define %%YTMP5 ymm4 +%define %%YTMP6 ymm5 +%define %%YTMP7 ymm6 +%define %%YTMP8 ymm7 +%define %%YTMP9 ymm8 +%define %%YTMP10 ymm9 +%define %%YTMP11 ymm10 +%define %%YTMP12 ymm11 +%define %%YTMP13 ymm12 +%define %%YTMP14 ymm13 +%define %%YTMP15 ymm14 +%define %%YTMP16 ymm15 + +%define %%W %%YTMP10 +%define %%X3 %%YTMP11 +%define %%MASK_31 %%YTMP16 ; Load read-only registers vmovdqa %%MASK_31, [rel mask31] ; Generate N*4B of keystream in N rounds -%assign N 1 -%assign round (%%INITIAL_ROUND + N) +%assign %%N 1 +%assign %%round (%%INITIAL_ROUND + %%N) %rep %%NROUNDS - BITS_REORG8 rax, round, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - ymm6, ymm7, ymm8, ymm9, ymm10 - NONLIN_FUN8 rax, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7 - ; OFS_X3 XOR W (ymm7) - vpxor ymm10, ymm7 - vmovdqa [rsp + (N-1)*32], ymm10 - LFSR_UPDT8 rax, round, ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, \ - %%MASK_31, ymm7, work -%assign N N+1 -%assign round (round + 1) + BITS_REORG8 pState, %%round, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10, %%X3 + NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ + %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor %%X3, %%W + vmovdqa [rsp + (%%N-1)*32], %%X3 + LFSR_UPDT8 pState, %%round, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%MASK_31, %%YTMP8, work +%assign %%N (%%N + 1) +%assign %%round (%%round + 1) %endrep -%assign N 0 -%assign idx 8 +%assign %%N 1 %rep %%NROUNDS - vmovdqa APPEND(ymm, idx), [rsp + N*32] -%assign N N+1 -%assign idx (idx+1) + vmovdqa APPEND(%%YTMP, %%N), [rsp + (%%N-1)*32] +%assign %%N (%%N + 1) %endrep - TRANSPOSE8_U32 ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, \ - ymm15, ymm0, ymm1 + TRANSPOSE8_U32 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 ;; XOR Input buffer with keystream in rounds of 32B mov r12, [pIn] @@ -1100,18 +1150,18 @@ asm_ZucGenKeystream4B_8_avx2: ;; Read in r10 the word containing the number of final bytes to read for each lane movzx r10d, word [rsp + 8*32] - simd_load_avx2 ymm0, r12 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP9, r12 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 2] - simd_load_avx2 ymm1, r13 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP10, r13 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 4] - simd_load_avx2 ymm2, r14 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP11, r14 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 6] - simd_load_avx2 ymm3, r15 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP12, r15 + %%OFFSET, r10, %%TMP1, %%TMP2 %else - vmovdqu ymm0, [r12 + %%OFFSET] - vmovdqu ymm1, [r13 + %%OFFSET] - vmovdqu ymm2, [r14 + %%OFFSET] - vmovdqu ymm3, [r15 + %%OFFSET] + vmovdqu %%YTMP9, [r12 + %%OFFSET] + vmovdqu %%YTMP10, [r13 + %%OFFSET] + vmovdqu %%YTMP11, [r14 + %%OFFSET] + vmovdqu %%YTMP12, [r15 + %%OFFSET] %endif mov r12, [pIn + 32] @@ -1120,25 +1170,25 @@ asm_ZucGenKeystream4B_8_avx2: mov r15, [pIn + 56] %if (%%LAST_CALL == 1) movzx r10d, word [rsp + 8*32 + 8] - simd_load_avx2 ymm4, r12 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP13, r12 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 10] - simd_load_avx2 ymm5, r13 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP14, r13 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 12] - simd_load_avx2 ymm6, r14 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP15, r14 + %%OFFSET, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 14] - simd_load_avx2 ymm7, r15 + %%OFFSET, r10, %%TMP1, %%TMP2 + simd_load_avx2 %%YTMP16, r15 + %%OFFSET, r10, %%TMP1, %%TMP2 %else - vmovdqu ymm4, [r12 + %%OFFSET] - vmovdqu ymm5, [r13 + %%OFFSET] - vmovdqu ymm6, [r14 + %%OFFSET] - vmovdqu ymm7, [r15 + %%OFFSET] + vmovdqu %%YTMP13, [r12 + %%OFFSET] + vmovdqu %%YTMP14, [r13 + %%OFFSET] + vmovdqu %%YTMP15, [r14 + %%OFFSET] + vmovdqu %%YTMP16, [r15 + %%OFFSET] %endif ; Shuffle all keystreams and XOR with plaintext -%assign %%I 0 -%assign %%J 8 +%assign %%I 1 +%assign %%J 9 %rep 8 - vpshufb ymm %+ %%J, [rel swap_mask] - vpxor ymm %+ %%J, ymm %+ %%I + vpshufb APPEND(%%YTMP, %%I), [rel swap_mask] + vpxor APPEND(%%YTMP, %%I), APPEND(%%YTMP, %%J) %assign %%I (%%I + 1) %assign %%J (%%J + 1) %endrep @@ -1156,18 +1206,18 @@ asm_ZucGenKeystream4B_8_avx2: add r15, %%OFFSET ;; Read in r10 the word containing the number of final bytes to write for each lane movzx r10d, word [rsp + 8*32] - simd_store_avx2 r12, ymm8, r10, %%TMP1, %%TMP2 + simd_store_avx2 r12, %%YTMP1, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 2] - simd_store_avx2 r13, ymm9, r10, %%TMP1, %%TMP2 + simd_store_avx2 r13, %%YTMP2, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 4] - simd_store_avx2 r14, ymm10, r10, %%TMP1, %%TMP2 + simd_store_avx2 r14, %%YTMP3, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 6] - simd_store_avx2 r15, ymm11, r10, %%TMP1, %%TMP2 + simd_store_avx2 r15, %%YTMP4, r10, %%TMP1, %%TMP2 %else - vmovdqu [r12 + %%OFFSET], ymm8 - vmovdqu [r13 + %%OFFSET], ymm9 - vmovdqu [r14 + %%OFFSET], ymm10 - vmovdqu [r15 + %%OFFSET], ymm11 + vmovdqu [r12 + %%OFFSET], %%YTMP1 + vmovdqu [r13 + %%OFFSET], %%YTMP2 + vmovdqu [r14 + %%OFFSET], %%YTMP3 + vmovdqu [r15 + %%OFFSET], %%YTMP4 %endif mov r12, [pOut + 32] @@ -1181,22 +1231,22 @@ asm_ZucGenKeystream4B_8_avx2: add r14, %%OFFSET add r15, %%OFFSET movzx r10d, word [rsp + 8*32 + 8] - simd_store_avx2 r12, ymm12, r10, %%TMP1, %%TMP2 + simd_store_avx2 r12, %%YTMP5, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 10] - simd_store_avx2 r13, ymm13, r10, %%TMP1, %%TMP2 + simd_store_avx2 r13, %%YTMP6, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 12] - simd_store_avx2 r14, ymm14, r10, %%TMP1, %%TMP2 + simd_store_avx2 r14, %%YTMP7, r10, %%TMP1, %%TMP2 movzx r10d, word [rsp + 8*32 + 14] - simd_store_avx2 r15, ymm15, r10, %%TMP1, %%TMP2 + simd_store_avx2 r15, %%YTMP8, r10, %%TMP1, %%TMP2 ; Restore registers mov %%TMP1, [rsp + 32*8 + 16 + 8] mov %%TMP2, [rsp + 32*8 + 16 + 16] %else - vmovdqu [r12 + %%OFFSET], ymm12 - vmovdqu [r13 + %%OFFSET], ymm13 - vmovdqu [r14 + %%OFFSET], ymm14 - vmovdqu [r15 + %%OFFSET], ymm15 + vmovdqu [r12 + %%OFFSET], %%YTMP5 + vmovdqu [r13 + %%OFFSET], %%YTMP6 + vmovdqu [r14 + %%OFFSET], %%YTMP7 + vmovdqu [r15 + %%OFFSET], %%YTMP8 %endif %endmacro @@ -1221,24 +1271,15 @@ asm_ZucGenKeystream4B_8_avx2: ;; MKGLOBAL(asm_ZucCipher_8_avx2,function,internal) asm_ZucCipher_8_avx2: - endbranch64 -%ifdef LINUX - %define pState rdi - %define pIn rsi - %define pOut rdx - %define lengths rcx - %define arg5 r8 -%else - %define pState rcx - %define pIn rdx - %define pOut r8 - %define lengths r9 - %define arg5 [rsp + 40] -%endif +%define pState arg1 +%define pIn arg2 +%define pOut arg3 +%define lengths arg4 %define min_length r10 %define buf_idx r11 + endbranch64 mov min_length, arg5 or min_length, min_length -- GitLab From 6b627f30a9c46c0a41fd2825796d626119762933 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 3 Mar 2022 14:07:23 +0000 Subject: [PATCH 016/369] avx2: [ZUC] refactor store keystream functions Refactored 4-byte and 32-byte keystream storing functions. --- lib/avx2/zuc_x8_avx2.asm | 127 ++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 322c3051..5ab2cea1 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -418,9 +418,9 @@ align 64 %endmacro ; -; store32B_kstr8() +; Stores 32 bytes of keystream for 8 lanes ; -%macro store32B_kstr8 8 +%macro STORE32B_KSTR8 13 %define %%DATA32B_L0 %1 ; [in] 32 bytes of keystream for lane 0 %define %%DATA32B_L1 %2 ; [in] 32 bytes of keystream for lane 1 %define %%DATA32B_L2 %3 ; [in] 32 bytes of keystream for lane 2 @@ -429,70 +429,71 @@ align 64 %define %%DATA32B_L5 %6 ; [in] 32 bytes of keystream for lane 5 %define %%DATA32B_L6 %7 ; [in] 32 bytes of keystream for lane 6 %define %%DATA32B_L7 %8 ; [in] 32 bytes of keystream for lane 7 - - mov r12, [rsp] - mov rdx, [rsp + 8] - mov r8, [rsp + 16] - mov r9, [rsp + 24] - vmovdqu [r12], %%DATA32B_L0 - vmovdqu [rdx], %%DATA32B_L1 - vmovdqu [r8], %%DATA32B_L2 - vmovdqu [r9], %%DATA32B_L3 - - mov r12, [rsp + 32] - mov rdx, [rsp + 40] - mov r8, [rsp + 48] - mov r9, [rsp + 56] - vmovdqu [r12], %%DATA32B_L4 - vmovdqu [rdx], %%DATA32B_L5 - vmovdqu [r8], %%DATA32B_L6 - vmovdqu [r9], %%DATA32B_L7 +%define %%OUT_PTRS %9 ; [in] Keystream pointers for all 8 lanes +%define %%TMP1 %10 ; [clobbered] Temporary GP register +%define %%TMP2 %11 ; [clobbered] Temporary GP register +%define %%TMP3 %12 ; [clobbered] Temporary GP register +%define %%TMP4 %13 ; [clobbered] Temporary GP register + + mov %%TMP1, [%%OUT_PTRS] + mov %%TMP2, [%%OUT_PTRS + 8] + mov %%TMP3, [%%OUT_PTRS + 16] + mov %%TMP4, [%%OUT_PTRS + 24] + vmovdqu [%%TMP1], %%DATA32B_L0 + vmovdqu [%%TMP2], %%DATA32B_L1 + vmovdqu [%%TMP3], %%DATA32B_L2 + vmovdqu [%%TMP4], %%DATA32B_L3 + + mov %%TMP1, [%%OUT_PTRS + 32] + mov %%TMP2, [%%OUT_PTRS + 40] + mov %%TMP3, [%%OUT_PTRS + 48] + mov %%TMP4, [%%OUT_PTRS + 56] + vmovdqu [%%TMP1], %%DATA32B_L4 + vmovdqu [%%TMP2], %%DATA32B_L5 + vmovdqu [%%TMP3], %%DATA32B_L6 + vmovdqu [%%TMP4], %%DATA32B_L7 %endmacro ; -; store4B_kstr8() -; -; params -; -; %1 - YMM register with OFS_X3 -; return +; Stores 4 bytes of keystream for 8 lanes ; -%macro store4B_kstr8 1 - mov r12, [rsp] - mov r13, [rsp + 8] - mov r14, [rsp + 16] - mov r15, [rsp + 24] - vpextrd [r15], XWORD(%1), 3 - vpextrd [r14], XWORD(%1), 2 - vpextrd [r13], XWORD(%1), 1 - vmovd [r12], XWORD(%1) - add r12, 4 - add r13, 4 - add r14, 4 - add r15, 4 - mov [rsp], r12 - mov [rsp + 8], r13 - mov [rsp + 16], r14 - mov [rsp + 24], r15 - - vextracti128 XWORD(%1), %1, 1 - mov r12, [rsp + 32] - mov r13, [rsp + 40] - mov r14, [rsp + 48] - mov r15, [rsp + 56] - vpextrd [r15], XWORD(%1), 3 - vpextrd [r14], XWORD(%1), 2 - vpextrd [r13], XWORD(%1), 1 - vmovd [r12], XWORD(%1) - add r12, 4 - add r13, 4 - add r14, 4 - add r15, 4 - mov [rsp + 32], r12 - mov [rsp + 40], r13 - mov [rsp + 48], r14 - mov [rsp + 56], r15 +%macro STORE4B_KSTR8 6 +%define %%DATA4B_L07 %1 ; [in] 4 bytes of keystream for lanes 0-7 +%define %%OUT_PTRS %2 ; [in] Keystream pointers for all 8 lanes +%define %%TMP1 %3 ; [clobbered] Temporary GP register +%define %%TMP2 %4 ; [clobbered] Temporary GP register +%define %%TMP3 %5 ; [clobbered] Temporary GP register +%define %%TMP4 %6 ; [clobbered] Temporary GP register + + mov %%TMP1, [%%OUT_PTRS] + mov %%TMP2, [%%OUT_PTRS + 8] + mov %%TMP3, [%%OUT_PTRS + 16] + mov %%TMP4, [%%OUT_PTRS + 24] + vpextrd [%%TMP4], XWORD(%%DATA4B_L07), 3 + vpextrd [%%TMP3], XWORD(%%DATA4B_L07), 2 + vpextrd [%%TMP2], XWORD(%%DATA4B_L07), 1 + vmovd [%%TMP1], XWORD(%%DATA4B_L07) + mov DWORD(%%TMP1), 4 + add [%%OUT_PTRS], %%TMP1 + add [%%OUT_PTRS + 8], %%TMP1 + add [%%OUT_PTRS + 16], %%TMP1 + add [%%OUT_PTRS + 24], %%TMP1 + + vextracti128 XWORD(%1), %1, 1 + mov %%TMP1, [%%OUT_PTRS + 32] + mov %%TMP2, [%%OUT_PTRS + 40] + mov %%TMP3, [%%OUT_PTRS + 48] + mov %%TMP4, [%%OUT_PTRS + 56] + vpextrd [%%TMP4], XWORD(%%DATA4B_L07), 3 + vpextrd [%%TMP3], XWORD(%%DATA4B_L07), 2 + vpextrd [%%TMP2], XWORD(%%DATA4B_L07), 1 + vmovd [%%TMP1], XWORD(%%DATA4B_L07) + mov DWORD(%%TMP1), 4 + add [%%OUT_PTRS + 32], %%TMP1 + add [%%OUT_PTRS + 40], %%TMP1 + add [%%OUT_PTRS + 48], %%TMP1 + add [%%OUT_PTRS + 56], %%TMP1 %endmacro @@ -983,13 +984,13 @@ asm_Zuc256Initialization_8_avx2: TRANSPOSE8_U32_PRELOADED %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 - store32B_kstr8 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8 + STORE32B_KSTR8 %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%YTMP8, rsp, r12, r13, r14, r15 %else ;; NUM_ROUNDS == 8 %assign %%I 1 %rep %%NUM_ROUNDS vmovdqa APPEND(%%YTMP, %%I), [rsp + 8*8 + (%%I-1)*32] - store4B_kstr8 APPEND(%%YTMP, %%I) + STORE4B_KSTR8 APPEND(%%YTMP, %%I), rsp, r12, r13, r14, r15 %assign %%I (%%I + 1) %endrep %endif ;; NUM_ROUNDS == 8 -- GitLab From b66ac0b22beecbecb3004c002f0dc71746842bef Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 3 Mar 2022 14:23:33 +0000 Subject: [PATCH 017/369] avx2: [ZUC] remove unused variables --- lib/avx2/zuc_top_avx2.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index 5276d199..01c46c92 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -594,9 +594,6 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[NUM_AVX2_BUFS], /* Process each packet separately for the remaining bits */ for (i = 0; i < NUM_AVX2_BUFS; i++) { - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; @@ -628,7 +625,6 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[NUM_AVX2_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 32 bytes */ if (!remainBits) @@ -778,9 +774,6 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], if (job_in_lane[i] == NULL) continue; - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; @@ -812,7 +805,6 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 32 bytes */ if (!remainBits) -- GitLab From b99569b9efb6e05c38d5d73cb902cd4960ae684b Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 23 Feb 2022 11:20:16 +0000 Subject: [PATCH 018/369] avx512: [AES-ECB] add VAES implementation --- README | 6 +- README.md | 6 +- ReleaseNotes.txt | 1 + lib/Makefile | 1 + lib/avx512/aes_ecb_vaes_avx512.asm | 250 ++++++++++++++++++++++++ lib/avx512/mb_mgr_avx512.c | 60 ++++-- lib/include/aes_common.asm | 302 +++++++++++++++++++++++++++++ lib/include/asm.h | 14 ++ lib/win_x64.mak | 1 + 9 files changed, 623 insertions(+), 18 deletions(-) create mode 100644 lib/avx512/aes_ecb_vaes_avx512.asm diff --git a/README b/README index 4cdf8e1b..bbb43105 100644 --- a/README +++ b/README @@ -75,9 +75,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES128-ECB | N | Y by4 | Y by4 | N | N | N | - | AES192-ECB | N | Y by4 | Y by4 | N | N | N | - | AES256-ECB | N | Y by4 | Y by4 | N | N | N | + | AES128-ECB | N | Y by4 | Y by4 | N | N | Y by16 | + | AES192-ECB | N | Y by4 | Y by4 | N | N | Y by16 | + | AES256-ECB | N | Y by4 | Y by4 | N | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | diff --git a/README.md b/README.md index b219683c..87753547 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | -| AES128-ECB | N | Y by4 | Y by4 | N | N | N | -| AES192-ECB | N | Y by4 | Y by4 | N | N | N | -| AES256-ECB | N | Y by4 | Y by4 | N | N | N | +| AES128-ECB | N | Y by4 | Y by4 | N | N | Y by16 | +| AES192-ECB | N | Y by4 | Y by4 | N | N | Y by16 | +| AES256-ECB | N | Y by4 | Y by4 | N | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 080f55fd..5c7e27c0 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -6,6 +6,7 @@ Unreleased Library - ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE +- AES-ECB AVX512-VAES implementation added v1.2 February 2022 ======================================================================== diff --git a/lib/Makefile b/lib/Makefile index c4ebebc8..ea4d5c89 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -564,6 +564,7 @@ asm_avx512_lib_objs := \ sha256_x16_avx512.o \ sha512_x8_avx512.o \ des_x16_avx512.o \ + aes_ecb_vaes_avx512.o \ aes_cntr_api_by16_vaes_avx512.o \ aes_cntr_bit_api_by16_vaes_avx512.o \ aes_cntr_ccm_api_by16_vaes_avx512.o \ diff --git a/lib/avx512/aes_ecb_vaes_avx512.asm b/lib/avx512/aes_ecb_vaes_avx512.asm new file mode 100644 index 00000000..c0e8bca9 --- /dev/null +++ b/lib/avx512/aes_ecb_vaes_avx512.asm @@ -0,0 +1,250 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB encrypt/decrypt on 16n bytes doing AES by 16 + +; YMM registers are clobbered. Saving/restoring must be done at a higher level + +; void aes_ecb_x_y_vaes_avx512(void *in, +; UINT128 keys[], +; void *out, +; UINT64 len_bytes); +; +; x = direction (enc/dec) +; y = key size (128/192/256) +; arg 1: IN: pointer to input (cipher text) +; arg 2: KEYS: pointer to keys +; arg 3: OUT: pointer to output (plain text) +; arg 4: LEN: length in bytes (multiple of 16) +; + +%include "include/os.asm" +%include "include/clear_regs.asm" +%include "include/aes_common.asm" +%define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx512 +%define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx512 +%define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx512 +%define AES_ECB_DEC_192 aes_ecb_dec_192_vaes_avx512 +%define AES_ECB_ENC_256 aes_ecb_enc_256_vaes_avx512 +%define AES_ECB_DEC_256 aes_ecb_dec_256_vaes_avx512 +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define LEN rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define LEN r9 +%endif +%define IDX rax +%define TMP r11 + +%define YKEY1 ymm1 +%define YDATA0 ymm2 +%define YDATA1 ymm3 +%define YDATA2 ymm4 +%define YDATA3 ymm5 +%define YDATA4 ymm6 +%define YDATA5 ymm7 +%define YDATA6 ymm8 +%define YDATA7 ymm9 + +mksection .text + +%macro AES_ECB 2 +%define %%NROUNDS %1 ; [in] Number of AES rounds, numerical value +%define %%DIR %2 ; [in] Direction (encrypt/decrypt) +%ifidn %%DIR, ENC +%define AES YMM_AESENC_ROUND_BLOCKS_0_16 +%else ; DIR = DEC +%define AES YMM_AESDEC_ROUND_BLOCKS_0_16 +%endif + + or LEN, LEN + jz %%done + + xor IDX, IDX + mov TMP, LEN + and TMP, 255 + jz %%main_loop + + ; branch to different code block based on remainder + cmp TMP, 8*16 + je %%initial_num_blocks_is_8 + jb %%initial_num_blocks_is_7_1 + cmp TMP, 12*16 + je %%initial_num_blocks_is_12 + jb %%initial_num_blocks_is_11_9 + ;; 15, 14 or 13 + cmp TMP, 14*16 + ja %%initial_num_blocks_is_15 + je %%initial_num_blocks_is_14 + jmp %%initial_num_blocks_is_13 +%%initial_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp TMP, 10*16 + ja %%initial_num_blocks_is_11 + je %%initial_num_blocks_is_10 + jmp %%initial_num_blocks_is_9 +%%initial_num_blocks_is_7_1: + cmp TMP, 4*16 + je %%initial_num_blocks_is_4 + jb %%initial_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp TMP, 6*16 + ja %%initial_num_blocks_is_7 + je %%initial_num_blocks_is_6 + jmp %%initial_num_blocks_is_5 +%%initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2*16 + ja %%initial_num_blocks_is_3 + je %%initial_num_blocks_is_2 + ;; fall through for `jmp %%initial_num_blocks_is_1` + +%assign num_blocks 1 +%rep 15 + + %%initial_num_blocks_is_ %+ num_blocks : +%assign %%I 0 + ; load initial blocks + YMM_LOAD_BLOCKS_0_16 num_blocks, IN, 0, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + +; Perform AES encryption/decryption on initial blocks +%rep (%%NROUNDS + 1) ; 10/12/14 + vbroadcasti128 YKEY1, [KEYS + %%I*16] + AES YDATA0, YDATA1, YDATA2, YDATA3, YDATA4,\ + YDATA5, YDATA6, YDATA7, YKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_blocks, (%%NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + + ; store initial blocks + YMM_STORE_BLOCKS_0_16 num_blocks, OUT, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + add IDX, num_blocks*16 + cmp IDX, LEN + je %%done + +%assign num_blocks (num_blocks + 1) + jmp %%main_loop +%endrep + +align 16 +%%main_loop: + ; load the next 16 blocks into ymm registers + YMM_LOAD_BLOCKS_0_16 16, {IN + IDX}, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + ; Perform AES encryption/decryption on 16 blocks +%assign %%ROUNDNO 0 ; current key number +%rep (%%NROUNDS + 1) ; 10/12/14 + vbroadcasti128 YKEY1, [KEYS + %%ROUNDNO*16] + + AES YDATA0, YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7, YKEY1, %%ROUNDNO, no_data, no_data,\ + no_data, no_data, no_data, no_data, no_data, no_data,\ + 16, (%%NROUNDS - 1) + +%assign %%ROUNDNO (%%ROUNDNO + 1) +%endrep + + ; write 16 blocks to output + YMM_STORE_BLOCKS_0_16 16, (OUT + IDX), 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + add IDX, 16*16 + cmp IDX, LEN + jne %%main_loop + +%%done: + +%ifdef SAFE_DATA + clear_all_ymms_asm +%else + vzeroupper +%endif +%endmacro + +;; +;; AES-ECB 128 functions +;; +align 16 +MKGLOBAL(AES_ECB_ENC_128,function,internal) +AES_ECB_ENC_128: + + AES_ECB 10, ENC + ret + +align 16 +MKGLOBAL(AES_ECB_DEC_128,function,internal) +AES_ECB_DEC_128: + + AES_ECB 10, DEC + ret + +;; +;; AES-ECB 192 functions +;; +align 16 +MKGLOBAL(AES_ECB_ENC_192,function,internal) +AES_ECB_ENC_192: + + AES_ECB 12, ENC + ret + +align 16 +MKGLOBAL(AES_ECB_DEC_192,function,internal) +AES_ECB_DEC_192: + + AES_ECB 12, DEC + ret + +;; +;; AES-ECB 256 functions +;; +align 16 +MKGLOBAL(AES_ECB_ENC_256,function,internal) +AES_ECB_ENC_256: + + AES_ECB 14, ENC + ret + +align 16 +MKGLOBAL(AES_ECB_DEC_256,function,internal) +AES_ECB_DEC_256: + + AES_ECB 14, DEC + ret + +mksection stack-noexec diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 122bfd5e..56645912 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -205,12 +205,12 @@ IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx512 #define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx512 -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx512 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx512 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx512 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx512 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx512 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx512 #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx512 #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx512 @@ -235,12 +235,12 @@ IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx512 #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx512 -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx512 +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx512 +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx512 +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx512 +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx512 +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx512 #define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx512 #define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx512 @@ -927,6 +927,36 @@ static void uint64_t len_bytes, void *next_iv) = aes_cbcs_1_9_dec_128_avx; +static void +(*aes_ecb_enc_128_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_enc_128_avx; + +static void +(*aes_ecb_enc_192_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_enc_192_avx; + +static void +(*aes_ecb_enc_256_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_enc_256_avx; + +static void +(*aes_ecb_dec_128_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_dec_128_avx; + +static void +(*aes_ecb_dec_192_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_dec_192_avx; + +static void +(*aes_ecb_dec_256_avx512) (const void *in, const void *keys, + void *out, const uint64_t len_bytes) = + aes_ecb_dec_256_avx; + /* ====================================================================== */ __forceinline @@ -1775,6 +1805,12 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) aes_cbc_dec_128_avx512 = aes_cbc_dec_128_vaes_avx512; aes_cbc_dec_192_avx512 = aes_cbc_dec_192_vaes_avx512; aes_cbc_dec_256_avx512 = aes_cbc_dec_256_vaes_avx512; + aes_ecb_enc_128_avx512 = aes_ecb_enc_128_vaes_avx512; + aes_ecb_enc_192_avx512 = aes_ecb_enc_192_vaes_avx512; + aes_ecb_enc_256_avx512 = aes_ecb_enc_256_vaes_avx512; + aes_ecb_dec_128_avx512 = aes_ecb_dec_128_vaes_avx512; + aes_ecb_dec_192_avx512 = aes_ecb_dec_192_vaes_avx512; + aes_ecb_dec_256_avx512 = aes_ecb_dec_256_vaes_avx512; submit_job_aes128_enc_avx512 = submit_job_aes128_enc_vaes_avx512; flush_job_aes128_enc_avx512 = diff --git a/lib/include/aes_common.asm b/lib/include/aes_common.asm index d428bb9d..cfb11289 100644 --- a/lib/include/aes_common.asm +++ b/lib/include/aes_common.asm @@ -126,6 +126,41 @@ %endmacro +;; ============================================================================= +;; Loads specified number of AES blocks into YMM registers +%macro YMM_LOAD_BLOCKS_0_16 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] YMM register with loaded data +%define %%DST1 %5 ; [out] YMM register with loaded data +%define %%DST2 %6 ; [out] YMM register with loaded data +%define %%DST3 %7 ; [out] YMM register with loaded data +%define %%DST4 %8 ; [out] YMM register with loaded data +%define %%DST5 %9 ; [out] YMM register with loaded data +%define %%DST6 %10 ; [out] YMM register with loaded data +%define %%DST7 %11 ; [out] YMM register with loaded data + +%assign src_offset 0 +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%DSTREG %%DST %+ dst_idx + vmovdqu8 %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 32) +%assign dst_idx (dst_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 2) +%xdefine %%DSTREG %%DST %+ dst_idx + +%if blocks_left == 1 + vmovdqu8 XWORD(%%DSTREG), [%%INP + %%DATA_OFFSET + src_offset] +%endif + +%endmacro + ;; ============================================================================= ;; Loads specified number of AES blocks at offsets into ZMM registers ;; DATA_OFFSET specifies the offset between blocks to load @@ -276,6 +311,41 @@ %endmacro +;; ============================================================================= +;; Stores specified number of AES blocks from YMM registers +%macro YMM_STORE_BLOCKS_0_16 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] YMM register with data to store +%define %%SRC1 %5 ; [in] YMM register with data to store +%define %%SRC2 %6 ; [in] YMM register with data to store +%define %%SRC3 %7 ; [in] YMM register with data to store +%define %%SRC4 %8 ; [in] YMM register with data to store +%define %%SRC5 %9 ; [in] YMM register with data to store +%define %%SRC6 %10 ; [in] YMM register with data to store +%define %%SRC7 %11 ; [in] YMM register with data to store + +%assign dst_offset 0 +%assign src_idx 0 + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%SRCREG %%SRC %+ src_idx + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 32) +%assign src_idx (src_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 2) +%xdefine %%SRCREG %%SRC %+ src_idx + +%if blocks_left == 1 + vmovdqu8 [%%OUTP + %%DATA_OFFSET + dst_offset], XWORD(%%SRCREG) +%endif + +%endmacro + ;; ============================================================================= ;; Stores specified number of AES blocks from ZMM registers with mask register ;; for the last loaded register (xmm, ymm or zmm). @@ -445,4 +515,236 @@ %endmacro +;; ============================================================================= +;; Generic macro to produce code that executes %%OPCODE instruction +;; on selected number of AES blocks (16 bytes long) between 0 and 16. +;; All three operands of the instruction come from registers. +%macro YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 26 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OPCODE %2 ; [in] instruction name +%define %%DST0 %3 ; [out] destination YMM register +%define %%DST1 %4 ; [out] destination YMM register +%define %%DST2 %5 ; [out] destination YMM register +%define %%DST3 %6 ; [out] destination YMM register +%define %%DST4 %7 ; [out] destination YMM register +%define %%DST5 %8 ; [out] destination YMM register +%define %%DST6 %9 ; [out] destination YMM register +%define %%DST7 %10 ; [out] destination YMM register +%define %%SRC1_0 %11 ; [in] source 1 YMM register +%define %%SRC1_1 %12 ; [in] source 1 YMM register +%define %%SRC1_2 %13 ; [in] source 1 YMM register +%define %%SRC1_3 %14 ; [in] source 1 YMM register +%define %%SRC1_4 %15 ; [in] source 1 YMM register +%define %%SRC1_5 %16 ; [in] source 1 YMM register +%define %%SRC1_6 %17 ; [in] source 1 YMM register +%define %%SRC1_7 %18 ; [in] source 1 YMM register +%define %%SRC2_0 %19 ; [in] source 2 YMM register +%define %%SRC2_1 %20 ; [in] source 2 YMM register +%define %%SRC2_2 %21 ; [in] source 2 YMM register +%define %%SRC2_3 %22 ; [in] source 2 YMM register +%define %%SRC2_4 %23 ; [in] source 2 YMM register +%define %%SRC2_5 %24 ; [in] source 2 YMM register +%define %%SRC2_6 %25 ; [in] source 2 YMM register +%define %%SRC2_7 %26 ; [in] source 2 YMM register + +%assign _reg_idx 0 +%assign _blocks_left %%NUM_BLOCKS + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%DSTREG %%DST %+ _reg_idx +%xdefine %%SRC1REG %%SRC1_ %+ _reg_idx +%xdefine %%SRC2REG %%SRC2_ %+ _reg_idx + %%OPCODE %%DSTREG, %%SRC1REG, %%SRC2REG +%undef %%DSTREG +%undef %%SRC1REG +%undef %%SRC2REG +%assign _reg_idx (_reg_idx + 1) +%assign _blocks_left (_blocks_left - 2) +%endrep + +%xdefine %%DSTREG %%DST %+ _reg_idx +%xdefine %%SRC1REG %%SRC1_ %+ _reg_idx +%xdefine %%SRC2REG %%SRC2_ %+ _reg_idx + +%if _blocks_left == 1 + %%OPCODE XWORD(%%DSTREG), XWORD(%%SRC1REG), XWORD(%%SRC2REG) +%endif + +%endmacro + +;;; =========================================================================== +;;; Handles AES encryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro YMM_AESENC_ROUND_BLOCKS_0_16 20 +%define %%L0B0_1 %1 ; [in/out] ymm; ciphered blocks +%define %%L0B2_3 %2 ; [in/out] ymm; ciphered blocks +%define %%L0B4_5 %3 ; [in/out] ymm; ciphered blocks +%define %%L0B6_7 %4 ; [in/out] ymm; ciphered blocks +%define %%L0B8_9 %5 ; [in/out] ymm; ciphered blocks +%define %%L0B10_11 %6 ; [in/out] ymm; ciphered blocks +%define %%L0B12_13 %7 ; [in/out] ymm; ciphered blocks +%define %%L0B14_15 %8 ; [in/out] ymm; ciphered blocks +%define %%KEY %9 ; [in] ymm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0_1 %11 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D2_3 %12 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D4_5 %13 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D6_7 %14 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D8_9 %15 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D10_11 %16 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D12_13 %17 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D14_15 %18 ; [in] ymm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenc, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenclast, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +%ifnidn %%D0_1, no_data +%ifnidn %%D2_3, no_data +%ifnidn %%D4_5, no_data +%ifnidn %%D6_7, no_data +%ifnidn %%D8_9, no_data +%ifnidn %%D10_11, no_data +%ifnidn %%D12_13, no_data +%ifnidn %%D14_15, no_data + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%D0_1, %%D2_3, %%D4_5, %%D6_7, \ + %%D8_9, %%D10_11, %%D12_13, %%D14_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +;;; =========================================================================== +;;; Handles AES decryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameterto check what needs to be done for the current round. +%macro YMM_AESDEC_ROUND_BLOCKS_0_16 20 +%define %%L0B0_1 %1 ; [in/out] ymm; ciphered blocks +%define %%L0B2_3 %2 ; [in/out] ymm; ciphered blocks +%define %%L0B4_5 %3 ; [in/out] ymm; ciphered blocks +%define %%L0B6_7 %4 ; [in/out] ymm; ciphered blocks +%define %%L0B8_9 %5 ; [in/out] ymm; ciphered blocks +%define %%L0B10_11 %6 ; [in/out] ymm; ciphered blocks +%define %%L0B12_13 %7 ; [in/out] ymm; ciphered blocks +%define %%L0B14_15 %8 ; [in/out] ymm; ciphered blocks +%define %%KEY %9 ; [in] ymm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0_1 %11 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D2_3 %12 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D4_5 %13 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D6_7 %14 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D8_9 %15 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D10_11 %16 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D12_13 %17 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D14_15 %18 ; [in] ymm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdec, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdeclast, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0_1, no_data +%ifnidn %%D2_3, no_data +%ifnidn %%D4_5, no_data +%ifnidn %%D6_7, no_data +%ifnidn %%D8_9, no_data +%ifnidn %%D10_11, no_data +%ifnidn %%D12_13, no_data +%ifnidn %%D14_15, no_data + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxorq, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%D0_1, %%D2_3, %%D4_5, %%D6_7, \ + %%D8_9, %%D10_11, %%D12_13, %%D14_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + %endif ;; _AES_COMMON_ASM diff --git a/lib/include/asm.h b/lib/include/asm.h index 2f9a56a5..34814bbb 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -173,6 +173,13 @@ void aes_ecb_enc_192_avx(const void *in, const void *keys, void aes_ecb_enc_128_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_enc_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); + void aes_ecb_dec_256_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_sse(const void *in, const void *keys, @@ -195,6 +202,13 @@ void aes_ecb_dec_192_avx(const void *in, const void *keys, void aes_ecb_dec_128_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); + /* AES128-ECBENC */ void aes128_ecbenc_x3_sse(const void *in, void *keys, void *out1, void *out2, void *out3); diff --git a/lib/win_x64.mak b/lib/win_x64.mak index cbb4cee1..2f280e56 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -165,6 +165,7 @@ lib_objs1 = \ $(OBJ_DIR)\aes128_ecb_by4_avx.obj \ $(OBJ_DIR)\aes192_ecb_by4_avx.obj \ $(OBJ_DIR)\aes256_ecb_by4_avx.obj \ + $(OBJ_DIR)\aes_ecb_vaes_avx512.obj \ $(OBJ_DIR)\pon_by8_sse.obj \ $(OBJ_DIR)\aes128_cntr_by8_sse.obj \ $(OBJ_DIR)\pon_by8_avx.obj \ -- GitLab From e5b024783c74aac8dbbbd12113a4e362cef9274d Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Thu, 10 Mar 2022 14:42:54 +0000 Subject: [PATCH 019/369] lib: add cipher direction checks --- lib/include/error.inc | 3 ++- lib/include/mb_mgr_code.h | 6 ++++++ lib/intel-ipsec-mb.h | 1 + lib/x86_64/error.c | 5 ++++- test/api_test.c | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 48 insertions(+), 2 deletions(-) diff --git a/lib/include/error.inc b/lib/include/error.inc index 65c8c9f8..75859798 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -89,7 +89,8 @@ SET_ERRNO_TYPES \ IMB_ERR_JOB_NULL_HMAC_IPAD, \ IMB_ERR_JOB_NULL_XCBC_K1_EXP, \ IMB_ERR_JOB_NULL_XCBC_K2, \ - IMB_ERR_JOB_NULL_XCBC_K3 + IMB_ERR_JOB_NULL_XCBC_K3, \ + IMB_ERR_JOB_CIPH_DIR ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 970477a5..f7ff69ad 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1373,6 +1373,12 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) * of XGEM header */ const uint64_t max_pon_len = (1 << 14) + 8; + if (job->cipher_direction != IMB_DIR_DECRYPT && + job->cipher_direction != IMB_DIR_ENCRYPT && + job->cipher_mode != IMB_CIPHER_NULL) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); + return 1; + } switch (job->cipher_mode) { case IMB_CIPHER_CBC: case IMB_CIPHER_CBCS_1_9: diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index a386b031..06c2529d 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -222,6 +222,7 @@ typedef enum { IMB_ERR_JOB_NULL_XCBC_K1_EXP, IMB_ERR_JOB_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K3, + IMB_ERR_JOB_CIPH_DIR, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index a46efe69..96f4e8ee 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -83,7 +83,8 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_JOB_NULL_HMAC_IPAD, IMB_ERR_JOB_NULL_XCBC_K1_EXP, IMB_ERR_JOB_NULL_XCBC_K2, - IMB_ERR_JOB_NULL_XCBC_K3 + IMB_ERR_JOB_NULL_XCBC_K3, + IMB_ERR_JOB_CIPH_DIR }; #ifdef DEBUG @@ -198,6 +199,8 @@ imb_get_strerror(int errnum) return "Null pointer to context (direct API)"; case IMB_ERR_NO_AESNI_EMU: return "No AESNI emulation support"; + case IMB_ERR_JOB_CIPH_DIR: + return "Invalid cipher direction"; default: return strerror(errnum); } diff --git a/test/api_test.c b/test/api_test.c index 06d1eef3..184a831b 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -63,6 +63,7 @@ enum { TEST_CIPH_MSG_LEN_GT_MAX, TEST_CIPH_NEXT_IV_NULL, TEST_CIPH_IV_LEN, + TEST_CIPH_DIR, TEST_INVALID_PON_PLI = 300, }; @@ -1139,6 +1140,40 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) return 1; printf("."); } + /* + * CIPHER_DIR = Invalid dir + */ + for (dir = 0; dir <= 10; dir++) { + /* skip valid directions */ + if (dir == IMB_DIR_ENCRYPT || dir == IMB_DIR_DECRYPT) + continue; + + for (cipher = IMB_CIPHER_CBC; + cipher < IMB_CIPHER_NUM; cipher++) { + + if (cipher == IMB_CIPHER_NULL || + cipher == IMB_CIPHER_CUSTOM) + continue; + + /* + * Skip cipher algorithms belonging to AEAD + * algorithms, as the test is for cipher + * only algorithms */ + if (check_aead(hash, cipher)) + continue; + + order = IMB_ORDER_CIPHER_HASH; + + fill_in_job(&template_job, cipher, dir, + hash, order, &chacha_ctx, &gcm_ctx); + + if (!is_submit_invalid(mb_mgr, &template_job, + TEST_CIPH_DIR, + IMB_ERR_JOB_CIPH_DIR)) + return 1; + printf("."); + } + } /* ======== (encrypt test) * AES_ENC_KEY_EXPANDED = NULL -- GitLab From f1b4866686422e9e677b1b40ba06b62bec2393ba Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 23 Mar 2022 12:31:38 +0000 Subject: [PATCH 020/369] lib: add macro to reserve/restore stack space Add macros to reserve and restore stack space based on number of arguments passed to a function. --- lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 68 ++++-------- lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm | 70 ++++-------- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 58 ++++------ lib/include/os.asm | 38 +++++++ lib/sse/mb_mgr_zuc_submit_flush_sse.asm | 100 +++++------------- 5 files changed, 128 insertions(+), 206 deletions(-) diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index 9a519d18..72e299a7 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -323,11 +323,8 @@ mksection .text %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 24 bytes for 3 parameters - sub rsp, 24 -%endif + RESERVE_STACK_SPACE 3 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -343,9 +340,7 @@ mksection .text call asm_Zuc256Initialization_4_avx %endif -%ifndef LINUX - add rsp, 24 -%endif + RESTORE_STACK_SPACE 3 cmp byte [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_submit_restoring_state @@ -385,11 +380,8 @@ mksection .text %endif mov byte [r12 + _zuc_init_not_done], 0 ; Init done for all lanes - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -398,9 +390,7 @@ mksection .text call asm_ZucCipher_4_avx -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -537,11 +527,8 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 24 bytes for 3 parameters - sub rsp, 24 -%endif + RESERVE_STACK_SPACE 3 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -557,9 +544,8 @@ APPEND(%%skip_eea3_,I): call asm_Zuc256Initialization_4_avx %endif -%ifndef LINUX - add rsp, 24 -%endif + RESTORE_STACK_SPACE 3 + cmp word [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_flush_restoring_state @@ -617,11 +603,8 @@ APPEND3(%%skip_eea3_copy_,I,J): vmovdqa [r12 + _zuc_state + 16*I], xmm1 ; Save new state %assign I (I+1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -630,9 +613,8 @@ APPEND3(%%skip_eea3_copy_,I,J): call asm_ZucCipher_4_avx -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] ; Clear ZUC state of the lane that is returned and NULL lanes @@ -817,11 +799,8 @@ FLUSH_JOB_ZUC256_EEA3: ; to pass parameter to next function mov r11, state - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 + lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -842,9 +821,8 @@ FLUSH_JOB_ZUC256_EEA3: call zuc256_eia3_4_buffer_job_avx %endif -%ifndef LINUX - add rsp, 48 -%endif + RESTORE_STACK_SPACE 6 + mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -971,10 +949,8 @@ APPEND(%%skip_eia3_,I): ; to pass parameter to next function mov r11, state -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 + lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -995,9 +971,7 @@ APPEND(%%skip_eia3_,I): call zuc256_eia3_4_buffer_job_avx %endif -%ifndef LINUX - add rsp, 48 -%endif + RESTORE_STACK_SPACE 6 mov tmp5, [rsp + _null_len_save] mov state, [rsp + _gpr_save + 8*8] diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm index 96ca6ebc..dcc21b00 100644 --- a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm @@ -302,11 +302,8 @@ mksection .text %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 32 bytes for 4 parameters - sub rsp, 32 -%endif + RESERVE_STACK_SPACE 4 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -322,9 +319,7 @@ mksection .text call ZUC256_INIT_8 %endif -%ifndef LINUX - add rsp, 32 -%endif + RESTORE_STACK_SPACE 4 cmp word [r12 + _zuc_init_not_done], 0xff ; Init done for all lanes je %%skip_submit_restoring_state @@ -380,11 +375,8 @@ mksection .text mov word [r12 + _zuc_init_not_done], 0 ; Init done for all lanes - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -393,9 +385,8 @@ mksection .text call asm_ZucCipher_8_avx2 -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -531,11 +522,8 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 32 bytes for 4 parameters - sub rsp, 32 -%endif + RESERVE_STACK_SPACE 4 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -551,9 +539,8 @@ APPEND(%%skip_eea3_,I): call ZUC256_INIT_8 %endif -%ifndef LINUX - add rsp, 32 -%endif + RESTORE_STACK_SPACE 4 + cmp word [r12 + _zuc_init_not_done], 0xff ; Init done for all lanes je %%skip_flush_restoring_state @@ -656,11 +643,8 @@ APPEND3(%%skip_eea3_copy_,I,J): %assign I (I+1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -669,9 +653,8 @@ APPEND3(%%skip_eea3_copy_,I,J): call asm_ZucCipher_8_avx2 -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] ; Clear ZUC state of the lane that is returned and NULL lanes @@ -866,11 +849,8 @@ FLUSH_JOB_ZUC256_EEA3: ; to pass parameter to next function mov r11, state - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 + lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -891,9 +871,8 @@ FLUSH_JOB_ZUC256_EEA3: call zuc256_eia3_8_buffer_job_avx2 %endif -%ifndef LINUX - add rsp, 48 -%endif + RESTORE_STACK_SPACE 6 + mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -1019,10 +998,8 @@ APPEND(%%skip_eia3_,I): ; to pass parameter to next function mov r11, state -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 + lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -1043,9 +1020,8 @@ APPEND(%%skip_eia3_,I): call zuc256_eia3_8_buffer_job_avx2 %endif -%ifndef LINUX - add rsp, 48 -%endif + RESTORE_STACK_SPACE 6 + vmovdqa xmm2, [rsp + _null_len_save] mov state, [rsp + _gpr_save + 8*8] diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index 9029e6b7..de7dd907 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -274,11 +274,8 @@ mksection .text mov word [r11 + _zuc_init_not_done], 0 ; Init done for all lanes - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r11 + _zuc_state] lea arg2, [r11 + _zuc_args_in] lea arg3, [r11 + _zuc_args_out] @@ -287,9 +284,8 @@ mksection .text call ZUC_CIPHER -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -488,11 +484,8 @@ mksection .text vpbroadcastd zmm0, DWORD(tmp4) vmovdqa32 [r12 + _zuc_state + OFS_R2]{k1}, zmm0 - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -501,9 +494,8 @@ mksection .text call ZUC_CIPHER -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] ; Prepare bitmask to clear ZUC state with lane @@ -628,10 +620,8 @@ FLUSH_JOB_ZUC256_EEA3: ; Generate L KS words (less than 16), except for old buffers, which only need L-2, ; since 2 words are reused from previous iteration -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [%%OOO + _zuc_state] lea arg2, [%%OOO + _zuc_args_KS] xor arg3, arg3 ; offset = 0 @@ -645,9 +635,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_KEYGEN_SKIP8_16 -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + jmp %%_exit %%_above_eq_16: @@ -690,10 +679,8 @@ FLUSH_JOB_ZUC256_EEA3: %%_above_eq_16_loop: ; Generate next 16 KS words and digest 64 bytes of data -%ifndef LINUX - ;; 48 bytes for 6 parameters - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 + mov DWORD(%%TMP), %%L shr DWORD(%%TMP), 4 ; Number of rounds of 64 bytes @@ -716,10 +703,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_EIA3_N64B -%ifndef LINUX - ;; 48 bytes for 6 parameters - add rsp, 48 -%endif + RESTORE_STACK_SPACE 6 + and %%L, 0xf ; Remaining words of KS left to generate jmp %%_loop @@ -736,10 +721,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_KEYGEN_16 %%_exit: -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + ; Digest final bytes of data and generate tag for finished buffers lea arg1, [%%OOO + _zuc_args_digest] lea arg2, [%%OOO + _zuc_args_KS] @@ -757,9 +740,8 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC256_REMAINDER_16 %endif -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov word [%%OOO + _zuc_init_not_done], 0 %endmacro diff --git a/lib/include/os.asm b/lib/include/os.asm index b0f293c3..90d1c5ab 100644 --- a/lib/include/os.asm +++ b/lib/include/os.asm @@ -76,4 +76,42 @@ %endif %endmacro +;; Macro to reserve stack space before function call, +;; based on number of arguments +%macro RESERVE_STACK_SPACE 1 +%define %%N_ARGS %1 ; [immediate] Number of arguments + +%ifdef LINUX +%if %%N_ARGS > 6 + sub rsp, 8*(%%N_ARGS - 6) +%endif +%else ; Windows +%if %%N_ARGS <= 4 + ; Reserve 32 bytes if number of arguments is <= 4 + sub rsp, 8*4 +%else + sub rsp, 8*%%N_ARGS +%endif +%endif ; LINUX +%endmacro + +;; Macro to restore stack pointer after function call, +;; based on number of arguments +%macro RESTORE_STACK_SPACE 1 +%define %%N_ARGS %1 ; [immediate] Number of arguments + +%ifdef LINUX +%if %%N_ARGS > 6 + add rsp, 8*(%%N_ARGS - 6) +%endif +%else ; Windows +%if %%N_ARGS <= 4 + ; Reserve 32 bytes if number of arguments is <= 4 + add rsp, 8*4 +%else + add rsp, 8*%%N_ARGS +%endif +%endif ; LINUX +%endmacro + %endif ; OS_ASM_FILE diff --git a/lib/sse/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse/mb_mgr_zuc_submit_flush_sse.asm index 0b060b27..731250e5 100644 --- a/lib/sse/mb_mgr_zuc_submit_flush_sse.asm +++ b/lib/sse/mb_mgr_zuc_submit_flush_sse.asm @@ -334,11 +334,8 @@ mksection .text %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 8*5 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -349,9 +346,7 @@ mksection .text call ZUC256_INIT_4 %endif -%ifndef LINUX - add rsp, 8*5 -%endif + RESTORE_STACK_SPACE 5 cmp byte [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_submit_restoring_state @@ -392,11 +387,8 @@ mksection .text %endif mov byte [r12 + _zuc_init_not_done], 0 ; Init done for all lanes - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -405,9 +397,8 @@ mksection .text call ZUC_CIPHER_4 -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -542,11 +533,8 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 8*5 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] @@ -558,9 +546,8 @@ APPEND(%%skip_eea3_,I): call ZUC256_INIT_4 %endif -%ifndef LINUX - add rsp, 8*5 -%endif + RESTORE_STACK_SPACE 5 + cmp word [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_flush_restoring_state @@ -619,11 +606,8 @@ APPEND3(%%skip_eea3_copy_,I,J): movdqa [r12 + _zuc_state + 16*I], xmm1 ; Save new state %assign I (I+1) %endrep - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 40 bytes for 5 parameters - sub rsp, 40 -%endif + RESERVE_STACK_SPACE 5 + lea arg1, [r12 + _zuc_state] lea arg2, [r12 + _zuc_args_in] lea arg3, [r12 + _zuc_args_out] @@ -632,9 +616,8 @@ APPEND3(%%skip_eea3_copy_,I,J): call ZUC_CIPHER_4 -%ifndef LINUX - add rsp, 40 -%endif + RESTORE_STACK_SPACE 5 + mov state, [rsp + _gpr_save + 8*8] ; Clear ZUC state of the lane that is returned and NULL lanes @@ -825,20 +808,10 @@ FLUSH_JOB_ZUC256_EEA3: mov r11, state %if %%KEY_SIZE == 128 - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 %else ; %%KEY_SIZE == 256 -%ifndef LINUX - ;; 56 bytes for 7 parameters - sub rsp, 8*7 -%else - ;; 8 bytes for one extra parameter (apart from first 6) - sub rsp, 8 + RESERVE_STACK_SPACE 7 %endif -%endif ;; %%KEY_SIZE lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] @@ -864,16 +837,10 @@ FLUSH_JOB_ZUC256_EEA3: %endif %if %%KEY_SIZE == 128 -%ifndef LINUX - add rsp, 48 -%endif -%else ;; %%KEY_SIZE == 256 -%ifndef LINUX - add rsp, 8*7 -%else - add rsp, 8 + RESTORE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 %endif -%endif ;; %%KEY_SIZE mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -1005,20 +972,11 @@ APPEND(%%skip_eia3_,I): mov r11, state %if %%KEY_SIZE == 128 - ;; If Windows, reserve memory in stack for parameter transferring -%ifndef LINUX - ;; 48 bytes for 6 parameters (already aligned to 16 bytes) - sub rsp, 48 -%endif + RESERVE_STACK_SPACE 6 %else ; %%KEY_SIZE == 256 -%ifndef LINUX - ;; 56 bytes for 7 parameters - sub rsp, 8*7 -%else - ;; 8 bytes for one extra parameter (apart from first 6) - sub rsp, 8 + RESERVE_STACK_SPACE 7 %endif -%endif ;; %%KEY_SIZE + lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -1043,16 +1001,10 @@ APPEND(%%skip_eia3_,I): %endif %if %%KEY_SIZE == 128 -%ifndef LINUX - add rsp, 48 -%endif -%else ;; %%KEY_SIZE == 256 -%ifndef LINUX - add rsp, 8*7 -%else - add rsp, 8 + RESTORE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 %endif -%endif ;; %%KEY_SIZE mov tmp5, [rsp + _null_len_save] mov state, [rsp + _gpr_save + 8*8] -- GitLab From 1deb393994b6012e76394b4b0d7f3ce2fe12ea35 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 15 Mar 2022 21:03:10 +0000 Subject: [PATCH 021/369] avx512: [AES-CBC] Encrypt flush optimization Avoid copying good lane data to NULL lanes by using zero k mask loads & stores for NULL lanes while processing --- lib/avx512/aes_cbc_enc_vaes_avx512.asm | 471 ++++++++++++++---- .../mb_mgr_aes128_cbc_enc_flush_avx512.asm | 52 +- .../mb_mgr_aes192_cbc_enc_flush_avx512.asm | 2 +- .../mb_mgr_aes256_cbc_enc_flush_avx512.asm | 2 +- 4 files changed, 398 insertions(+), 129 deletions(-) diff --git a/lib/avx512/aes_cbc_enc_vaes_avx512.asm b/lib/avx512/aes_cbc_enc_vaes_avx512.asm index baad52d8..2368ab14 100644 --- a/lib/avx512/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512/aes_cbc_enc_vaes_avx512.asm @@ -34,30 +34,36 @@ %include "include/cet.inc" struc STACK _gpr_save: resq 4 +_lane_masks: resw 16 endstruc %define GPR_SAVE_AREA rsp + _gpr_save +%define LANE_MASKS rsp + _lane_masks %ifdef LINUX %define arg1 rdi %define arg2 rsi -%define arg3 rcx -%define arg4 rdx +%define arg3 rdx +%define arg4 rcx +%define IA0 arg3 +%define IN arg4 %else %define arg1 rcx %define arg2 rdx -%define arg3 rdi -%define arg4 rsi +%define arg3 r8 +%define arg4 r9 +%define IA0 rsi +%define IN rdi %endif +%define IA1 rbx +%define IA2 rax +%define OUT rbp + +%define VALID_LANES k7 %define ARG arg1 %define LEN arg2 -%define IA0 rax -%define IA1 rbx -%define IA2 arg3 -%define IN arg4 -%define OUT rbp %define IN_L0 r8 %define IN_L1 r9 %define IN_L2 r10 @@ -108,6 +114,9 @@ endstruc %define MAC_TYPE_CBC 1 %define MAC_TYPE_XCBC 2 +%define SUBMIT 0 +%define FLUSH 1 + ;; Save registers states %macro FUNC_SAVE 0 sub rsp, STACK_size @@ -165,6 +174,104 @@ endstruc vshufi64x2 %%IN_OUT_3, %%ZTMP_2, %%ZTMP_3, 0xee %endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; LOAD_STORE_VAR_x4 - variable load/store of 0-4 blocks (16 bytes) for 4 lanes +; Number of blocks determined by masks in LANE_MASKS table on the stack +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro LOAD_STORE_VAR_x4 15 +%define %%LANE_A %1 ; [in] lane index to load/store (numerical) +%define %%LANE_B %2 ; [in] lane index to load/store (numerical) +%define %%LANE_C %3 ; [in] lane index to load/store (numerical) +%define %%LANE_D %4 ; [in] lane index to load/store (numerical) +%define %%DATA_PTR %5 ; [in] GP reg with ptr to lane input table +%define %%OFFSET %6 ; [in] GP reg input/output buffer offset +%define %%ZDATA0 %7 ; [in/out] ZMM reg to load/store data +%define %%ZDATA1 %8 ; [in/out] ZMM reg to load/store data +%define %%ZDATA2 %9 ; [in/out] ZMM reg to load/store data +%define %%ZDATA3 %10 ; [in/out] ZMM reg to load/store data +%define %%GP0 %11 ; [clobbered] tmp GP reg +%define %%GP1 %12 ; [clobbered] tmp GP reg +%define %%LOAD_STORE %13 ; [in] string value to select LOAD or STORE +%define %%M1 %14 ; [clobbered] mask reg +%define %%M2 %15 ; [clobbered] mask reg + + mov %%GP0, [%%DATA_PTR + 8*(%%LANE_A)] + mov %%GP1, [%%DATA_PTR + 8*(%%LANE_B)] + + kmovw %%M1, [LANE_MASKS + 2*(%%LANE_A)] + kmovw %%M2, [LANE_MASKS + 2*(%%LANE_B)] + +%ifidn %%LOAD_STORE, LOAD + vmovdqu32 %%ZDATA0{%%M1}{z}, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA1{%%M2}{z}, [%%GP1 + %%OFFSET] + + mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] + mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] + + kmovw %%M1, [LANE_MASKS + 2*(%%LANE_C)] + kmovw %%M2, [LANE_MASKS + 2*(%%LANE_D)] + + vmovdqu32 %%ZDATA2{%%M1}{z}, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA3{%%M2}{z}, [%%GP1 + %%OFFSET] +%else ; STORE + vmovdqu32 [%%GP0 + %%OFFSET]{%%M1}, %%ZDATA0 + vmovdqu32 [%%GP1 + %%OFFSET]{%%M2}, %%ZDATA1 + + mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] + mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] + + kmovw %%M1, [LANE_MASKS + 2*(%%LANE_C)] + kmovw %%M2, [LANE_MASKS + 2*(%%LANE_D)] + + vmovdqu32 [%%GP0 + %%OFFSET]{%%M1}, %%ZDATA2 + vmovdqu32 [%%GP1 + %%OFFSET]{%%M2}, %%ZDATA3 +%endif +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; PRELOADED_LOAD_STORE_VAR_x4 +; - Variable size load/store of 0-4 blocks for 4 lanes +; - Input pointers are already loaded into GP registers +; - Number of blocks determined by masks in LANE_MASKS table on the stack +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro PRELOADED_LOAD_STORE_VAR_x4 18 +%define %%IN0 %1 ; [in] GP reg with lane input pointer +%define %%IN1 %2 ; [in] GP reg with lane input pointer +%define %%IN2 %3 ; [in] GP reg with lane input pointer +%define %%IN3 %4 ; [in] GP reg with lane input pointer +%define %%OFFSET %5 ; [in] GP reg input/output buffer offset +%define %%ZDATA0 %6 ; [in/out] ZMM reg to load/store data +%define %%ZDATA1 %7 ; [in/out] ZMM reg to load/store data +%define %%ZDATA2 %8 ; [in/out] ZMM reg to load/store data +%define %%ZDATA3 %9 ; [in/out] ZMM reg to load/store data +%define %%LOAD_STORE %10 ; [in] string value to select LOAD or STORE +%define %%LANE_A %11 ; [in] lane ID +%define %%LANE_B %12 ; [in] lane ID +%define %%LANE_C %13 ; [in] lane ID +%define %%LANE_D %14 ; [in] lane ID +%define %%M1 %15 ; [clobbered] mask reg +%define %%M2 %16 ; [clobbered] mask reg +%define %%M3 %17 ; [clobbered] mask reg +%define %%M4 %18 ; [clobbered] mask reg + + kmovw %%M1, [LANE_MASKS + 2*(%%LANE_A)] + kmovw %%M2, [LANE_MASKS + 2*(%%LANE_B)] + kmovw %%M3, [LANE_MASKS + 2*(%%LANE_C)] + kmovw %%M4, [LANE_MASKS + 2*(%%LANE_D)] + +%ifidn %%LOAD_STORE, LOAD + vmovdqu32 %%ZDATA0{%%M1}{z}, [%%IN0 + %%OFFSET] + vmovdqu32 %%ZDATA1{%%M2}{z}, [%%IN1 + %%OFFSET] + vmovdqu32 %%ZDATA2{%%M3}{z}, [%%IN2 + %%OFFSET] + vmovdqu32 %%ZDATA3{%%M4}{z}, [%%IN3 + %%OFFSET] +%else ; STORE + vmovdqu32 [%%IN0 + %%OFFSET]{%%M1}, %%ZDATA0 + vmovdqu32 [%%IN1 + %%OFFSET]{%%M2}, %%ZDATA1 + vmovdqu32 [%%IN2 + %%OFFSET]{%%M3}, %%ZDATA2 + vmovdqu32 [%%IN3 + %%OFFSET]{%%M4}, %%ZDATA3 +%endif +%endmacro + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; LOAD_STORE - loads/stores 1-4 blocks (16 bytes) for 4 lanes into ZMM registers ; - Loads 4 blocks by default @@ -192,43 +299,43 @@ endstruc %if %%NUM_ARGS <= 13 ;; %%MASK_REG not set, assume 4 block load/store %ifidn %%LOAD_STORE, LOAD - vmovdqu8 %%ZDATA0, [%%GP0 + %%OFFSET] - vmovdqu8 %%ZDATA1, [%%GP1 + %%OFFSET] + vmovdqu32 %%ZDATA0, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA1, [%%GP1 + %%OFFSET] mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] - vmovdqu8 %%ZDATA2, [%%GP0 + %%OFFSET] - vmovdqu8 %%ZDATA3, [%%GP1 + %%OFFSET] + vmovdqu32 %%ZDATA2, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA3, [%%GP1 + %%OFFSET] %else ; STORE - vmovdqu8 [%%GP0 + %%OFFSET], %%ZDATA0 - vmovdqu8 [%%GP1 + %%OFFSET], %%ZDATA1 + vmovdqu32 [%%GP0 + %%OFFSET], %%ZDATA0 + vmovdqu32 [%%GP1 + %%OFFSET], %%ZDATA1 mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] - vmovdqu8 [%%GP0 + %%OFFSET], %%ZDATA2 - vmovdqu8 [%%GP1 + %%OFFSET], %%ZDATA3 + vmovdqu32 [%%GP0 + %%OFFSET], %%ZDATA2 + vmovdqu32 [%%GP1 + %%OFFSET], %%ZDATA3 %endif %else ;; %%MASK_REG argument passed - 1, 2, or 3 block load/store %ifidn %%LOAD_STORE, LOAD - vmovdqu8 %%ZDATA0{%%MASK_REG}{z}, [%%GP0 + %%OFFSET] - vmovdqu8 %%ZDATA1{%%MASK_REG}{z}, [%%GP1 + %%OFFSET] + vmovdqu32 %%ZDATA0{%%MASK_REG}{z}, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA1{%%MASK_REG}{z}, [%%GP1 + %%OFFSET] mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] - vmovdqu8 %%ZDATA2{%%MASK_REG}{z}, [%%GP0 + %%OFFSET] - vmovdqu8 %%ZDATA3{%%MASK_REG}{z}, [%%GP1 + %%OFFSET] + vmovdqu32 %%ZDATA2{%%MASK_REG}{z}, [%%GP0 + %%OFFSET] + vmovdqu32 %%ZDATA3{%%MASK_REG}{z}, [%%GP1 + %%OFFSET] %else ; STORE - vmovdqu8 [%%GP0 + %%OFFSET]{%%MASK_REG}, %%ZDATA0 - vmovdqu8 [%%GP1 + %%OFFSET]{%%MASK_REG}, %%ZDATA1 + vmovdqu32 [%%GP0 + %%OFFSET]{%%MASK_REG}, %%ZDATA0 + vmovdqu32 [%%GP1 + %%OFFSET]{%%MASK_REG}, %%ZDATA1 mov %%GP0, [%%DATA_PTR + 8*(%%LANE_C)] mov %%GP1, [%%DATA_PTR + 8*(%%LANE_D)] - vmovdqu8 [%%GP0 + %%OFFSET]{%%MASK_REG}, %%ZDATA2 - vmovdqu8 [%%GP1 + %%OFFSET]{%%MASK_REG}, %%ZDATA3 + vmovdqu32 [%%GP0 + %%OFFSET]{%%MASK_REG}, %%ZDATA2 + vmovdqu32 [%%GP1 + %%OFFSET]{%%MASK_REG}, %%ZDATA3 %endif %endif ;; %%NUM_ARGS %endmacro @@ -255,31 +362,102 @@ endstruc %if %%NUM_ARGS <= 10 ;; %%MASK_REG not set, assume 4 block load/store %ifidn %%LOAD_STORE, LOAD - vmovdqu8 %%ZDATA0, [%%IN0 + %%OFFSET] - vmovdqu8 %%ZDATA1, [%%IN1 + %%OFFSET] - vmovdqu8 %%ZDATA2, [%%IN2 + %%OFFSET] - vmovdqu8 %%ZDATA3, [%%IN3 + %%OFFSET] + vmovdqu32 %%ZDATA0, [%%IN0 + %%OFFSET] + vmovdqu32 %%ZDATA1, [%%IN1 + %%OFFSET] + vmovdqu32 %%ZDATA2, [%%IN2 + %%OFFSET] + vmovdqu32 %%ZDATA3, [%%IN3 + %%OFFSET] %else ; STORE - vmovdqu8 [%%IN0 + %%OFFSET], %%ZDATA0 - vmovdqu8 [%%IN1 + %%OFFSET], %%ZDATA1 - vmovdqu8 [%%IN2 + %%OFFSET], %%ZDATA2 - vmovdqu8 [%%IN3 + %%OFFSET], %%ZDATA3 + vmovdqu32 [%%IN0 + %%OFFSET], %%ZDATA0 + vmovdqu32 [%%IN1 + %%OFFSET], %%ZDATA1 + vmovdqu32 [%%IN2 + %%OFFSET], %%ZDATA2 + vmovdqu32 [%%IN3 + %%OFFSET], %%ZDATA3 %endif %else ;; %%MASK_REG argument passed - 1, 2, or 3 block load/store %ifidn %%LOAD_STORE, LOAD - vmovdqu8 %%ZDATA0{%%MASK_REG}{z}, [%%IN0 + %%OFFSET] - vmovdqu8 %%ZDATA1{%%MASK_REG}{z}, [%%IN1 + %%OFFSET] - vmovdqu8 %%ZDATA2{%%MASK_REG}{z}, [%%IN2 + %%OFFSET] - vmovdqu8 %%ZDATA3{%%MASK_REG}{z}, [%%IN3 + %%OFFSET] + vmovdqu32 %%ZDATA0{%%MASK_REG}{z}, [%%IN0 + %%OFFSET] + vmovdqu32 %%ZDATA1{%%MASK_REG}{z}, [%%IN1 + %%OFFSET] + vmovdqu32 %%ZDATA2{%%MASK_REG}{z}, [%%IN2 + %%OFFSET] + vmovdqu32 %%ZDATA3{%%MASK_REG}{z}, [%%IN3 + %%OFFSET] %else ; STORE - vmovdqu8 [%%IN0 + %%OFFSET]{%%MASK_REG}, %%ZDATA0 - vmovdqu8 [%%IN1 + %%OFFSET]{%%MASK_REG}, %%ZDATA1 - vmovdqu8 [%%IN2 + %%OFFSET]{%%MASK_REG}, %%ZDATA2 - vmovdqu8 [%%IN3 + %%OFFSET]{%%MASK_REG}, %%ZDATA3 + vmovdqu32 [%%IN0 + %%OFFSET]{%%MASK_REG}, %%ZDATA0 + vmovdqu32 [%%IN1 + %%OFFSET]{%%MASK_REG}, %%ZDATA1 + vmovdqu32 [%%IN2 + %%OFFSET]{%%MASK_REG}, %%ZDATA2 + vmovdqu32 [%%IN3 + %%OFFSET]{%%MASK_REG}, %%ZDATA3 %endif %endif ;; %%NUM_ARGS %endmacro +;; LOAD_STORE wrapper used to select load/store operation mode +%macro LOAD_STORE_4 14 +%define %%LANE_A %1 ; [in] lane index to load/store (numerical) +%define %%LANE_B %2 ; [in] lane index to load/store (numerical) +%define %%LANE_C %3 ; [in] lane index to load/store (numerical) +%define %%LANE_D %4 ; [in] lane index to load/store (numerical) +%define %%DATA_PTR %5 ; [in] GP reg with ptr to lane input table +%define %%OFFSET %6 ; [in] GP reg input/output buffer offset +%define %%ZDATA0 %7 ; [in/out] ZMM reg to load/store data +%define %%ZDATA1 %8 ; [in/out] ZMM reg to load/store data +%define %%ZDATA2 %9 ; [in/out] ZMM reg to load/store data +%define %%ZDATA3 %10 ; [in/out] ZMM reg to load/store data +%define %%GP0 %11 ; [clobbered] tmp GP reg +%define %%GP1 %12 ; [clobbered] tmp GP reg +%define %%LOAD_STORE %13 ; [in] string value to select LOAD or STORE +%define %%MODE %14 ; [in] load/store operation mode + +%ifidn %%MODE, SUBMIT_MODE + LOAD_STORE_x4 %%LANE_A, %%LANE_B, %%LANE_C, %%LANE_D, %%DATA_PTR, \ + %%OFFSET, %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%GP0, %%GP1, %%LOAD_STORE +%endif + +%ifidn %%MODE, SUBMIT_FINAL_MODE + LOAD_STORE_x4 %%LANE_A, %%LANE_B, %%LANE_C, %%LANE_D, %%DATA_PTR, \ + %%OFFSET, %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%GP0, %%GP1, %%LOAD_STORE, k1 +%endif +%ifidn %%MODE, FLUSH_MODE + LOAD_STORE_VAR_x4 %%LANE_A, %%LANE_B, %%LANE_C, %%LANE_D, %%DATA_PTR, \ + %%OFFSET, %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%GP0, %%GP1, %%LOAD_STORE, k1, k2 +%endif +%endmacro + +;; PRELOADED_LOAD_STORE wrapper used to select load/store operation mode +%macro PRELOADED_LOAD_STORE_4 11-15 +%define %%IN0 %1 ; [in] GP reg with lane input pointer +%define %%IN1 %2 ; [in] GP reg with lane input pointer +%define %%IN2 %3 ; [in] GP reg with lane input pointer +%define %%IN3 %4 ; [in] GP reg with lane input pointer +%define %%OFFSET %5 ; [in] GP reg input/output buffer offset +%define %%ZDATA0 %6 ; [in/out] ZMM reg to load/store data +%define %%ZDATA1 %7 ; [in/out] ZMM reg to load/store data +%define %%ZDATA2 %8 ; [in/out] ZMM reg to load/store data +%define %%ZDATA3 %9 ; [in/out] ZMM reg to load/store data +%define %%LOAD_STORE %10 ; [in] string value to select LOAD or STORE +%define %%MODE %11 ; [in] load/store operation mode +%define %%LANE_A %12 ; [in] lane ID +%define %%LANE_B %13 ; [in] lane ID +%define %%LANE_C %14 ; [in] lane ID +%define %%LANE_D %15 ; [in] lane ID + +%ifidn %%MODE, SUBMIT_MODE + PRELOADED_LOAD_STORE_x4 %%IN0, %%IN1, %%IN2, %%IN3, %%OFFSET, \ + %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%LOAD_STORE +%endif +%ifidn %%MODE, SUBMIT_FINAL_MODE + PRELOADED_LOAD_STORE_x4 %%IN0, %%IN1, %%IN2, %%IN3, %%OFFSET, \ + %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%LOAD_STORE, k1 +%endif +%ifidn %%MODE, FLUSH_MODE + PRELOADED_LOAD_STORE_VAR_x4 %%IN0, %%IN1, %%IN2, %%IN3, %%OFFSET, \ + %%ZDATA0, %%ZDATA1, %%ZDATA2, %%ZDATA3, \ + %%LOAD_STORE, %%LANE_A, %%LANE_B, \ + %%LANE_C, %%LANE_D, k1, k2, k3, k4 +%endif +%endmacro + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; AESENC_ROUNDS_x16 macro ; - 16 lanes, 1 block per lane @@ -335,7 +513,7 @@ endstruc ; - each loop encrypts 4 blocks across 16 lanes ; - stop when %%LENGTH is less than 64 bytes (4 blocks) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro ENCRYPT_16_PARALLEL 30 +%macro ENCRYPT_16_PARALLEL 31 %define %%ZIV00_03 %1 ;; [in] lane 0-3 IVs %define %%ZIV04_07 %2 ;; [in] lane 4-7 IVs %define %%ZIV08_11 %3 ;; [in] lane 8-11 IVs @@ -365,7 +543,8 @@ endstruc %define %%ZTMP3 %27 ;; [clobbered] tmp ZMM register %define %%TMP0 %28 ;; [clobbered] tmp GP register %define %%TMP1 %29 ;; [clobbered] tmp GP register -%define %%MAC_TYPE %30 ;; MAC_TYPE_NONE/CBC/XCBC flag +%define %%MAC_TYPE %30 ;; [in] MAC_TYPE_NONE/CBC/XCBC flag +%define %%SUBMIT_FLUSH %31 ;; [in] SUBMIT/FLUSH flag %if %%MAC_TYPE == MAC_TYPE_XCBC %define %%KP ARG + _aes_xcbc_args_key_tab @@ -381,6 +560,17 @@ endstruc cmp %%LENGTH, 64 jl %%encrypt_16_done +%if %%SUBMIT_FLUSH == FLUSH +%define %%MODE FLUSH_MODE + + ;; update lane mask table + mov WORD(%%TMP0), 0xffff + vpbroadcastw YWORD(%%ZTMP0){VALID_LANES}{z}, WORD(%%TMP0) + vmovdqu16 [LANE_MASKS], YWORD(%%ZTMP0) + +%else +%define %%MODE SUBMIT_MODE +%endif xor %%IDX, %%IDX ;; skip length check on first loop jmp %%encrypt_16_first @@ -390,32 +580,33 @@ endstruc jl %%encrypt_16_end %%encrypt_16_first: + ;; load 4 plaintext blocks for lanes 0-3 - PRELOADED_LOAD_STORE_x4 IN_L0, IN_L1, IN_L2, IN_L3, %%IDX, \ - %%B0L00_03, %%B1L00_03, %%B2L00_03, \ - %%B3L00_03, LOAD + PRELOADED_LOAD_STORE_4 IN_L0, IN_L1, IN_L2, IN_L3, %%IDX, \ + %%B0L00_03, %%B1L00_03, %%B2L00_03, \ + %%B3L00_03, LOAD, %%MODE, 0, 1, 2, 3 TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 4-7 - LOAD_STORE_x4 4, 5, 6, 7, IN, %%IDX, %%B0L04_07, %%B1L04_07, \ - %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, LOAD + LOAD_STORE_4 4, 5, 6, 7, IN, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, LOAD, %%MODE TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 8-11 - PRELOADED_LOAD_STORE_x4 IN_L8, IN_L9, IN_L10, IN_L11, %%IDX, \ + PRELOADED_LOAD_STORE_4 IN_L8, IN_L9, IN_L10, IN_L11, %%IDX, \ %%B0L08_11, %%B1L08_11, %%B2L08_11, \ - %%B3L08_11, LOAD + %%B3L08_11, LOAD, %%MODE, 8, 9, 10, 11 TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 12-15 - LOAD_STORE_x4 12, 13, 14, 15, IN, %%IDX, %%B0L12_15, %%B1L12_15, \ - %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, LOAD + LOAD_STORE_4 12, 13, 14, 15, IN, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, LOAD, %%MODE TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 @@ -468,29 +659,29 @@ endstruc TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 0, 1, 2, 3, OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ - %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, STORE + LOAD_STORE_4 0, 1, 2, 3, OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 4-7 TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 4, 5, 6, 7, OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ - %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, STORE + LOAD_STORE_4 4, 5, 6, 7, OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 8-11 TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 8, 9, 10, 11, OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ - %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, STORE + LOAD_STORE_4 8, 9, 10, 11, OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 12-15 TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 12, 13, 14, 15, OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ - %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, STORE + LOAD_STORE_4 12, 13, 14, 15, OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, STORE, %%MODE %endif ;; MAC_TYPE sub %%LENGTH, 64 @@ -522,10 +713,11 @@ endstruc %%encrypt_16_done: %endmacro + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; ENCRYPT_16_FINAL Encodes final blocks (less than 4) across 16 lanes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro ENCRYPT_16_FINAL 30 +%macro ENCRYPT_16_FINAL 31 %define %%ZIV00_03 %1 ;; [in] lane 0-3 IVs %define %%ZIV04_07 %2 ;; [in] lane 4-7 IVs %define %%ZIV08_11 %3 ;; [in] lane 8-11 IVs @@ -556,6 +748,8 @@ endstruc %define %%TMP1 %28 ;; [clobbered] tmp GP register %define %%NUM_BLKS %29 ;; [in] number of blocks (numerical value) %define %%MAC_TYPE %30 ;; MAC_TYPE_NONE/CBC/XCBC flag +%define %%SUBMIT_FLUSH %31 ;; SUBMIT/FLUSH flag + %if %%MAC_TYPE == MAC_TYPE_XCBC %define %%KP ARG + _aesxcbcarg_key_tab @@ -568,43 +762,53 @@ endstruc %define %%K12_15_OFFSET 192 %if %%NUM_BLKS == 1 - mov %%TMP0, 0x0000_0000_0000_ffff - kmovq k1, %%TMP0 + mov DWORD(%%TMP0), 0xf %elif %%NUM_BLKS == 2 - mov %%TMP0, 0x0000_0000_ffff_ffff - kmovq k1, %%TMP0 + mov DWORD(%%TMP0), 0xff %elif %%NUM_BLKS == 3 - mov %%TMP0, 0x0000_ffff_ffff_ffff - kmovq k1, %%TMP0 + mov DWORD(%%TMP0), 0xfff +%endif + +%if %%SUBMIT_FLUSH == FLUSH +%define %%MODE FLUSH_MODE + + ;; update lane mask table + vpbroadcastw YWORD(%%ZTMP0){VALID_LANES}{z}, WORD(%%TMP0) + vmovdqu16 [LANE_MASKS], YWORD(%%ZTMP0) + +%else +%define %%MODE SUBMIT_FINAL_MODE + kmovw k1, DWORD(%%TMP0) + %endif xor %%IDX, %%IDX ;; load 4 plaintext blocks for lanes 0-3 - PRELOADED_LOAD_STORE_x4 IN_L0, IN_L1, IN_L2, IN_L3, %%IDX, \ + PRELOADED_LOAD_STORE_4 IN_L0, IN_L1, IN_L2, IN_L3, %%IDX, \ %%B0L00_03, %%B1L00_03, %%B2L00_03, \ - %%B3L00_03, LOAD, k1 + %%B3L00_03, LOAD, %%MODE, 0, 1, 2, 3 TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 4-7 - LOAD_STORE_x4 4, 5, 6, 7, IN, %%IDX, %%B0L04_07, %%B1L04_07, \ - %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, LOAD, k1 + LOAD_STORE_4 4, 5, 6, 7, IN, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, LOAD, %%MODE TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 8-11 - PRELOADED_LOAD_STORE_x4 IN_L8, IN_L9, IN_L10, IN_L11, %%IDX, \ + PRELOADED_LOAD_STORE_4 IN_L8, IN_L9, IN_L10, IN_L11, %%IDX, \ %%B0L08_11, %%B1L08_11, %%B2L08_11, \ - %%B3L08_11, LOAD, k1 + %%B3L08_11, LOAD, %%MODE, 8, 9, 10, 11 TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 ;; load 4 plaintext blocks for lanes 12-15 - LOAD_STORE_x4 12, 13, 14, 15, IN, %%IDX, %%B0L12_15, %%B1L12_15, \ - %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, LOAD, k1 + LOAD_STORE_4 12, 13, 14, 15, IN, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, LOAD, %%MODE TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 @@ -667,29 +871,29 @@ endstruc TRANSPOSE_4x4 %%B0L00_03, %%B1L00_03, %%B2L00_03, %%B3L00_03, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 0, 1, 2, 3, OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ - %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, STORE, k1 + LOAD_STORE_4 0, 1, 2, 3, OUT, %%IDX, %%B0L00_03, %%B1L00_03, \ + %%B2L00_03, %%B3L00_03, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 4-7 TRANSPOSE_4x4 %%B0L04_07, %%B1L04_07, %%B2L04_07, %%B3L04_07, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 4, 5, 6, 7, OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ - %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, STORE, k1 + LOAD_STORE_4 4, 5, 6, 7, OUT, %%IDX, %%B0L04_07, %%B1L04_07, \ + %%B2L04_07, %%B3L04_07, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 8-11 TRANSPOSE_4x4 %%B0L08_11, %%B1L08_11, %%B2L08_11, %%B3L08_11, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 8, 9, 10, 11, OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ - %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, STORE, k1 + LOAD_STORE_4 8, 9, 10, 11, OUT, %%IDX, %%B0L08_11, %%B1L08_11, \ + %%B2L08_11, %%B3L08_11, %%TMP0, %%TMP1, STORE, %%MODE ;; write back cipher text for lanes 12-15 TRANSPOSE_4x4 %%B0L12_15, %%B1L12_15, %%B2L12_15, %%B3L12_15, \ %%ZTMP0, %%ZTMP1, %%ZTMP2, %%ZTMP3 - LOAD_STORE_x4 12, 13, 14, 15, OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ - %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, STORE, k1 + LOAD_STORE_4 12, 13, 14, 15, OUT, %%IDX, %%B0L12_15, %%B1L12_15, \ + %%B2L12_15, %%B3L12_15, %%TMP0, %%TMP1, STORE, %%MODE %endif ;; !CBC_MAC ;; update in/out pointers @@ -716,9 +920,10 @@ endstruc ; First encrypts block up to multiple of 4 ; Then encrypts final blocks (less than 4) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro CBC_ENC 2 +%macro CBC_ENC 3 %define %%ROUNDS %1 %define %%MAC_TYPE %2 +%define %%SUBMIT_FLUSH %3 %define %%K00_03_OFFSET 0 %define %%K04_07_OFFSET 64 @@ -735,6 +940,11 @@ endstruc %define %%OUT ARG + _aesarg_out %endif +;; check if flush +%if %%SUBMIT_FLUSH == FLUSH + kmovw VALID_LANES, DWORD(arg3) ;; store valid lane mask +%endif + ;; load transpose tables vmovdqa64 TAB_A0B0A1B1, [rel A0B0A1B1] vmovdqa64 TAB_A2B2A3B3, [rel A2B2A3B3] @@ -770,7 +980,8 @@ endstruc ENCRYPT_16_PARALLEL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ LEN, %%ROUNDS, IA0, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, \ ZT6, ZT7, ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, \ - ZT15, ZT16, ZT17, ZT18, ZT19, IA1, IA2, %%MAC_TYPE + ZT15, ZT16, ZT17, ZT18, ZT19, IA1, IA2, \ + %%MAC_TYPE, %%SUBMIT_FLUSH ;; get num remaining blocks shr LEN, 4 @@ -785,19 +996,19 @@ endstruc ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ %%ROUNDS, IA0, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ - ZT18, ZT19, IA1, IA2, 3, %%MAC_TYPE - jmp %%_cbc_enc_done + ZT18, ZT19, IA1, IA2, 3, %%MAC_TYPE, %%SUBMIT_FLUSH + jmp %%_cbc_enc_done %%_final_blocks_1: ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ %%ROUNDS, IA0, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ - ZT18, ZT19, IA1, IA2, 1, %%MAC_TYPE - jmp %%_cbc_enc_done + ZT18, ZT19, IA1, IA2, 1, %%MAC_TYPE, %%SUBMIT_FLUSH + jmp %%_cbc_enc_done %%_final_blocks_2: ENCRYPT_16_FINAL ZIV00_03, ZIV04_07, ZIV08_11, ZIV12_15, \ %%ROUNDS, IA0, ZT0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, \ ZT8, ZT9, ZT10, ZT11, ZT12, ZT13, ZT14, ZT15, ZT16, ZT17, \ - ZT18, ZT19, IA1, IA2, 2, %%MAC_TYPE + ZT18, ZT19, IA1, IA2, 2, %%MAC_TYPE, %%SUBMIT_FLUSH %%_cbc_enc_done: ;; store IV's per lane vmovdqa64 [%%IV + 16*0], ZIV00_03 @@ -829,7 +1040,7 @@ MKGLOBAL(aes_cbc_enc_128_vaes_avx512,function,internal) aes_cbc_enc_128_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 9, MAC_TYPE_NONE + CBC_ENC 9, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE %ifdef SAFE_DATA @@ -847,7 +1058,7 @@ MKGLOBAL(aes_cbc_enc_192_vaes_avx512,function,internal) aes_cbc_enc_192_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 11, MAC_TYPE_NONE + CBC_ENC 11, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE %ifdef SAFE_DATA @@ -857,6 +1068,7 @@ aes_cbc_enc_192_vaes_avx512: %endif ;; SAFE_DATA ret + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; void aes_cbc_enc_256_vaes_avx512(AES_ARGS *args, uint64_t len_in_bytes); ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -864,8 +1076,15 @@ MKGLOBAL(aes_cbc_enc_256_vaes_avx512,function,internal) aes_cbc_enc_256_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 13, MAC_TYPE_NONE + CBC_ENC 13, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -875,7 +1094,7 @@ MKGLOBAL(aes128_cbc_mac_vaes_avx512,function,internal) aes128_cbc_mac_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 9, MAC_TYPE_CBC + CBC_ENC 9, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE %ifdef SAFE_DATA @@ -893,7 +1112,7 @@ MKGLOBAL(aes256_cbc_mac_vaes_avx512,function,internal) aes256_cbc_mac_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 13, MAC_TYPE_CBC + CBC_ENC 13, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE %ifdef SAFE_DATA @@ -911,7 +1130,67 @@ MKGLOBAL(aes_xcbc_mac_128_vaes_avx512,function,internal) aes_xcbc_mac_128_vaes_avx512: endbranch64 FUNC_SAVE - CBC_ENC 9, MAC_TYPE_XCBC + CBC_ENC 9, MAC_TYPE_XCBC, SUBMIT + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_128_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_128_flush_vaes_avx512,function,internal) +aes_cbc_enc_128_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 9, MAC_TYPE_NONE, FLUSH + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_192_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_192_flush_vaes_avx512,function,internal) +aes_cbc_enc_192_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 11, MAC_TYPE_NONE, FLUSH + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_cbc_enc_256_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_cbc_enc_256_flush_vaes_avx512,function,internal) +aes_cbc_enc_256_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 13, MAC_TYPE_NONE, FLUSH FUNC_RESTORE %ifdef SAFE_DATA diff --git a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm index 1c50bd3d..c2f79180 100644 --- a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -32,7 +32,7 @@ %include "include/reg_sizes.asm" %include "include/cet.inc" %ifndef AES_CBC_ENC_X16 -%define AES_CBC_ENC_X16 aes_cbc_enc_128_vaes_avx512 +%define AES_CBC_ENC_X16 aes_cbc_enc_128_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes128_enc_vaes_avx512 %define NUM_KEYS 11 %endif @@ -47,9 +47,11 @@ mksection .text %ifdef LINUX %define arg1 rdi %define arg2 rsi +%define arg3 rdx %else %define arg1 rcx %define arg2 rdx +%define arg3 r8 %endif %define state arg1 @@ -62,7 +64,6 @@ mksection .text %define unused_lanes rbx %define tmp1 rbx -%define good_lane rdx %define iv rdx %define tmp2 rax @@ -71,8 +72,8 @@ mksection .text %define tmp rbp %define idx rbp -%define tmp3 r8 -%define tmp4 r9 +%define tmp3 r9 +%define tmp4 r10 %endif ; copy IV's and round keys into NULL lanes @@ -180,31 +181,22 @@ FLUSH_JOB_AES_ENC: vmovdqu64 zmm2, [state + _aes_job_in_lane + (8*PTR_SZ)] vpcmpq k1, zmm1, zmm0, 4 ; NEQ vpcmpq k2, zmm2, zmm0, 4 ; NEQ - kmovw DWORD(tmp), k1 - kmovw DWORD(tmp1), k2 - mov DWORD(tmp2), DWORD(tmp1) - shl DWORD(tmp2), 8 - or DWORD(tmp2), DWORD(tmp) ; mask of non-null jobs in tmp2 - not BYTE(tmp) - kmovw k4, DWORD(tmp) - not BYTE(tmp1) - kmovw k5, DWORD(tmp1) - mov DWORD(tmp), DWORD(tmp2) - not WORD(tmp) - kmovw k6, DWORD(tmp) ; mask of NULL jobs in k4, k5 and k6 - mov DWORD(tmp), DWORD(tmp2) + kshiftlw k2, k2, 8 + korw k6, k2, k1 + kmovw DWORD(arg3), k6 ; mask of non-null lanes in arg3 + knotw k6, k6 + kshiftrw k5, k6, 8 ; mask of NULL jobs in k4, k5 and k6 + xor tmp2, tmp2 - bsf WORD(tmp2), WORD(tmp) ; index of the 1st set bit in tmp2 + bsf WORD(tmp2), WORD(arg3) ; index of the 1st set bit in tmp2 - ;; copy good lane data into NULL lanes - mov tmp, [state + _aes_args_in + tmp2*8] - vpbroadcastq zmm1, tmp - vmovdqa64 [state + _aes_args_in + (0*PTR_SZ)]{k4}, zmm1 - vmovdqa64 [state + _aes_args_in + (8*PTR_SZ)]{k5}, zmm1 - ;; - out pointer + ;; copy good lane output pointer into NULL lanes in & out + ;; NOTE: NULL lanes not updated so any valid address can be used mov tmp, [state + _aes_args_out + tmp2*8] vpbroadcastq zmm1, tmp - vmovdqa64 [state + _aes_args_out + (0*PTR_SZ)]{k4}, zmm1 + vmovdqa64 [state + _aes_args_in + (0*PTR_SZ)]{k6}, zmm1 + vmovdqa64 [state + _aes_args_in + (8*PTR_SZ)]{k5}, zmm1 + vmovdqa64 [state + _aes_args_out + (0*PTR_SZ)]{k6}, zmm1 vmovdqa64 [state + _aes_args_out + (8*PTR_SZ)]{k5}, zmm1 ;; - set len to UINT16_MAX @@ -217,16 +209,10 @@ FLUSH_JOB_AES_ENC: ;; Find min length for lanes 0-7 vphminposuw xmm2, xmm0 - ;; scale up good lane idx before copying IV and keys - shl tmp2, 4 - ; extract min length of lanes 0-7 vpextrw DWORD(len2), xmm2, 0 ; min value vpextrw DWORD(idx), xmm2, 1 ; min index - ;; - copy IV and round keys to null lanes - COPY_IV_KEYS_TO_NULL_LANES tmp2, tmp1, tmp3, xmm4, xmm5, k6 - ;; Update lens and find min for lanes 8-15 vextracti128 xmm1, ymm0, 1 vphminposuw xmm2, xmm1 @@ -237,12 +223,16 @@ FLUSH_JOB_AES_ENC: add DWORD(idx), 8 ; but index +8 mov len2, tmp3 ; min len use_min: + or len2, len2 + je len_is_0 + vpbroadcastw ymm3, WORD(len2) vpsubw ymm0, ymm0, ymm3 vmovdqa [state + _aes_lens], ymm0 ; "state" and "args" are the same address, arg1 ; len is arg2 + ; valid lane mask is arg3 call AES_CBC_ENC_X16 ; state and idx are intact diff --git a/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm b/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm index dd6d9e0a..79a9998b 100644 --- a/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm +++ b/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%define AES_CBC_ENC_X16 aes_cbc_enc_192_vaes_avx512 +%define AES_CBC_ENC_X16 aes_cbc_enc_192_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_vaes_avx512 %define NUM_KEYS 13 %include "avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm b/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm index 3a057bf7..7425217c 100644 --- a/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm +++ b/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -%define AES_CBC_ENC_X16 aes_cbc_enc_256_vaes_avx512 +%define AES_CBC_ENC_X16 aes_cbc_enc_256_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_vaes_avx512 %define NUM_KEYS 15 %include "avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm" -- GitLab From f52a770d9f11c3f6d4cbb8804fddd5f25599f44f Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 16 Mar 2022 11:38:39 +0000 Subject: [PATCH 022/369] avx512: [AES-CMAC] Optimize flush to avoid copying lane data --- lib/avx512/aes_cbc_enc_vaes_avx512.asm | 40 ++++++++ ...s128_cmac_submit_flush_x16_vaes_avx512.asm | 91 +++++-------------- ...s256_cmac_submit_flush_x16_vaes_avx512.asm | 1 + 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/lib/avx512/aes_cbc_enc_vaes_avx512.asm b/lib/avx512/aes_cbc_enc_vaes_avx512.asm index 2368ab14..bf89fa6d 100644 --- a/lib/avx512/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512/aes_cbc_enc_vaes_avx512.asm @@ -1201,4 +1201,44 @@ aes_cbc_enc_256_flush_vaes_avx512: ret +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes128_cbc_mac_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes128_cbc_mac_flush_vaes_avx512,function,internal) +aes128_cbc_mac_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 9, MAC_TYPE_CBC, FLUSH + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes256_cbc_mac_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes256_cbc_mac_flush_vaes_avx512,function,internal) +aes256_cbc_mac_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 13, MAC_TYPE_CBC, FLUSH + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + mksection stack-noexec diff --git a/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm index 156b6c70..a65c40d5 100644 --- a/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm @@ -34,12 +34,14 @@ %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_vaes_avx512 +%define AES_CBC_MAC_FLUSH aes128_cbc_mac_flush_vaes_avx512 %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes128_cmac_auth_vaes_avx512 %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes128_cmac_auth_vaes_avx512 %define NUM_KEYS 11 %endif extern AES_CBC_MAC +extern AES_CBC_MAC_FLUSH mksection .rodata default rel @@ -51,9 +53,11 @@ mksection .text %ifdef LINUX %define arg1 rdi %define arg2 rsi +%define arg3 rdx %else %define arg1 rcx %define arg2 rdx +%define arg3 r8 %endif %define state arg1 @@ -148,42 +152,6 @@ endstruc %endif %endmacro -; copy IV's and round keys into NULL lanes -%macro COPY_IV_KEYS_TO_NULL_LANES 6 -%define %%IDX %1 ; [in] GP with good lane idx (scaled x16) -%define %%NULL_MASK %2 ; [clobbered] GP to store NULL lane mask -%define %%KEY_TAB %3 ; [clobbered] GP to store key table pointer -%define %%XTMP1 %4 ; [clobbered] temp XMM reg -%define %%XTMP2 %5 ; [clobbered] temp XMM reg -%define %%MASK_REG %6 ; [in] mask register - - vmovdqa64 %%XTMP1, [state + _aes_cmac_args_IV + %%IDX] - lea %%KEY_TAB, [state + _aes_cmac_args_key_tab] - kmovw DWORD(%%NULL_MASK), %%MASK_REG - -%assign j 0 ; outer loop to iterate through round keys -%rep 15 - vmovdqa64 %%XTMP2, [%%KEY_TAB + j + %%IDX] - -%assign k 0 ; inner loop to iterate through lanes -%rep 16 - bt %%NULL_MASK, k - jnc %%_skip_copy %+ j %+ _ %+ k - -%if j == 0 ;; copy IVs for each lane just once - vmovdqa64 [state + _aes_cmac_args_IV + (k*16)], %%XTMP1 -%endif - ;; copy key for each lane - vmovdqa64 [%%KEY_TAB + j + (k*16)], %%XTMP2 -%%_skip_copy %+ j %+ _ %+ k: -%assign k (k + 1) -%endrep - -%assign j (j + 256) -%endrep - -%endmacro - ; clear IVs, scratch buffers and round key's in NULL lanes %macro CLEAR_IV_KEYS_SCRATCH_IN_NULL_LANES 3 %define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask @@ -366,21 +334,15 @@ endstruc vmovdqu64 zmm2, [state + _aes_cmac_job_in_lane + (8*PTR_SZ)] vpcmpq k1, zmm1, zmm0, 4 ; NEQ vpcmpq k2, zmm2, zmm0, 4 ; NEQ - kmovw DWORD(tmp), k1 - kmovw DWORD(tmp4), k2 - mov DWORD(tmp2), DWORD(tmp4) - shl DWORD(tmp2), 8 - or DWORD(tmp2), DWORD(tmp) ; mask of non-null jobs in tmp2 - not BYTE(tmp) - kmovw k4, DWORD(tmp) - not BYTE(tmp4) - kmovw k5, DWORD(tmp4) - mov DWORD(tmp), DWORD(tmp2) - not WORD(tmp) - kmovw k6, DWORD(tmp) ; mask of NULL jobs in k4, k5 and k6 - mov DWORD(tmp), DWORD(tmp2) + + kshiftlw k2, k2, 8 + korw k6, k2, k1 + kmovw DWORD(arg3), k6 ; mask of non-null lanes in arg3 + knotw k6, k6 + kmovw k4, k6 + kshiftrw k5, k4, 8 ; mask of NULL jobs in k4, k5 and k6 xor tmp2, tmp2 - bsf WORD(tmp2), WORD(tmp) ; index of the 1st set bit in tmp2 + bsf WORD(tmp2), WORD(arg3) ; index of the 1st set bit in tmp2 ;; copy good lane data into NULL lanes mov tmp, [state + _aes_cmac_args_in + tmp2*8] @@ -395,12 +357,6 @@ endstruc vmovdqu16 ymm0{k6}, ymm3 vmovdqa64 [state + _aes_cmac_lens], ymm0 - ;; scale up good lane idx before copying IV and keys - shl tmp2, 4 - - ;; - copy IV and round keys to null lanes - COPY_IV_KEYS_TO_NULL_LANES tmp2, tmp4, tmp3, xmm4, xmm5, k6 - ;; Find min length for lanes 0-7 vphminposuw xmm2, xmm0 @@ -428,7 +384,12 @@ endstruc ; "state" and "args" are the same address, arg1 ; len2 is arg2 +%ifidn %%SUBMIT_FLUSH, SUBMIT call AES_CBC_MAC +%else + ; valid lane mask is arg3 + call AES_CBC_MAC_FLUSH +%endif ; state and idx are intact vmovdqa ymm0, [state + _aes_cmac_lens] ; preload lens @@ -442,16 +403,9 @@ endstruc ; Set len to 16 mov tmp3, 16 -%ifndef LINUX - mov tmp2, rcx ; save rcx -%endif - mov rcx, idx - mov DWORD(tmp4), 1 - shl DWORD(tmp4), cl -%ifndef LINUX - mov rcx, tmp2 ; restore rcx -%endif - kmovq k1, tmp4 + xor tmp4, tmp4 + bts WORD(tmp4), WORD(idx) + kmovw k1, DWORD(tmp4) vpbroadcastw ymm1, WORD(tmp3) vmovdqu16 ymm0{k1}, ymm1 @@ -478,6 +432,11 @@ endstruc kshiftrw k5, k4, 8 ;; lanes 8-15 mask in k5 vmovdqa64 [state + _aes_cmac_args_in + (0*PTR_SZ)]{k4}, zmm1 vmovdqa64 [state + _aes_cmac_args_in + (8*PTR_SZ)]{k5}, zmm1 + + ;; reset valid lanes in arg3 + knotw k4, k6 + kmovw DWORD(arg3), k4 + %else ;; only update processed lane input pointer on submit mov [state + _aes_cmac_args_in + idx*8], m_last diff --git a/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm index 8a43c6c3..119fb6fc 100644 --- a/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm @@ -27,6 +27,7 @@ %ifndef AES_CBC_MAC %define AES_CBC_MAC aes256_cbc_mac_vaes_avx512 +%define AES_CBC_MAC_FLUSH aes256_cbc_mac_flush_vaes_avx512 %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes256_cmac_auth_vaes_avx512 %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes256_cmac_auth_vaes_avx512 %define NUM_KEYS 15 -- GitLab From fd9fde0645c46962019be6cbac83f59eca54b504 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 16 Mar 2022 12:18:14 +0000 Subject: [PATCH 023/369] avx512: [AES-XCBC] Optimize flush to avoid copying lane data --- lib/avx512/aes_cbc_enc_vaes_avx512.asm | 20 ++++++ ...s128_xcbc_submit_flush_x16_vaes_avx512.asm | 70 ++++++------------- 2 files changed, 41 insertions(+), 49 deletions(-) diff --git a/lib/avx512/aes_cbc_enc_vaes_avx512.asm b/lib/avx512/aes_cbc_enc_vaes_avx512.asm index bf89fa6d..d94425c2 100644 --- a/lib/avx512/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512/aes_cbc_enc_vaes_avx512.asm @@ -1241,4 +1241,24 @@ aes256_cbc_mac_flush_vaes_avx512: ret +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void aes_xcbc_mac_128_flush_vaes_avx512(AES_ARGS *args, +;; uint64_t len_in_bytes, +;; uint16_t valid_lane_mask); +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(aes_xcbc_mac_128_flush_vaes_avx512,function,internal) +aes_xcbc_mac_128_flush_vaes_avx512: + endbranch64 + FUNC_SAVE + CBC_ENC 9, MAC_TYPE_XCBC, FLUSH + FUNC_RESTORE + +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + + ret + mksection stack-noexec diff --git a/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm index daaa17cb..790718e4 100644 --- a/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm @@ -35,12 +35,14 @@ %ifndef AES_XCBC_X16 %define AES_XCBC_X16 aes_xcbc_mac_128_vaes_avx512 +%define AES_XCBC_X16_FLUSH aes_xcbc_mac_128_flush_vaes_avx512 %define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_vaes_avx512 %define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_vaes_avx512 %define NUM_KEYS 11 %endif extern AES_XCBC_X16 +extern AES_XCBC_X16_FLUSH mksection .rodata default rel @@ -58,9 +60,11 @@ mksection .text %ifdef LINUX %define arg1 rdi %define arg2 rsi +%define arg3 rdx %else %define arg1 rcx %define arg2 rdx +%define arg3 r8 %endif %define state arg1 @@ -138,42 +142,6 @@ endstruc vextracti64x2 [%%COL + ROW*10], %%ZTMP, 2 %endmacro -; copy IV's and round keys into NULL lanes -%macro COPY_IV_KEYS_TO_NULL_LANES 6 -%define %%IDX %1 ; [in] GP with good lane idx (scaled x16) -%define %%NULL_MASK %2 ; [clobbered] GP to store NULL lane mask -%define %%KEY_TAB %3 ; [clobbered] GP to store key table pointer -%define %%XTMP1 %4 ; [clobbered] temp XMM reg -%define %%XTMP2 %5 ; [clobbered] temp XMM reg -%define %%MASK_REG %6 ; [in] mask register - - vmovdqa64 %%XTMP1, [state + _aes_xcbc_args_ICV + %%IDX] - lea %%KEY_TAB, [state + _aes_xcbc_args_key_tab] - kmovw DWORD(%%NULL_MASK), %%MASK_REG - -%assign j 0 ; outer loop to iterate through round keys -%rep NUM_KEYS - vmovdqa64 %%XTMP2, [%%KEY_TAB + j + %%IDX] - -%assign k 0 ; inner loop to iterate through lanes -%rep 16 - bt %%NULL_MASK, k - jnc %%_skip_copy %+ j %+ _ %+ k - -%if j == 0 ;; copy IVs for each lane just once - vmovdqa64 [state + _aes_xcbc_args_ICV + (k*16)], %%XTMP1 -%endif - ;; copy key for each lane - vmovdqa64 [%%KEY_TAB + j + (k*16)], %%XTMP2 -%%_skip_copy %+ j %+ _ %+ k: -%assign k (k + 1) -%endrep - -%assign j (j + 256) -%endrep - -%endmacro - ; clear final block buffers and round key's in NULL lanes %macro CLEAR_KEYS_FINAL_BLK_IN_NULL_LANES 3 %define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask @@ -304,10 +272,10 @@ endstruc %assign i (i - 1) %endrep kmovw k6, DWORD(tmp) - movzx tmp3, BYTE(tmp) - kmovw k4, DWORD(tmp3) - shr tmp, 8 - kmovw k5, DWORD(tmp) + kmovw k4, k6 + kshiftrw k5, k6, 8 + kmovw DWORD(arg3), k6 + not WORD(arg3) ; mask of non-null lanes in arg3 mov tmp, [state + _aes_xcbc_args_in + tmp2*8] vpbroadcastq zmm1, tmp @@ -321,12 +289,6 @@ endstruc vmovdqu16 ymm0{k6}, ymm3 vmovdqa64 [state + _aes_xcbc_lens], ymm0 - ;; scale up good lane idx before copying IV and keys - shl tmp2, 4 - - ;; - copy IV and round keys to null lanes - COPY_IV_KEYS_TO_NULL_LANES tmp2, tmp, tmp3, xmm4, xmm5, k6 - ;; Find min length for lanes 0-7 vphminposuw xmm2, xmm0 @@ -355,7 +317,12 @@ endstruc ; "state" and "args" are the same address, arg1 ; len is arg2 - call AES_XCBC_X16 +%ifidn %%SUBMIT_FLUSH, SUBMIT + call AES_XCBC_X16 +%else + ; valid lane mask is arg3 + call AES_XCBC_X16_FLUSH +%endif ; state and idx are intact %%_len_is_0: @@ -370,8 +337,9 @@ endstruc ;; Update lane len vmovdqa64 ymm0, [state + _aes_xcbc_lens] - SHIFT_GP 1, idx, tmp, tmp3, left - kmovq k1, tmp + xor tmp, tmp + bts WORD(tmp), WORD(idx) + kmovq k1, tmp mov tmp3, 16 vpbroadcastw ymm1, WORD(tmp3) @@ -399,6 +367,10 @@ endstruc kshiftrw k5, k4, 8 ;; lanes 8-15 mask in k5 vmovdqa64 [state + _aes_xcbc_args_in + (0*PTR_SZ)]{k4}, zmm1 vmovdqa64 [state + _aes_xcbc_args_in + (8*PTR_SZ)]{k5}, zmm1 + + ;; reset valid lanes in arg3 + knotw k4, k6 + kmovw DWORD(arg3), k4 %else ;; only update processed lane input pointer on submit mov [state + _aes_xcbc_args_in + 8*idx], tmp -- GitLab From 02345c46e65f74b3d400f375eadcab2747dcab59 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 22 Mar 2022 15:29:19 +0000 Subject: [PATCH 024/369] avx512: [CBC/CMAC/XCBC] Stop clearing null lane data at end of flush Not needed since copying good land data no longer happens. --- .../mb_mgr_aes128_cbc_enc_flush_avx512.asm | 19 +++++---- ...s128_cmac_submit_flush_x16_vaes_avx512.asm | 42 ------------------- ...s128_xcbc_submit_flush_x16_vaes_avx512.asm | 39 ----------------- 3 files changed, 10 insertions(+), 90 deletions(-) diff --git a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm index c2f79180..c8aaa5bf 100644 --- a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -248,15 +248,16 @@ len_is_0: sub qword [state + _aes_lanes_in_use], 1 %ifdef SAFE_DATA - ; Set bit of lane of returned job - xor DWORD(tmp3), DWORD(tmp3) - bts DWORD(tmp3), DWORD(idx) - kmovw k1, DWORD(tmp3) - korw k6, k1, k6 - - ;; Clear IV and expanded keys of returned job and "NULL lanes" - ;; (k6 contains the mask of the jobs) - CLEAR_IV_KEYS_IN_NULL_LANES tmp1, xmm0, k6 + ;; Clear IV + vpxorq xmm0, xmm0 + shl idx, 4 ; multiply by 16 + + ;; Clear expanded keys +%assign round 0 +%rep NUM_KEYS + vmovdqa [state + _aesarg_key_tab + round * (16*16) + idx], xmm0 +%assign round (round + 1) +%endrep %endif return: diff --git a/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm index a65c40d5..f0e51738 100644 --- a/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm @@ -152,35 +152,6 @@ endstruc %endif %endmacro -; clear IVs, scratch buffers and round key's in NULL lanes -%macro CLEAR_IV_KEYS_SCRATCH_IN_NULL_LANES 3 -%define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask -%define %%XTMP %2 ; [clobbered] temp XMM reg -%define %%MASK_REG %3 ; [in] mask register - - vpxorq %%XTMP, %%XTMP - kmovw DWORD(%%NULL_MASK), %%MASK_REG -%assign k 0 ; outer loop to iterate through lanes -%rep 16 - bt %%NULL_MASK, k - jnc %%_skip_clear %+ k - - ;; clean lane scratch and IV buffers - vmovdqa64 [state + _aes_cmac_scratch + (k*16)], %%XTMP - vmovdqa64 [state + _aes_cmac_args_IV + (k*16)], %%XTMP - -%assign j 0 ; inner loop to iterate through round keys -%rep NUM_KEYS - vmovdqa64 [state + _aes_cmac_args_key_tab + j + (k*16)], %%XTMP -%assign j (j + 256) - -%endrep -%%_skip_clear %+ k: -%assign k (k + 1) -%endrep - -%endmacro - ;;; =========================================================================== ;;; AES CMAC job submit & flush ;;; =========================================================================== @@ -496,7 +467,6 @@ endstruc %ifdef SAFE_DATA vpxor xmm0, xmm0 -%ifidn %%SUBMIT_FLUSH, SUBMIT ;; Clear IV and scratch memory of returned job shl idx, 4 vmovdqa [state + _aes_cmac_scratch + idx], xmm0 @@ -509,18 +479,6 @@ endstruc %assign round (round + 1) %endrep -%else - ;; Clear keys and scratch memory of returned job and "NULL lanes" - xor DWORD(tmp2), DWORD(tmp2) - bts DWORD(tmp2), DWORD(idx) - kmovw k1, DWORD(tmp2) - korw k6, k1, k6 - - ;; Clear IVs, keys and scratch buffers of returned job and "NULL lanes" - ;; (k6 contains the mask of the jobs) - CLEAR_IV_KEYS_SCRATCH_IN_NULL_LANES tmp4, xmm0, k6 -%endif ;; SUBMIT - %else vzeroupper %endif ;; SAFE_DATA diff --git a/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm index 790718e4..ba09c887 100644 --- a/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm @@ -142,34 +142,6 @@ endstruc vextracti64x2 [%%COL + ROW*10], %%ZTMP, 2 %endmacro -; clear final block buffers and round key's in NULL lanes -%macro CLEAR_KEYS_FINAL_BLK_IN_NULL_LANES 3 -%define %%NULL_MASK %1 ; [clobbered] GP to store NULL lane mask -%define %%YTMP %2 ; [clobbered] temp YMM reg -%define %%MASK_REG %3 ; [in] mask register - - vpxor %%YTMP, %%YTMP - kmovw DWORD(%%NULL_MASK), %%MASK_REG -%assign k 0 ; outer loop to iterate through lanes -%rep 16 - bt %%NULL_MASK, k - jnc %%_skip_clear %+ k - - ;; clear final blocks and ICV buffers - vmovdqa [state + _aes_xcbc_ldata + k * _XCBC_LANE_DATA_size + _xcbc_final_block], %%YTMP - vmovdqa [state + _aes_xcbc_args_ICV + k * 16], XWORD(%%YTMP) -%assign j 0 ; inner loop to iterate through round keys -%rep NUM_KEYS - vmovdqa [state + _aes_xcbc_args_key_tab + j + (k*16)], XWORD(%%YTMP) -%assign j (j + 256) - -%endrep -%%_skip_clear %+ k: -%assign k (k + 1) -%endrep - -%endmacro - ;;; =========================================================================== ;;; AES XCBC job submit & flush ;;; =========================================================================== @@ -399,7 +371,6 @@ endstruc %ifdef SAFE_DATA vpxor ymm0, ymm0 -%ifidn %%SUBMIT_FLUSH, SUBMIT ;; Clear final block (32 bytes) vmovdqa [lane_data + _xcbc_final_block], ymm0 @@ -410,16 +381,6 @@ endstruc %assign round (round + 1) %endrep -%else ;; FLUSH - ;; Clear keys and final blocks of returned job and "NULL lanes" - shr idx, 4 ;; divide by 16 to restore lane idx - xor DWORD(tmp), DWORD(tmp) - bts DWORD(tmp), DWORD(idx) - kmovw k1, DWORD(tmp) - korw k6, k1, k6 - ;; k6 contains the mask of the jobs - CLEAR_KEYS_FINAL_BLK_IN_NULL_LANES tmp, ymm0, k6 -%endif %else vzeroupper %endif -- GitLab From c4f356fbd44ea00a4c22eb1d824818620b54fe46 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 10 Mar 2022 18:04:12 +0000 Subject: [PATCH 025/369] avx: [ZUC-EIA3-256] Add tag size parameter to EIA3-256 C function Add tag size parameter to ZUC-EIA3-256 C function (in top file), in preparation for other tag sizes. It also makes pMacI parameter of void* type, instad of uint32_t *, as tag will be of different sizes. --- lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 38 +++++++++++++++++++------ lib/avx/zuc_top_avx.c | 9 +++--- lib/include/zuc_internal.h | 5 ++-- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index 72e299a7..000630ee 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -104,13 +104,15 @@ extern asm_ZucCipher_4_avx %define arg4 rcx %define arg5 r8 %define arg6 r9 +%define arg7 qword [rsp] %else %define arg1 rcx %define arg2 rdx %define arg3 r8 %define arg4 r9 -%define arg5 [rsp + 32] -%define arg6 [rsp + 40] +%define arg5 qword [rsp + 32] +%define arg6 qword [rsp + 40] +%define arg7 qword [rsp + 48] %endif %define state arg1 @@ -323,7 +325,7 @@ mksection .text %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 3 + RESERVE_STACK_SPACE 4 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] @@ -340,7 +342,7 @@ mksection .text call asm_Zuc256Initialization_4_avx %endif - RESTORE_STACK_SPACE 3 + RESTORE_STACK_SPACE 4 cmp byte [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_submit_restoring_state @@ -527,7 +529,7 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 3 + RESERVE_STACK_SPACE 4 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] @@ -544,7 +546,7 @@ APPEND(%%skip_eea3_,I): call asm_Zuc256Initialization_4_avx %endif - RESTORE_STACK_SPACE 3 + RESTORE_STACK_SPACE 4 cmp word [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_flush_restoring_state @@ -799,7 +801,11 @@ FLUSH_JOB_ZUC256_EEA3: ; to pass parameter to next function mov r11, state +%if %%KEY_SIZE == 128 RESERVE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESERVE_STACK_SPACE 7 +%endif lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] @@ -814,6 +820,9 @@ FLUSH_JOB_ZUC256_EEA3: lea r12, [r11 + _zuc_job_in_lane] mov arg6, r12 %endif +%if %%KEY_SIZE == 256 + mov arg7, 4 +%endif %if %%KEY_SIZE == 128 call zuc_eia3_4_buffer_job_avx @@ -821,8 +830,11 @@ FLUSH_JOB_ZUC256_EEA3: call zuc256_eia3_4_buffer_job_avx %endif +%if %%KEY_SIZE == 128 RESTORE_STACK_SPACE 6 - +%else ;; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 +%endif mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -949,8 +961,11 @@ APPEND(%%skip_eia3_,I): ; to pass parameter to next function mov r11, state +%if %%KEY_SIZE == 128 RESERVE_STACK_SPACE 6 - +%else ; %%KEY_SIZE == 256 + RESERVE_STACK_SPACE 7 +%endif lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_args_in] @@ -964,6 +979,9 @@ APPEND(%%skip_eia3_,I): lea r12, [r11 + _zuc_job_in_lane] mov arg6, r12 %endif +%if %%KEY_SIZE == 256 + mov arg7, 4 +%endif %if %%KEY_SIZE == 128 call zuc_eia3_4_buffer_job_avx @@ -971,7 +989,11 @@ APPEND(%%skip_eia3_,I): call zuc256_eia3_4_buffer_job_avx %endif +%if %%KEY_SIZE == 128 RESTORE_STACK_SPACE 6 +%else ;; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 +%endif mov tmp5, [rsp + _null_len_save] mov state, [rsp + _gpr_save + 8*8] diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 8333d806..105dc7d3 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -902,9 +902,10 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], const uint8_t *ivs, const void * const pBufferIn[NUM_AVX_BUFS], - uint32_t *pMacI[NUM_AVX_BUFS], + void *pMacI[NUM_AVX_BUFS], const uint16_t lengthInBits[NUM_AVX_BUFS], - const void * const job_in_lane[NUM_AVX_BUFS]) + const void * const job_in_lane[NUM_AVX_BUFS], + const uint64_t tag_size) { unsigned int i; DECLARE_ALIGNED(ZucState4_t state, 64); @@ -946,7 +947,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], } /* TODO: Handle 8 and 16-byte digest cases */ - asm_Zuc256Initialization_4_avx(&keys, ivs, &state, 4); + asm_Zuc256Initialization_4_avx(&keys, ivs, &state, tag_size); asm_ZucGenKeystream4B_4_avx(&state, pKeyStrArr); @@ -1041,7 +1042,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits); /* save the final MAC-I result */ - memcpy(pMacI[i], tag, 4); + memcpy(pMacI[i], tag, tag_size); } #ifdef SAFE_DATA diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 6c25b516..58feb58a 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1311,9 +1311,10 @@ IMB_DLL_LOCAL void zuc256_eia3_4_buffer_job_avx(const void * const pKey[4], const uint8_t *ivs, const void * const pBufferIn[4], - uint32_t *pMacI[4], + void *pMacI[4], const uint16_t lengthInBits[4], - const void * const job_in_lane[4]); + const void * const job_in_lane[4], + const uint64_t tag_size); IMB_DLL_LOCAL void zuc_eia3_8_buffer_job_avx2(const void * const pKey[8], -- GitLab From 680e87a07203911fb38124e9cce9dd4667f42387 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 10 Mar 2022 18:25:46 +0000 Subject: [PATCH 026/369] avx: [ZUC-EIA3-256] Intialize ZUC-256 tags at LFSR init Initalize the authentication tags for ZUC-256 in initialization function, also modifying how the constant tables are loaded (for cipher, it is no longer required to pass a "dummy" value of 2. --- lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 20 +--- lib/avx/zuc_top_avx.c | 10 +- lib/avx/zuc_x4_avx.asm | 142 ++++++++++++++++-------- lib/include/zuc_internal.h | 1 + lib/sse/zuc_top_sse.c | 2 +- lib/sse/zuc_x4_sse.asm | 4 +- 6 files changed, 110 insertions(+), 69 deletions(-) diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index 000630ee..eb275c34 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -325,24 +325,20 @@ mksection .text %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 4 + RESERVE_STACK_SPACE 5 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] -%if %%KEY_SIZE == 256 - ;; Setting "tag size" to 2 in case of ciphering - ;; (dummy size, just for constant selecion at Initialization) - mov arg4, 2 -%endif %if %%KEY_SIZE == 128 call asm_ZucInitialization_4_avx %else + mov arg5, 0 ; Tag size = 0, arg4 not used call asm_Zuc256Initialization_4_avx %endif - RESTORE_STACK_SPACE 4 + RESTORE_STACK_SPACE 5 cmp byte [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_submit_restoring_state @@ -529,24 +525,20 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 4 + RESERVE_STACK_SPACE 5 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] -%if %%KEY_SIZE == 256 - ;; Setting "tag size" to 2 in case of ciphering - ;; (dummy size, just for constant selecion at Initialization) - mov arg4, 2 -%endif %if %%KEY_SIZE == 128 call asm_ZucInitialization_4_avx %else + mov arg5, 0 ; Tag size = 0, arg4 not used call asm_Zuc256Initialization_4_avx %endif - RESTORE_STACK_SPACE 4 + RESTORE_STACK_SPACE 5 cmp word [r12 + _zuc_init_not_done], 0x0f ; Init done for all lanes je %%skip_flush_restoring_state diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 105dc7d3..3ffe0871 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -916,7 +916,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], const uint8_t *pIn8[NUM_AVX_BUFS] = {NULL}; uint32_t remainCommonBits; uint32_t numKeyStr = 0; - uint8_t T[NUM_AVX_BUFS*4] = {0}; + DECLARE_ALIGNED(uint8_t T[NUM_AVX_BUFS*4], 16) = {0}; const uint32_t keyStreamLengthInBits = KEYSTR_ROUND_LEN * 8; DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX_BUFS], 16) = {NULL}; unsigned int allCommonBits; @@ -947,13 +947,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], } /* TODO: Handle 8 and 16-byte digest cases */ - asm_Zuc256Initialization_4_avx(&keys, ivs, &state, tag_size); - - asm_ZucGenKeystream4B_4_avx(&state, pKeyStrArr); - - /* Initialize the tag with the first 4 bytes of the keystream */ - for (i = 0; i < NUM_AVX_BUFS; i++) - memcpy(&T[i], pKeyStrArr[i], 4); + asm_Zuc256Initialization_4_avx(&keys, ivs, &state, T, tag_size); /* Generate 16 bytes at a time */ asm_ZucGenKeystream16B_4_avx(&state, pKeyStrArr); diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 45156fca..e2d84e7d 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -60,7 +60,7 @@ dd 0x004D7800, 0x002F1300, 0x006BC400, 0x001AF100, dd 0x005E2600, 0x003C4D00, 0x00789A00, 0x0047AC00 ; Constants to be used to initialize the LFSR registers -; This table contains four different sets of constants: +; The tables contain four different sets of constants: ; 0-63 bytes: Encryption ; 64-127 bytes: Authentication with tag size = 4 ; 128-191 bytes: Authentication with tag size = 8 @@ -71,14 +71,23 @@ dd 0x00220000, 0x002F0000, 0x00240000, 0x002A0000, dd 0x006D0000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00520000, 0x00100000, 0x00300000 + +align 16 +EK256_EIA3_4: dd 0x00220000, 0x002F0000, 0x00250000, 0x002A0000, dd 0x006D0000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00520000, 0x00100000, 0x00300000 + +align 16 +EK256_EIA3_8: dd 0x00230000, 0x002F0000, 0x00240000, 0x002A0000, dd 0x006D0000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00520000, 0x00100000, 0x00300000 + +align 16 +EK256_EIA3_16: dd 0x00230000, 0x002F0000, 0x00250000, 0x002A0000, dd 0x006D0000, 0x00400000, 0x00400000, 0x00400000, dd 0x00400000, 0x00400000, 0x00400000, 0x00400000, @@ -559,6 +568,34 @@ align 64 vmovdqa [%%STATE + (( 0 + %%ROUND_NUM) % 16)*16], %%W %endmacro +; This macro reorder the LFSR registers +; after N rounds (1 <= N <= 15), since the registers +; are shifted every round +; +; The macro clobbers XMM0-15 +; +%macro REORDER_LFSR 2 +%define %%STATE %1 ; [in] Pointer to LFSR state +%define %%NUM_ROUNDS %2 ; [immediate] Number of key generation rounds + +%if %%NUM_ROUNDS != 16 +%assign %%i 0 +%rep 16 + vmovdqa APPEND(xmm,%%i), [%%STATE + 16*%%i] +%assign %%i (%%i+1) +%endrep + +%assign %%i 0 +%assign %%j %%NUM_ROUNDS +%rep 16 + vmovdqa [%%STATE + 16*%%i], APPEND(xmm,%%j) +%assign %%i (%%i+1) +%assign %%j ((%%j+1) % 16) +%endrep +%endif ;; %%NUM_ROUNDS != 16 + +%endmacro + ; ; Initialize LFSR registers for a single lane, for ZUC-128 ; @@ -600,8 +637,13 @@ align 64 %define %%LFSR12_15 %6 ;; [out] XMM register to contain initialized LFSR regs 12-15 %define %%XTMP %7 ;; [clobbered] XMM temporary register %define %%TMP %8 ;; [clobbered] GP temporary register -%define %%CONSTANTS %9 ;; [in] Address to constants +%define %%TAG_SIZE %9 ;; [in] Tag size (0, 4, 8 or 16 bytes) +%if %%TAG_SIZE == 0 +%define %%CONSTANTS rel EK256_d64 +%elif %%TAG_SIZE == 4 +%define %%CONSTANTS rel EK256_EIA3_4 +%endif ; s0 - s3 vpxor %%LFSR0_3, %%LFSR0_3 vpinsrb %%LFSR0_3, [%%KEY], 3 ; s0 @@ -724,13 +766,14 @@ align 64 vpor %%LFSR12_15, %%XTMP %endmacro -%macro ZUC_INIT_4 1 +%macro ZUC_INIT_4 2-3 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [in] Tag size (0 (for cipher), 4, 8 or 16) +%define %%TAGS %3 ; [in] Array of temporary tags %define pKe arg1 %define pIv arg2 %define pState arg3 -%define tag_sz arg4 ; Only used in ZUC-256 %define %%XTMP1 xmm0 %define %%XTMP2 xmm1 @@ -750,7 +793,10 @@ align 64 %define %%XTMP16 xmm15 %define %%W %%XTMP10 -%define %%X3 %%XTMP11 +%define %%KSTR1 %%XTMP12 +%define %%KSTR2 %%XTMP13 +%define %%KSTR3 %%XTMP14 +%define %%KSTR4 %%XTMP15 %define %%MASK_31 %%XTMP16 FUNC_SAVE @@ -811,14 +857,6 @@ align 64 %else ;; %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea r13, [rel EK256_d64] - bsf DWORD(tag_sz), DWORD(tag_sz) - dec DWORD(tag_sz) - shl DWORD(tag_sz), 6 - add r13, tag_sz - ;;; Initialize all LFSR registers %assign %%OFF 0 %rep 4 @@ -827,7 +865,7 @@ align 64 lea r10, [pIv + %%OFF*4] ; Initialize S0-15 for each packet - INIT_LFSR_256 r15, r10, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, r11, r13 + INIT_LFSR_256 r15, r10, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, r11, %%TAG_SIZE vmovdqa [pState + 2*%%OFF], %%XTMP1 vmovdqa [pState + 2*%%OFF + 64], %%XTMP2 @@ -860,16 +898,16 @@ align 64 vmovdqa %%MASK_31, [rel mask31] ; Shift LFSR 32-times, update state variables -%assign N 0 +%assign %%N 0 %rep 32 - BITS_REORG4 pState, N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + BITS_REORG4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10 NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W vpsrld %%W, 1 ; Shift out LSB of W - LFSR_UPDT4 pState, N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + LFSR_UPDT4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ %%MASK_31, %%W, init ; W used in LFSR update -%assign N N+1 +%assign %%N %%N+1 %endrep ; And once more, initial round from keygen phase = 33 times @@ -880,6 +918,36 @@ align 64 LFSR_UPDT4 pState, 0, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ %%MASK_31, %%XTMP8, work + ; Generate extra 4, 8 or 16 bytes of KS for initial tags +%if %%TAG_SIZE == 4 +%define %%NUM_ROUNDS 1 +%elif %%TAG_SIZE == 8 +%define %%NUM_ROUNDS 2 +%elif %%TAG_SIZE == 16 +%define %%NUM_ROUNDS 4 +%else +%define %%NUM_ROUNDS 0 +%endif + +%assign %%N 1 +%rep %%NUM_ROUNDS + BITS_REORG4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, \ + %%XTMP6, %%XTMP7, %%XTMP8, %%XTMP9, %%XTMP10, APPEND(%%KSTR, %%N) + NONLIN_FUN4 pState, %%XTMP1, %%XTMP2, %%XTMP3, \ + %%XTMP4, %%XTMP5, %%XTMP6, %%XTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor APPEND(%%KSTR, %%N), %%W + LFSR_UPDT4 pState, %%N, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, \ + %%MASK_31, %%XTMP8, work +%assign %%N %%N+1 +%endrep + +%if %%TAG_SIZE == 4 + vmovdqa [%%TAGS], %%KSTR1 + REORDER_LFSR pState, 1 +%elif %%TAG_SIZE == 8 ;;TODO +%elif %%TAG_SIZE == 16 ;;TODO +%endif FUNC_RESTORE ret @@ -887,39 +955,25 @@ align 64 MKGLOBAL(asm_ZucInitialization_4_avx,function,internal) asm_ZucInitialization_4_avx: - ZUC_INIT_4 128 + ZUC_INIT_4 128, 0 MKGLOBAL(asm_Zuc256Initialization_4_avx,function,internal) asm_Zuc256Initialization_4_avx: - ZUC_INIT_4 256 +%define tags arg4 +%define tag_sz arg5 -; This macro reorder the LFSR registers -; after N rounds (1 <= N <= 15), since the registers -; are shifted every round -; -; The macro clobbers XMM0-15 -; -%macro REORDER_LFSR 2 -%define %%STATE %1 -%define %%NUM_ROUNDS %2 + cmp tag_sz, 0 + je init_for_cipher -%if %%NUM_ROUNDS != 16 -%assign %%i 0 -%rep 16 - vmovdqa APPEND(xmm,%%i), [%%STATE + 16*%%i] -%assign %%i (%%i+1) -%endrep + ;; TODO: Check for 8B and 16B tags + cmp tag_sz, 4 + je init_for_auth_tag_4B -%assign %%i 0 -%assign %%j %%NUM_ROUNDS -%rep 16 - vmovdqa [%%STATE + 16*%%i], APPEND(xmm,%%j) -%assign %%i (%%i+1) -%assign %%j ((%%j+1) % 16) -%endrep -%endif ;; %%NUM_ROUNDS != 16 +init_for_cipher: + ZUC_INIT_4 256, 0 -%endmacro +init_for_auth_tag_4B: + ZUC_INIT_4 256, 4, tags ; ; Generate N*4 bytes of keystream diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 58feb58a..396f3f0f 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -314,6 +314,7 @@ IMB_DLL_LOCAL void asm_Zuc256Initialization_4_gfni_sse(ZucKey4_t *pKeys, IMB_DLL_LOCAL void asm_Zuc256Initialization_4_avx(ZucKey4_t *pKeys, const uint8_t *ivs, ZucState4_t *pState, + void *tags, const uint64_t tag_sz); diff --git a/lib/sse/zuc_top_sse.c b/lib/sse/zuc_top_sse.c index 7abbdd31..9f95761b 100755 --- a/lib/sse/zuc_top_sse.c +++ b/lib/sse/zuc_top_sse.c @@ -1091,7 +1091,7 @@ void _zuc256_eia3_4_buffer_job(const void * const pKey[NUM_SSE_BUFS], const uint8_t *pIn8[NUM_SSE_BUFS] = {NULL}; uint32_t remainCommonBits; uint32_t dataDigested = 0; - uint8_t T[NUM_SSE_BUFS*16] = {0}; + DECLARE_ALIGNED(uint8_t T[NUM_SSE_BUFS*16], 16) = {0}; const uint32_t keyStreamLengthInBits = KEYSTR_ROUND_LEN * 8; DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_SSE_BUFS], 16) = {NULL}; unsigned int allCommonBits; diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index f76e795b..7466f403 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -687,8 +687,8 @@ mksection .text ; The macro clobbers XMM0-15 ; %macro REORDER_LFSR 2 -%define %%STATE %1 -%define %%NUM_ROUNDS %2 +%define %%STATE %1 ; [in] Pointer to LFSR state +%define %%NUM_ROUNDS %2 ; [immediate] Number of key generation rounds %if %%NUM_ROUNDS != 16 %assign %%i 0 -- GitLab From 4ddca78721f746889f90a9c557a589a0f59416fc Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 11 Mar 2022 09:31:24 +0000 Subject: [PATCH 027/369] avx: [ZUC-EIA3-256] Initialize 8-byte and 16-byte digests --- lib/avx/zuc_top_avx.c | 1 - lib/avx/zuc_x4_avx.asm | 48 ++++++++++++++++++++++++++++++++++-------- lib/sse/zuc_x4_sse.asm | 7 +++--- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 3ffe0871..56879b59 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -946,7 +946,6 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], keys.pKeys[i] = pKey[i]; } - /* TODO: Handle 8 and 16-byte digest cases */ asm_Zuc256Initialization_4_avx(&keys, ivs, &state, T, tag_size); /* Generate 16 bytes at a time */ diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index e2d84e7d..4d3c5206 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -643,6 +643,10 @@ align 64 %define %%CONSTANTS rel EK256_d64 %elif %%TAG_SIZE == 4 %define %%CONSTANTS rel EK256_EIA3_4 +%elif %%TAG_SIZE == 8 +%define %%CONSTANTS rel EK256_EIA3_8 +%elif %%TAG_SIZE == 16 +%define %%CONSTANTS rel EK256_EIA3_16 %endif ; s0 - s3 vpxor %%LFSR0_3, %%LFSR0_3 @@ -945,18 +949,33 @@ align 64 %if %%TAG_SIZE == 4 vmovdqa [%%TAGS], %%KSTR1 REORDER_LFSR pState, 1 -%elif %%TAG_SIZE == 8 ;;TODO -%elif %%TAG_SIZE == 16 ;;TODO +%elif %%TAG_SIZE == 8 + ; Transpose the keystream and store the 8 bytes per buffer consecutively, + ; being the initial tag for each buffer + vpunpckldq %%XTMP1, %%KSTR1, %%KSTR2 + vpunpckhdq %%XTMP2, %%KSTR1, %%KSTR2 + vmovdqa [%%TAGS], %%XTMP1 + vmovdqa [%%TAGS + 16], %%XTMP2 + REORDER_LFSR pState, 2 +%elif %%TAG_SIZE == 16 + ; Transpose the keystream and store the 16 bytes per buffer consecutively, + ; being the initial tag for each buffer + TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 + vmovdqa [%%TAGS], %%KSTR1 + vmovdqa [%%TAGS + 16], %%KSTR2 + vmovdqa [%%TAGS + 16*2], %%KSTR3 + vmovdqa [%%TAGS + 16*3], %%KSTR4 + REORDER_LFSR pState, 4 %endif FUNC_RESTORE - - ret %endmacro MKGLOBAL(asm_ZucInitialization_4_avx,function,internal) asm_ZucInitialization_4_avx: ZUC_INIT_4 128, 0 + ret + MKGLOBAL(asm_Zuc256Initialization_4_avx,function,internal) asm_Zuc256Initialization_4_avx: %define tags arg4 @@ -965,15 +984,26 @@ asm_Zuc256Initialization_4_avx: cmp tag_sz, 0 je init_for_cipher - ;; TODO: Check for 8B and 16B tags - cmp tag_sz, 4 - je init_for_auth_tag_4B + cmp tag_sz, 8 + je init_for_auth_tag_8B + jb init_for_auth_tag_4B -init_for_cipher: - ZUC_INIT_4 256, 0 + ; Fall-through for tag size = 16 bytes +init_for_auth_tag_16B: + ZUC_INIT_4 256, 16, tags + ret + +init_for_auth_tag_8B: + ZUC_INIT_4 256, 8, tags + ret init_for_auth_tag_4B: ZUC_INIT_4 256, 4, tags + ret + +init_for_cipher: + ZUC_INIT_4 256, 0 + ret ; ; Generate N*4 bytes of keystream diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index 7466f403..3abfdfd8 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -1075,16 +1075,15 @@ mksection .text %elif %%TAG_SIZE == 8 ; Transpose the keystream and store the 8 bytes per buffer consecutively, ; being the initial tag for each buffer - pshufd %%KSTR1, %%KSTR1, 0xD8 - pshufd %%KSTR2, %%KSTR2, 0xD8 - movdqa %%XTMP1, %%KSTR1 punpckldq %%XTMP1, %%KSTR2 - punpckldq %%KSTR1, %%KSTR2 + punpckhdq %%KSTR1, %%KSTR2 movdqa [%%TAGS], %%XTMP1 movdqa [%%TAGS + 16], %%KSTR1 REORDER_LFSR pState, 2 %elif %%TAG_SIZE == 16 + ; Transpose the keystream and store the 16 bytes per buffer consecutively, + ; being the initial tag for each buffer TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 movdqa [%%TAGS], %%KSTR1 movdqa [%%TAGS + 16], %%KSTR2 -- GitLab From 04c544ef5aa6e1f4b834a9270ae3ac34dbbe77df Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 11 Mar 2022 12:49:15 +0000 Subject: [PATCH 028/369] avx: [ZUC-EIA3-256] Produce 8-byte and 16-byte digests in DIGEST_16_BYTES macro --- lib/avx/zuc_x4_avx.asm | 154 ++++++++++++++++++++++++++++++++--------- lib/sse/zuc_x4_sse.asm | 3 + 2 files changed, 123 insertions(+), 34 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index 4d3c5206..c067082d 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -135,10 +135,6 @@ align 16 bit_reverse_and_table: db 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f -align 16 -data_mask_64bits: -dd 0xffffffff, 0xffffffff, 0x00000000, 0x00000000 - align 16 swap_mask: db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 @@ -209,6 +205,20 @@ shuf_mask_dw2_0_dw3_0: db 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff db 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff +align 16 +bits_32_63: +dd 0x00000000, 0xffffffff, 0x00000000, 0x00000000 + +align 16 +shuf_mask_0_0_dw1_0: +db 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +db 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff + +align 16 +shuf_mask_0_0_0_dw1: +db 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +db 0xff, 0xff, 0xff, 0xff, 0x04, 0x05, 0x06, 0x07 + ; Stack frame for ZucCipher function struc STACK _rsp_save: resq 1 ; Space for rsp pointer @@ -1490,7 +1500,7 @@ exit_cipher: ; ; Processes 16 bytes of data and updates the digest ; -%macro DIGEST_16_BYTES 12 +%macro DIGEST_16_BYTES 17 %define %%KS %1 ; [in] Pointer to 24-byte keystream %define %%BIT_REV_L %2 ; [in] Bit reverse low table (XMM) %define %%BIT_REV_H %3 ; [in] Bit reverse high table (XMM) @@ -1500,9 +1510,14 @@ exit_cipher: %define %%XTMP2 %7 ; [clobbered] Temporary XMM register %define %%XTMP3 %8 ; [clobbered] Temporary XMM register %define %%XTMP4 %9 ; [clobbered] Temporary XMM register -%define %%KS_L %10 ; [clobbered] Temporary XMM register -%define %%KS_H %11 ; [clobbered] Temporary XMM register -%define %%OFF %12 ; [in] Offset into KS +%define %%XTMP5 %10 ; [clobbered] Temporary XMM register +%define %%XTMP6 %11 ; [clobbered] Temporary XMM register +%define %%KS_L %12 ; [clobbered] Temporary XMM register +%define %%KS_M1 %13 ; [clobbered] Temporary XMM register +%define %%KS_M2 %14 ; [clobbered] Temporary XMM register +%define %%KS_H %15 ; [clobbered] Temporary XMM register +%define %%OFF %16 ; [in] Offset into KS +%define %%TAG_SZ %17 ; [in] Tag size (4, 8 or 16) vpand %%XTMP2, %%XTMP1, %%BIT_REV_AND @@ -1511,35 +1526,97 @@ exit_cipher: vpshufb %%XTMP4, %%BIT_REV_H, %%XTMP2 vpshufb %%XTMP1, %%BIT_REV_L, %%XTMP3 - vpor %%XTMP4, %%XTMP1 ;; %%XTMP4 - bit reverse data bytes + vpor %%XTMP4, %%XTMP4, %%XTMP1 ;; %%XTMP4 - bit reverse data bytes ;; ZUC authentication part ;; - 4x32 data bits ;; - set up KS - vpshufd %%KS_L, [%%KS + %%OFF + (0*4)], 0x61 - vpshufd %%KS_H, [%%KS + %%OFF + (2*4)], 0x61 + vpshufd %%KS_L, [%%KS + %%OFF + (0*4)], 0x61 ; KS bits [63:32 31:0 95:64 63:32] + vpshufd %%KS_M1, [%%KS + %%OFF + (2*4)], 0x61 ; KS bits [127:96 95:64 159:128 127:96] +%if %%TAG_SZ != 4 ;; TAG_SZ == 8 or 16 + vpshufd %%KS_M2, [%%KS + %%OFF + (4*4)], 0x61 ; KS bits [191:160 159:128 223:192 191:160] +%if %%TAG_SZ == 16 + vpshufd %%KS_H, [%%KS + %%OFF + (4*4)], 0xBB ; KS bits [255:224 223:192 255:224 223:192] +%endif +%endif ;; TAG_SZ != 4 ;; - set up DATA ; Data bytes [31:0 0s 63:32 0s] vpshufb %%XTMP1, %%XTMP4, [rel shuf_mask_dw0_0_dw1_0] ; Data bytes [95:64 0s 127:96 0s] - vpshufb %%XTMP3, %%XTMP4, [rel shuf_mask_dw2_0_dw3_0] + vpshufb %%XTMP2, %%XTMP4, [rel shuf_mask_dw2_0_dw3_0] ;; - clmul ;; - xor the results from 4 32-bit words together - vpclmulqdq %%XTMP2, %%XTMP1, %%KS_L, 0x11 - vpclmulqdq %%XTMP1, %%KS_L, 0x00 - vpclmulqdq %%XTMP4, %%XTMP3, %%KS_H, 0x00 - vpclmulqdq %%XTMP3, %%KS_H, 0x11 - - vpxor %%XTMP2, %%XTMP1 - vpxor %%XTMP4, %%XTMP3 - vpxor %%XDIGEST, %%XTMP2 - vpxor %%XDIGEST, %%XTMP4 + vpclmulqdq %%XTMP3, %%XTMP1, %%KS_L, 0x00 + vpclmulqdq %%XTMP4, %%XTMP1, %%KS_L, 0x11 + vpclmulqdq %%XTMP5, %%XTMP2, %%KS_M1, 0x00 + vpclmulqdq %%XTMP6, %%XTMP2, %%KS_M1, 0x11 + + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpxor %%XTMP5, %%XTMP5, %%XTMP6 + vpxor %%XTMP3, %%XTMP3, %%XTMP5 +%if %%TAG_SZ == 4 + vpxor %%XDIGEST, %%XDIGEST, %%XTMP3 +%endif ; %%TAG_SZ == 4 +%if %%TAG_SZ >= 8 + ; Move previous result to low 32 bits and XOR with previous digest + vmovq %%XTMP3, %%XTMP3 ; Clear top 64 bits + vpsrldq %%XTMP3, %%XTMP3, 4 + vpxor %%XDIGEST, %%XDIGEST, %%XTMP3 + + vpclmulqdq %%XTMP3, %%XTMP1, %%KS_L, 0x10 + vpclmulqdq %%XTMP4, %%XTMP1, %%KS_M1, 0x01 + vpclmulqdq %%XTMP5, %%XTMP2, %%KS_M1, 0x10 + vpclmulqdq %%XTMP6, %%XTMP2, %%KS_M2, 0x01 + + ; XOR all the products and keep only 32-63 bits + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpxor %%XTMP5, %%XTMP5, %%XTMP6 + vpxor %%XTMP3, %%XTMP3, %%XTMP5 + vpand %%XTMP3, %%XTMP3, [rel bits_32_63] + + ; XOR with bits 32-63 of previous digest + vpxor %%XDIGEST, %%XDIGEST, %%XTMP3 + +%if %%TAG_SZ == 16 + ; Prepare data and calculate bits 95-64 of tag + vpclmulqdq %%XTMP3, %%XTMP1, %%KS_M1, 0x00 + vpclmulqdq %%XTMP4, %%XTMP1, %%KS_M1, 0x11 + vpclmulqdq %%XTMP5, %%XTMP2, %%KS_M2, 0x00 + vpclmulqdq %%XTMP6, %%XTMP2, %%KS_M2, 0x11 + + ; XOR all the products and move bits 63-32 to bits 95-64 + vpxor %%XTMP3, %%XTMP4 + vpxor %%XTMP5, %%XTMP6 + vpxor %%XTMP3, %%XTMP5 + + vpshufb %%XTMP3, %%XTMP3, [rel shuf_mask_0_0_dw1_0] + + ; XOR with previous bits 64-95 of previous digest + vpxor %%XDIGEST, %%XDIGEST, %%XTMP3 + + ; Prepare data and calculate bits 127-96 of tag + vpclmulqdq %%XTMP3, %%XTMP1, %%KS_M1, 0x10 + vpclmulqdq %%XTMP4, %%XTMP1, %%KS_M2, 0x01 + vpclmulqdq %%XTMP5, %%XTMP2, %%KS_M2, 0x10 + vpclmulqdq %%XTMP6, %%XTMP2, %%KS_H, 0x01 + + ; XOR all the products and move bits 63-32 to bits 127-96 + vpxor %%XTMP3, %%XTMP3, %%XTMP4 + vpxor %%XTMP5, %%XTMP5, %%XTMP6 + vpxor %%XTMP3, %%XTMP3, %%XTMP5 + vpshufb %%XTMP3, %%XTMP3, [rel shuf_mask_0_0_0_dw1] + + ; XOR with lower 96 bits, to construct 128 bits of tag + vpxor %%XDIGEST, %%XDIGEST, %%XTMP3 + +%endif ; %%TAG_SZ == 16 +%endif ; %%TAG_SZ >= 8 %endmacro -%macro REMAINDER 18 +%macro REMAINDER 22 %define %%T %1 ; [in] Pointer to authentication tag %define %%KS %2 ; [in/clobbered] Pointer to 32-byte keystream %define %%DATA %3 ; [in/clobbered] Pointer to input data @@ -1556,8 +1633,12 @@ exit_cipher: %define %%XTMP2 %14 ; [clobbered] Temporary XMM register %define %%XTMP3 %15 ; [clobbered] Temporary XMM register %define %%XTMP4 %16 ; [clobbered] Temporary XMM register -%define %%KS_L %17 ; [clobbered] Temporary XMM register -%define %%KS_H %18 ; [clobbered] Temporary XMM register +%define %%XTMP5 %17 ; [clobbered] Temporary XMM register +%define %%XTMP6 %18 ; [clobbered] Temporary XMM register +%define %%KS_L %19 ; [clobbered] Temporary XMM register +%define %%KS_M1 %20 ; [clobbered] Temporary XMM register +%define %%KS_M2 %21 ; [clobbered] Temporary XMM register +%define %%KS_H %22 ; [clobbered] Temporary XMM register FUNC_SAVE @@ -1571,7 +1652,7 @@ exit_cipher: vmovdqu %%XTMP1, [%%DATA] DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ - %%KS_L, %%KS_H, 0 + %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, 0, 4 add %%DATA, 16 add %%KS, 16 @@ -1607,7 +1688,7 @@ exit_cipher: DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ - %%KS_L, %%KS_H, 0 + %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, 0, 4 %%Eia3RoundsAVX_end: @@ -1676,11 +1757,11 @@ asm_Eia3Remainder_avx: REMAINDER T, KS, DATA, N_BITS, r12, r13, r14, r15, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13 ret -%macro EIA3_ROUND 15 +%macro EIA3_ROUND 19 %define %%T %1 ; [in] Pointer to authentication tag %define %%KS %2 ; [in/clobbered] Pointer to 32-byte keystream %define %%DATA %3 ; [in/clobbered] Pointer to input data @@ -1693,9 +1774,13 @@ asm_Eia3Remainder_avx: %define %%XTMP2 %10 ; [clobbered] Temporary XMM register %define %%XTMP3 %11 ; [clobbered] Temporary XMM register %define %%XTMP4 %12 ; [clobbered] Temporary XMM register -%define %%KS_L %13 ; [clobbered] Temporary XMM register -%define %%KS_H %14 ; [clobbered] Temporary XMM register -%define %%NUM_16B_ROUNDS %15 ; [in] Number of 16-byte rounds +%define %%XTMP5 %13 ; [clobbered] Temporary XMM register +%define %%XTMP6 %14 ; [clobbered] Temporary XMM register +%define %%KS_L %15 ; [clobbered] Temporary XMM register +%define %%KS_M1 %16 ; [clobbered] Temporary XMM register +%define %%KS_M2 %17 ; [clobbered] Temporary XMM register +%define %%KS_H %18 ; [clobbered] Temporary XMM register +%define %%NUM_16B_ROUNDS %19 ; [in] Number of 16-byte rounds vpxor %%XDIGEST, %%XDIGEST @@ -1705,7 +1790,8 @@ asm_Eia3Remainder_avx: DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ - %%KS_L, %%KS_H, %%OFF + %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, \ + %%OFF, 4 %assign %%OFF (%%OFF + 16) %endrep @@ -1747,7 +1833,7 @@ asm_Eia3Round32B_avx: EIA3_ROUND T, KS, DATA, r11, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9, 2 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 2 ;; Copy last 32 bytes of KS to the front vmovdqa xmm0, [KS + 32] @@ -1789,7 +1875,7 @@ asm_Eia3Round16B_avx: EIA3_ROUND T, KS, DATA, r11, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9, 1 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 1 ;; Copy last 16 bytes of KS to the front vmovdqa xmm0, [KS + 16] diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm index 3abfdfd8..76abf2df 100755 --- a/lib/sse/zuc_x4_sse.asm +++ b/lib/sse/zuc_x4_sse.asm @@ -1626,6 +1626,9 @@ exit_cipher: ret +; +; Processes 16 bytes of data and updates the digest +; %macro DIGEST_16_BYTES 14 %define %%KS %1 ; [in] Pointer to keystream %define %%XDATA %2 ; [in] XMM register with input data -- GitLab From 433ef2a12be7f5638a256c2895e6595361f7b8b0 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 11 Mar 2022 13:22:19 +0000 Subject: [PATCH 029/369] avx: [ZUC-EIA3-256] Support 8-byte and 16-byte tags in Round function --- lib/avx/zuc_top_avx.c | 14 +++---- lib/avx/zuc_x4_avx.asm | 82 ++++++++++++++++++++++++++++++++------ lib/avx2/zuc_top_avx2.c | 14 +++---- lib/include/zuc_internal.h | 6 ++- 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 56879b59..16e198c3 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -542,7 +542,7 @@ void _zuc_eia3_1_buffer_avx(const void *pKey, asm_ZucGenKeystream8B_avx(&keyStream[4], &zucState); else asm_ZucGenKeystream16B_avx(&keyStream[4], &zucState); - asm_Eia3Round16B_avx(&T, keyStream, pIn8); + asm_Eia3Round16B_avx(&T, keyStream, pIn8, 4); pIn8 = &pIn8[KEYSTR_ROUND_LEN]; } @@ -632,7 +632,7 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], else asm_ZucGenKeystream16B_4_avx(&state, pKeyStrArr); for (i = 0; i < NUM_AVX_BUFS; i++) { - asm_Eia3Round16B_avx(&T[i], keyStr[i], pIn8[i]); + asm_Eia3Round16B_avx(&T[i], keyStr[i], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -678,7 +678,7 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], else asm_ZucGenKeystream16B_avx(&keyStr32[4], &singlePktState); - asm_Eia3Round16B_avx(&T[i], keyStr32, pIn8[i]); + asm_Eia3Round16B_avx(&T[i], keyStr32, pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } @@ -824,7 +824,7 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], for (i = 0; i < NUM_AVX_BUFS; i++) { if (job_in_lane[i] == NULL) continue; - asm_Eia3Round16B_avx(&T[i], keyStr[i], pIn8[i]); + asm_Eia3Round16B_avx(&T[i], keyStr[i], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -873,7 +873,7 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], else asm_ZucGenKeystream16B_avx(&keyStr32[4], &singlePktState); - asm_Eia3Round16B_avx(&T[i], keyStr32, pIn8[i]); + asm_Eia3Round16B_avx(&T[i], keyStr32, pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } @@ -970,7 +970,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], if (job_in_lane[i] == NULL) continue; - asm_Eia3Round16B_avx(tag, keyStr[i], pIn8[i]); + asm_Eia3Round16B_avx(tag, keyStr[i], pIn8[i], tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -1021,7 +1021,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], else asm_ZucGenKeystream16B_avx(&keyStr32[4], &singlePktState); - asm_Eia3Round16B_avx(tag, keyStr32, pIn8[i]); + asm_Eia3Round16B_avx(tag, keyStr32, pIn8[i], tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index c067082d..efeff52f 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -1761,7 +1761,7 @@ asm_Eia3Remainder_avx: ret -%macro EIA3_ROUND 19 +%macro EIA3_ROUND 20 %define %%T %1 ; [in] Pointer to authentication tag %define %%KS %2 ; [in/clobbered] Pointer to 32-byte keystream %define %%DATA %3 ; [in/clobbered] Pointer to input data @@ -1781,6 +1781,7 @@ asm_Eia3Remainder_avx: %define %%KS_M2 %17 ; [clobbered] Temporary XMM register %define %%KS_H %18 ; [clobbered] Temporary XMM register %define %%NUM_16B_ROUNDS %19 ; [in] Number of 16-byte rounds +%define %%TAG_SZ %20 ; [constant] Tag size (4, 8 or 16 bytes) vpxor %%XDIGEST, %%XDIGEST @@ -1791,20 +1792,31 @@ asm_Eia3Remainder_avx: DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, \ - %%OFF, 4 + %%OFF, %%TAG_SZ %assign %%OFF (%%OFF + 16) %endrep +%if %%TAG_SZ == 4 ;; - update T vmovq %%TMP, %%XDIGEST shr %%TMP, 32 xor [%%T], DWORD(%%TMP) +%elif %%TAG_SZ == 8 + ;; - update T + vmovq %%TMP, %%XDIGEST + xor [%%T], %%TMP +%else ;; %%TAG_SZ == 16 + vmovdqa %%XTMP1, [%%T] + vpxor %%XTMP1, %%XDIGEST + vmovdqa [%%T], %%XTMP1 +%endif %endmacro ;; -;;extern void asm_Eia3Round32B_avx(void *T, const void *KS, const void *DATA) +;;extern void asm_Eia3Round32B_avx(void *T, const void *KS, const void *DATA, +;; const uint64_t tag_sz) ;; ;; Updates authentication tag T based on keystream KS and DATA. ;; - it processes 32 bytes of DATA @@ -1816,14 +1828,16 @@ asm_Eia3Remainder_avx: ;; @param [in] T (digest pointer) ;; @param [in] KS (key stream pointer) ;; @param [in] DATA (data pointer) +;; @param [in] TAG_SZ (Tag size: 4, 8 or 16 bytes) ;; align 64 MKGLOBAL(asm_Eia3Round32B_avx,function,internal) asm_Eia3Round32B_avx: -%define T arg1 -%define KS arg2 -%define DATA arg3 +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define TAG_SZ arg4 FUNC_SAVE @@ -1831,9 +1845,30 @@ asm_Eia3Round32B_avx: vmovdqa xmm1, [bit_reverse_table_h] vmovdqa xmm2, [bit_reverse_and_table] + cmp TAG_SZ, 8 + je round32B_tag_8B + ja round32B_tag_16B + + ; Fall-through for 4 bytes +round32B_tag_4B: EIA3_ROUND T, KS, DATA, r11, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 2 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 2, 4 + + jmp end_round32B + +round32B_tag_8B: + EIA3_ROUND T, KS, DATA, r11, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 2, 8 + + jmp end_round32B + +round32B_tag_16B: + EIA3_ROUND T, KS, DATA, r11, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 2, 16 +end_round32B: ;; Copy last 32 bytes of KS to the front vmovdqa xmm0, [KS + 32] @@ -1846,7 +1881,8 @@ asm_Eia3Round32B_avx: ret ;; -;;extern void asm_Eia3Round16B_avx(void *T, const void *KS, const void *DATA) +;;extern void asm_Eia3Round16B_avx(void *T, const void *KS, const void *DATA, +;; const uint64_t tag_sz) ;; ;; Updates authentication tag T based on keystream KS and DATA. ;; - it processes 16 bytes of DATA @@ -1858,14 +1894,16 @@ asm_Eia3Round32B_avx: ;; @param [in] T (digest pointer) ;; @param [in] KS (key stream pointer) ;; @param [in] DATA (data pointer) +;; @param [in] TAG_SZ (Tag size: 4, 8 or 16 bytes) ;; align 64 MKGLOBAL(asm_Eia3Round16B_avx,function,internal) asm_Eia3Round16B_avx: -%define T arg1 -%define KS arg2 -%define DATA arg3 +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define TAG_SZ arg4 FUNC_SAVE @@ -1873,10 +1911,30 @@ asm_Eia3Round16B_avx: vmovdqa xmm1, [bit_reverse_table_h] vmovdqa xmm2, [bit_reverse_and_table] + cmp TAG_SZ, 8 + je round16B_tag_8B + ja round16B_tag_16B + + ; Fall-through for 4 bytes +round16B_tag_4B: EIA3_ROUND T, KS, DATA, r11, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 1 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 1, 4 + + jmp end_round16B +round16B_tag_8B: + EIA3_ROUND T, KS, DATA, r11, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 1, 8 + + jmp end_round16B + +round16B_tag_16B: + EIA3_ROUND T, KS, DATA, r11, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 1, 16 +end_round16B: ;; Copy last 16 bytes of KS to the front vmovdqa xmm0, [KS + 16] vmovdqa [KS], xmm0 diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index 01c46c92..7a9760bb 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -515,7 +515,7 @@ void _zuc_eia3_1_buffer_avx2(const void *pKey, asm_ZucGenKeystream8B_avx(&keyStream[8], &zucState); else asm_ZucGenKeystream32B_avx(&keyStream[8], &zucState); - asm_Eia3Round32B_avx(&T, &keyStream[0], pIn8); + asm_Eia3Round32B_avx(&T, &keyStream[0], pIn8, 4); pIn8 = &pIn8[KEYSTR_ROUND_LEN]; } @@ -587,7 +587,7 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[NUM_AVX2_BUFS], asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); for (i = 0; i < NUM_AVX2_BUFS; i++) { - asm_Eia3Round32B_avx(&T[i], &keyStr[i][0], pIn8[i]); + asm_Eia3Round32B_avx(&T[i], &keyStr[i][0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -633,7 +633,7 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[NUM_AVX2_BUFS], else asm_ZucGenKeystream32B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Round32B_avx(&T[i], &keyStr32[0], pIn8[i]); + asm_Eia3Round32B_avx(&T[i], &keyStr32[0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } @@ -764,7 +764,7 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], if (job_in_lane[i] == NULL) continue; - asm_Eia3Round32B_avx(&T[i], &keyStr[i][0], pIn8[i]); + asm_Eia3Round32B_avx(&T[i], &keyStr[i][0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -813,7 +813,7 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], else asm_ZucGenKeystream32B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Round32B_avx(&T[i], &keyStr32[0], pIn8[i]); + asm_Eia3Round32B_avx(&T[i], &keyStr32[0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } @@ -902,7 +902,7 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], if (job_in_lane[i] == NULL) continue; - asm_Eia3Round32B_avx(tag, &keyStr[i][0], pIn8[i]); + asm_Eia3Round32B_avx(tag, &keyStr[i][0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } @@ -953,7 +953,7 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], else asm_ZucGenKeystream32B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Round32B_avx(tag, &keyStr32[0], pIn8[i]); + asm_Eia3Round32B_avx(tag, &keyStr32[0], pIn8[i], 4); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 396f3f0f..442c48af 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1087,10 +1087,12 @@ IMB_DLL_LOCAL void asm_Eia3Round16B_gfni_sse(void *T, const void *ks, const uint64_t tag_sz); IMB_DLL_LOCAL void asm_Eia3Round16B_avx(void *T, const void *ks, - const void *data); + const void *data, + const uint64_t tag_sz); IMB_DLL_LOCAL void asm_Eia3Round32B_avx(void *T, const void *ks, - const void *data); + const void *data, + const uint64_t tag_sz); /** ****************************************************************************** -- GitLab From bfcaa01e060a27e4810eec28c8c844f971757759 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 11 Mar 2022 13:40:03 +0000 Subject: [PATCH 030/369] avx: [ZUC-EIA3-256] Support 8-byte and 16-byte tags in Remainder function --- lib/avx/zuc_top_avx.c | 11 ++- lib/avx/zuc_x4_avx.asm | 172 ++++++++++++++++++++++++++++++------- lib/avx2/zuc_top_avx2.c | 11 ++- lib/include/zuc_internal.h | 26 ++---- 4 files changed, 161 insertions(+), 59 deletions(-) diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 16e198c3..9b5dd242 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -552,7 +552,7 @@ void _zuc_eia3_1_buffer_avx(const void *pKey, */ if (remainingBits > (2 * 32)) asm_ZucGenKeystream8B_avx(&keyStream[4], &zucState); - asm_Eia3Remainder_avx(&T, &keyStream[0], pIn8, remainingBits); + asm_Eia3Remainder_avx(&T, &keyStream[0], pIn8, remainingBits, 128, 4); *pMacI = T; @@ -691,7 +691,8 @@ void _zuc_eia3_4_buffer_avx(const void * const pKey[NUM_AVX_BUFS], asm_ZucGenKeystream8B_avx(&keyStr32[4], &singlePktState); - asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits, + 128, 4); /* save the final MAC-I result */ *(pMacI[i]) = T[i]; } @@ -885,7 +886,8 @@ void zuc_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], asm_ZucGenKeystream8B_avx(&keyStr32[4], &singlePktState); - asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits, + 128, 4); /* save the final MAC-I result */ *(pMacI[i]) = T[i]; } @@ -1033,7 +1035,8 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], asm_ZucGenKeystream_avx(&keyStr32[4], &singlePktState, 1); - asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits, + 256, tag_size); /* save the final MAC-I result */ memcpy(pMacI[i], tag, tag_size); } diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index efeff52f..ae3f56d6 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -39,12 +39,14 @@ %define arg3 rdx %define arg4 rcx %define arg5 r8 +%define arg6 r9 %else %define arg1 rcx %define arg2 rdx %define arg3 r8 %define arg4 r9 %define arg5 qword [rsp + 40] +%define arg6 qword [rsp + 48] %endif %define APPEND(a,b) a %+ b @@ -1616,29 +1618,32 @@ exit_cipher: %endif ; %%TAG_SZ >= 8 %endmacro -%macro REMAINDER 22 +%macro REMAINDER 25 %define %%T %1 ; [in] Pointer to authentication tag %define %%KS %2 ; [in/clobbered] Pointer to 32-byte keystream %define %%DATA %3 ; [in/clobbered] Pointer to input data %define %%N_BITS %4 ; [in/clobbered] Number of bits to digest %define %%N_BYTES %5 ; [clobbered] Number of bytes to digest -%define %%TMP %6 ; [clobbered] Temporary GP register +%define %%TMP1 %6 ; [clobbered] Temporary GP register %define %%TMP2 %7 ; [clobbered] Temporary GP register %define %%TMP3 %8 ; [clobbered] Temporary GP register -%define %%BIT_REV_L %9 ; [in] Bit reverse low table (XMM) -%define %%BIT_REV_H %10 ; [in] Bit reverse high table (XMM) -%define %%BIT_REV_AND %11 ; [in] Bit reverse and table (XMM) -%define %%XDIGEST %12 ; [clobbered] Temporary digest (XMM) -%define %%XTMP1 %13 ; [clobbered] Temporary XMM register -%define %%XTMP2 %14 ; [clobbered] Temporary XMM register -%define %%XTMP3 %15 ; [clobbered] Temporary XMM register -%define %%XTMP4 %16 ; [clobbered] Temporary XMM register -%define %%XTMP5 %17 ; [clobbered] Temporary XMM register -%define %%XTMP6 %18 ; [clobbered] Temporary XMM register -%define %%KS_L %19 ; [clobbered] Temporary XMM register -%define %%KS_M1 %20 ; [clobbered] Temporary XMM register -%define %%KS_M2 %21 ; [clobbered] Temporary XMM register -%define %%KS_H %22 ; [clobbered] Temporary XMM register +%define %%TMP4 %9 ; [clobbered] Temporary GP register +%define %%BIT_REV_L %10 ; [in] Bit reverse low table (XMM) +%define %%BIT_REV_H %11 ; [in] Bit reverse high table (XMM) +%define %%BIT_REV_AND %12 ; [in] Bit reverse and table (XMM) +%define %%XDIGEST %13 ; [clobbered] Temporary digest (XMM) +%define %%XTMP1 %14 ; [clobbered] Temporary XMM register +%define %%XTMP2 %15 ; [clobbered] Temporary XMM register +%define %%XTMP3 %16 ; [clobbered] Temporary XMM register +%define %%XTMP4 %17 ; [clobbered] Temporary XMM register +%define %%XTMP5 %18 ; [clobbered] Temporary XMM register +%define %%XTMP6 %19 ; [clobbered] Temporary XMM register +%define %%KS_L %20 ; [clobbered] Temporary XMM register +%define %%KS_M1 %21 ; [clobbered] Temporary XMM register +%define %%KS_M2 %22 ; [clobbered] Temporary XMM register +%define %%KS_H %23 ; [clobbered] Temporary XMM register +%define %%KEY_SZ %24 ; [in] Key size (128 or 256) +%define %%TAG_SZ %25 ; [in] Key size (4, 8 or 16) FUNC_SAVE @@ -1652,7 +1657,8 @@ exit_cipher: vmovdqu %%XTMP1, [%%DATA] DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ - %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, 0, 4 + %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, \ + 0, %%TAG_SZ add %%DATA, 16 add %%KS, 16 @@ -1669,11 +1675,11 @@ exit_cipher: ;; read up to 16 bytes of data, zero bits not needed if partial byte and bit-reverse simd_load_avx_16_1 %%XTMP1, %%DATA, %%N_BYTES ; check if there is a partial byte (less than 8 bits in last byte) - mov %%TMP, %%N_BITS - and %%TMP, 0x7 - shl %%TMP, 4 + mov %%TMP1, %%N_BITS + and %%TMP1, 0x7 + shl %%TMP1, 4 lea %%TMP2, [rel bit_mask_table] - add %%TMP2, %%TMP + add %%TMP2, %%TMP1 ; Get mask to clear last bits vmovdqa %%XTMP2, [%%TMP2] @@ -1682,17 +1688,19 @@ exit_cipher: ; to apply mask, then restore by shifting right same amount of bytes mov %%TMP2, 16 sub %%TMP2, %%N_BYTES - XVPSLLB %%XTMP1, %%TMP2, %%XTMP3, %%TMP + XVPSLLB %%XTMP1, %%TMP2, %%XTMP3, %%TMP1 vpand %%XTMP1, %%XTMP2 - XVPSRLB %%XTMP1, %%TMP2, %%XTMP3, %%TMP + XVPSRLB %%XTMP1, %%TMP2, %%XTMP3, %%TMP1 DIGEST_16_BYTES %%KS, %%BIT_REV_L, %%BIT_REV_H, %%BIT_REV_AND, \ %%XDIGEST, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, \ - %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, 0, 4 + %%XTMP5, %%XTMP6, %%KS_L, %%KS_M1, %%KS_M2, %%KS_H, \ + 0, %%TAG_SZ %%Eia3RoundsAVX_end: -%define %%TAG DWORD(%%TMP) +%if %%TAG_SZ == 4 +%define %%TAG DWORD(%%TMP1) ;; - update T mov %%TAG, [%%T] vmovq %%TMP2, %%XDIGEST @@ -1716,6 +1724,7 @@ exit_cipher: ; XOR with previous digest calculation xor %%TAG, DWORD(%%TMP3) +%if %%KEY_SZ == 128 ;; XOR with keyStr[L-1] ; Read keyStr[L - 1] (last double word of keyStr) @@ -1725,15 +1734,86 @@ exit_cipher: ; XOR with previous digest calculation xor %%TAG, [%%KS + %%TMP2 * 4] +%endif bswap %%TAG mov [%%T], %%TAG +%else ; %%TAG_SZ == 8 or 16 +%define %%TAG %%TMP1 + ;; Update lower 64 bits of T + vmovq %%TAG, %%XDIGEST + xor %%TAG, [%%T] + + ;; XOR with keyStr[n_bits] (Z_length, from spec) + + ; Read keyStr[N_BITS / 32] + mov %%TMP2, %%N_BITS + shr %%TMP2, 5 + mov %%TMP3, [%%KS + %%TMP2*4] + mov %%TMP4, [%%KS + %%TMP2*4 + 4] + + ; Rotate left by N_BITS % 32 + mov %%TMP2, rcx ; Save RCX + mov rcx, %%N_BITS + and rcx, 0x1F + rol %%TMP3, cl + rol %%TMP4, cl + mov rcx, %%TMP2 ; Restore RCX + + shl %%TMP4, 32 + mov DWORD(%%TMP3), DWORD(%%TMP3) ; Clear top 32 bits + or %%TMP4, %%TMP3 + + ; XOR with previous digest calculation + xor %%TAG, %%TMP4 + + ; Byte swap both dwords of the digest before writing out + bswap %%TAG + ror %%TAG, 32 + mov [%%T], %%TAG +%if %%TAG_SZ == 16 + ;; Update higher 64 bits of T + vpextrq %%TAG, %%XDIGEST, 1 + xor %%TAG, [%%T + 8] + + ;; XOR with keyStr[n_bits] (Z_length, from spec) + + ; Read keyStr[N_BITS / 32] + mov %%TMP2, %%N_BITS + shr %%TMP2, 5 + mov %%TMP3, [%%KS + %%TMP2*4 + 4*2] + mov %%TMP4, [%%KS + %%TMP2*4 + 4*3] + + ; Rotate left by N_BITS % 32 + mov %%TMP2, rcx ; Save RCX + mov rcx, %%N_BITS + and rcx, 0x1F + rol %%TMP3, cl + rol %%TMP4, cl + mov rcx, %%TMP2 ; Restore RCX + + shl %%TMP4, 32 + mov DWORD(%%TMP3), DWORD(%%TMP3) ; Clear top 32 bits + or %%TMP4, %%TMP3 + + ; XOR with previous digest calculation + xor %%TAG, %%TMP4 + + ; Byte swap both dwords of the digest before writing out + bswap %%TAG + ror %%TAG, 32 + mov [%%T + 8], %%TAG +%endif ; %%TAG_SZ == 16 +%endif ; %%TAG_SZ == 4 FUNC_RESTORE %endmacro ;; -;; extern void asm_Eia3Remainder_avx(void *T, const void *ks, const void *data, uint64_t n_bits) +;; extern void asm_Eia3Remainder_avx(void *T, const void *ks, +;; const void *data, const uint64_t n_bits, +;; const uint64_t key_size, +;; const uint64_t tag_size); ;; ;; Returns authentication update value to be XOR'ed with current authentication tag ;; @@ -1741,6 +1821,8 @@ exit_cipher: ;; @param [in] KS (key stream pointer) ;; @param [in] DATA (data pointer) ;; @param [in] N_BITS (number of bits to digest) +;; @param [in] KEY_SZ (Key size: 128 or 256 bits) +;; @param [in] TAG_SZ (Tag size: 4, 8 or 16 bytes) ;; align 64 MKGLOBAL(asm_Eia3Remainder_avx,function,internal) @@ -1750,15 +1832,44 @@ asm_Eia3Remainder_avx: %define KS arg2 %define DATA arg3 %define N_BITS arg4 +%define KEY_SZ arg5 +%define TAG_SZ arg6 vmovdqa xmm0, [rel bit_reverse_table_l] vmovdqa xmm1, [rel bit_reverse_table_h] vmovdqa xmm2, [rel bit_reverse_and_table] - REMAINDER T, KS, DATA, N_BITS, r12, r13, r14, r15, \ + cmp KEY_SZ, 128 + je remainder_key_sz_128 + + cmp TAG_SZ, 8 + je remainder_tag_sz_8 + ja remainder_tag_sz_16 + + ; Key size = 256 + ; Fall-through for tag size = 4 bytes +remainder_tag_sz_4: + REMAINDER T, KS, DATA, N_BITS, r11, r12, r13, r14, r15, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 256, 4 + ret + +remainder_tag_sz_8: + REMAINDER T, KS, DATA, N_BITS, r11, r12, r13, r14, r15, \ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ - xmm8, xmm9, xmm10, xmm11, xmm12, xmm13 + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 256, 8 + ret +remainder_tag_sz_16: + REMAINDER T, KS, DATA, N_BITS, r11, r12, r13, r14, r15, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 256, 16 + ret + +remainder_key_sz_128: + REMAINDER T, KS, DATA, N_BITS, r11, r12, r13, r14, r15, \ + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, 128, 4 ret %macro EIA3_ROUND 20 @@ -1807,9 +1918,8 @@ asm_Eia3Remainder_avx: vmovq %%TMP, %%XDIGEST xor [%%T], %%TMP %else ;; %%TAG_SZ == 16 - vmovdqa %%XTMP1, [%%T] - vpxor %%XTMP1, %%XDIGEST - vmovdqa [%%T], %%XTMP1 + vpxor %%XDIGEST, [%%T] + vmovdqa [%%T], %%XDIGEST %endif %endmacro diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index 7a9760bb..62baae7d 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -525,7 +525,7 @@ void _zuc_eia3_1_buffer_avx2(const void *pKey, */ if (remainingBits > (6 * 32)) asm_ZucGenKeystream8B_avx(&keyStream[8], &zucState); - asm_Eia3Remainder_avx(&T, &keyStream[0], pIn8, remainingBits); + asm_Eia3Remainder_avx(&T, &keyStream[0], pIn8, remainingBits, 128, 4); *pMacI = T; #ifdef SAFE_DATA @@ -646,7 +646,8 @@ void _zuc_eia3_8_buffer_avx2(const void * const pKey[NUM_AVX2_BUFS], asm_ZucGenKeystream8B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits, + 128, 4); /* save the final MAC-I result */ *(pMacI[i]) = T[i]; } @@ -826,7 +827,8 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], asm_ZucGenKeystream8B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(&T[i], keyStr32, pIn8[i], remainBits, + 128, 4); /* save the final MAC-I result */ *(pMacI[i]) = T[i]; @@ -966,7 +968,8 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], asm_ZucGenKeystream_avx(&keyStr32[8], &singlePktState, 1); - asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits); + asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits, + 256, 4); /* save the final MAC-I result */ memcpy(pMacI[i], tag, 4); diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 442c48af..5428fdaa 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1128,6 +1128,12 @@ IMB_DLL_LOCAL void asm_Eia3Remainder_gfni_sse(void *T, const void *ks, const uint64_t key_size, const uint64_t tag_size); +IMB_DLL_LOCAL void asm_Eia3Remainder_avx(void *T, const void *ks, + const void *data, + const uint64_t n_bits, + const uint64_t key_size, + const uint64_t tag_size); + /** ****************************************************************************** * @description @@ -1158,26 +1164,6 @@ IMB_DLL_LOCAL void asm_Eia3Round64B_16_VPCLMUL(uint32_t *T, IMB_DLL_LOCAL void asm_Eia3Round64BAVX512(uint32_t *T, const void *ks, const void *data); -/** - ****************************************************************************** - * @description - * Definition of the external function to return the authentication - * update value to be XOR'ed with current authentication tag (AVX variant) - * - * @param[in] ks Pointer to key stream - * - * @param[in] data Pointer to the data - * - * @param[in] n_bits Number of data bits to be processed - * - * @pre - * None - * - *****************************************************************************/ -IMB_DLL_LOCAL void asm_Eia3Remainder_avx(void *T, const void *ks, - const void *data, - const uint64_t n_bits); - /** ****************************************************************************** * @description -- GitLab From f0458393ea5cd3fabd705f35db3eb1cfbe1ecfd1 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 11 Mar 2022 14:09:06 +0000 Subject: [PATCH 031/369] avx: [ZUC] Pass tag size as argument to submit/flush --- ReleaseNotes.txt | 2 +- lib/avx/mb_mgr_avx.c | 15 +++--- lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 65 ++++++++++++++++++++----- lib/avx/zuc_top_avx.c | 54 +++++++++++--------- 4 files changed, 93 insertions(+), 43 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 5c7e27c0..c2bbe634 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -5,7 +5,7 @@ Unreleased ======================================================================== Library -- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE +- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE and AVX - AES-ECB AVX512-VAES implementation added v1.2 February 2022 diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 6087c6b4..112c3e92 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -79,12 +79,11 @@ IMB_JOB *submit_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state); -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - +IMB_JOB *submit_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); uint32_t hec_32_avx(const uint8_t *in); uint64_t hec_64_avx(const uint8_t *in); @@ -127,8 +126,8 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx #define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx #define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx #define AES_CBC_DEC_128 aes_cbc_dec_128_avx #define AES_CBC_DEC_192 aes_cbc_dec_192_avx diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index eb275c34..a26c055d 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -687,8 +687,9 @@ MKGLOBAL(FLUSH_JOB_ZUC256_EEA3,function,internal) FLUSH_JOB_ZUC256_EEA3: FLUSH_JOB_ZUC_EEA3 256 -%macro SUBMIT_JOB_ZUC_EIA3 1 +%macro SUBMIT_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16) ; idx needs to be in rbp %define len rbp @@ -813,7 +814,7 @@ FLUSH_JOB_ZUC256_EEA3: mov arg6, r12 %endif %if %%KEY_SIZE == 256 - mov arg7, 4 + mov arg7, %%TAG_SIZE %endif %if %%KEY_SIZE == 128 @@ -859,15 +860,18 @@ FLUSH_JOB_ZUC256_EEA3: %endif mov rsp, [rsp + _rsp_save] ; original SP - ret + jmp %%exit_submit_eia3 %%return_null_submit_eia3: xor job_rax, job_rax jmp %%return_submit_eia3 + +%%exit_submit_eia3: %endmacro -%macro FLUSH_JOB_ZUC_EIA3 1 +%macro FLUSH_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16) %define unused_lanes rbx %define tmp1 rbx @@ -972,7 +976,7 @@ APPEND(%%skip_eia3_,I): mov arg6, r12 %endif %if %%KEY_SIZE == 256 - mov arg7, 4 + mov arg7, %%TAG_SIZE %endif %if %%KEY_SIZE == 128 @@ -1019,11 +1023,13 @@ APPEND(%%skip_eia3_,I): %endif mov rsp, [rsp + _rsp_save] ; original SP - ret + jmp %%exit_flush_eia3 %%return_null_flush_eia3: xor job_rax, job_rax jmp %%return_flush_eia3 + +%%exit_flush_eia3: %endmacro ; JOB* SUBMIT_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -1031,25 +1037,60 @@ APPEND(%%skip_eia3_,I): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: - SUBMIT_JOB_ZUC_EIA3 128 + SUBMIT_JOB_ZUC_EIA3 128, 4 + ret -; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) +; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job, +; const uint64_t tag_sz) ; arg 1 : state ; arg 2 : job +; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: - SUBMIT_JOB_ZUC_EIA3 256 + cmp arg3, 8 + je submit_tag_8B + jb submit_tag_4B + + ; Fall-through for 16-byte tag +submit_tag_16B: + SUBMIT_JOB_ZUC_EIA3 256, 16 + ret +submit_tag_8B: + SUBMIT_JOB_ZUC_EIA3 256, 8 + ret +submit_tag_4B: + SUBMIT_JOB_ZUC_EIA3 256, 4 + ret ; JOB* FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: - FLUSH_JOB_ZUC_EIA3 128 + FLUSH_JOB_ZUC_EIA3 128, 4 + ret -; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state) +; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, +; const uint64_t tag_sz) ; arg 1 : state +; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: - FLUSH_JOB_ZUC_EIA3 256 + endbranch64 + cmp arg2, 8 + je flush_tag_8B + jb flush_tag_4B + + ; Fall-through for 16-byte tag +flush_tag_16B: + FLUSH_JOB_ZUC_EIA3 256, 16 + ret + +flush_tag_8B: + FLUSH_JOB_ZUC_EIA3 256, 8 + ret + +flush_tag_4B: + FLUSH_JOB_ZUC_EIA3 256, 4 + ret mksection stack-noexec diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c index 9b5dd242..a6f46e8a 100755 --- a/lib/avx/zuc_top_avx.c +++ b/lib/avx/zuc_top_avx.c @@ -918,7 +918,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], const uint8_t *pIn8[NUM_AVX_BUFS] = {NULL}; uint32_t remainCommonBits; uint32_t numKeyStr = 0; - DECLARE_ALIGNED(uint8_t T[NUM_AVX_BUFS*4], 16) = {0}; + DECLARE_ALIGNED(uint8_t T[NUM_AVX_BUFS*16], 16) = {0}; const uint32_t keyStreamLengthInBits = KEYSTR_ROUND_LEN * 8; DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX_BUFS], 16) = {NULL}; unsigned int allCommonBits; @@ -961,13 +961,19 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], while (remainCommonBits >= keyStreamLengthInBits) { remainCommonBits -= keyStreamLengthInBits; numKeyStr++; - /* Generate the next key stream 4 bytes or 16 bytes */ - if (!remainCommonBits && allCommonBits) - asm_ZucGenKeystream4B_4_avx(&state, pKeyStrArr); - else + /* Generate the next key stream 4/8 bytes or 16 bytes */ + if (!remainCommonBits && allCommonBits) { + if (tag_size == 4) + asm_ZucGenKeystream4B_4_avx(&state, pKeyStrArr); + else if (tag_size == 8) + asm_ZucGenKeystream8B_4_avx(&state, pKeyStrArr); + else + asm_ZucGenKeystream16B_4_avx(&state, + pKeyStrArr); + } else asm_ZucGenKeystream16B_4_avx(&state, pKeyStrArr); for (i = 0; i < NUM_AVX_BUFS; i++) { - uint32_t *tag = (uint32_t *) &T[i*4]; + void *tag = (void *) &T[i*tag_size]; if (job_in_lane[i] == NULL) continue; @@ -979,7 +985,7 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], /* Process each packet separately for the remaining bits */ for (i = 0; i < NUM_AVX_BUFS; i++) { - uint32_t *tag = (uint32_t *) &T[i*4]; + void *tag = (void *) &T[i*tag_size]; if (job_in_lane[i] == NULL) continue; @@ -987,11 +993,15 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; + const uint32_t N = remainBits + ((uint32_t) tag_size << 3); + uint32_t L = ((N + 31) / ZUC_WORD_BITS); - /* If remaining bits are more than 4 bytes, we need to generate - * at least 4B more of keystream, so we need to copy - * the zuc state to single packet state first */ - if (remainBits > 32) { + /* 4 KS words are generated already */ + L = (L > 4) ? (L - 4) : 0; + + /* Copy the ZUC state to single packet state, + * if more KS is needed */ + if (L > 0) { singlePktState.lfsrState[0] = state.lfsrState[0][i]; singlePktState.lfsrState[1] = state.lfsrState[1][i]; singlePktState.lfsrState[2] = state.lfsrState[2][i]; @@ -1016,24 +1026,24 @@ void zuc256_eia3_4_buffer_job_avx(const void * const pKey[NUM_AVX_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - /* Generate the next key stream 4 bytes or 16 bytes */ - if (!remainBits) - asm_ZucGenKeystream_avx(&keyStr32[4], - &singlePktState, 1); - else + /* Generate the next key stream (16 bytes max) */ + if (L > 3) { asm_ZucGenKeystream16B_avx(&keyStr32[4], &singlePktState); + L -= 4; + } else { + asm_ZucGenKeystream_avx(&keyStr32[4], + &singlePktState, L); + L = 0; + } asm_Eia3Round16B_avx(tag, keyStr32, pIn8[i], tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } - /* - * If remaining bits has more than 1 ZUC WORD (double words), - * keystream needs to have another ZUC WORD (4B) - */ - if (remainBits > (32)) + /* Generate final keystream if needed */ + if (L > 0) asm_ZucGenKeystream_avx(&keyStr32[4], - &singlePktState, 1); + &singlePktState, L); asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits, 256, tag_size); -- GitLab From 1898a3bcd7957eddb96dc9fc09baf11c3af36795 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Sat, 19 Mar 2022 09:13:16 +0000 Subject: [PATCH 032/369] lib: [ZUC-256] Fix 23-byte IV expansion Fixes #102. The expansion of 23-byte IV to 25 bytes for ZUC-256 was incorrect, breaking the LFSR initialization. --- ReleaseNotes.txt | 3 +++ lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 24 ++++++++++------- lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm | 26 +++++++++++-------- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 26 ++++++++++--------- lib/sse/mb_mgr_zuc_submit_flush_sse.asm | 24 ++++++++++------- 5 files changed, 60 insertions(+), 43 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index c2bbe634..2fa9d34f 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -8,6 +8,9 @@ Library - ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE and AVX - AES-ECB AVX512-VAES implementation added +Fixes +- Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) + v1.2 February 2022 ======================================================================== diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index a26c055d..f862e6b3 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -257,12 +257,14 @@ mksection .text ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes and write as 8 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out EXPAND_FROM_6_TO_8_BYTES tmp2, tmp, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read @@ -742,12 +744,14 @@ FLUSH_JOB_ZUC256_EEA3: ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes and write as 8 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out EXPAND_FROM_6_TO_8_BYTES tmp2, tmp, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm index dcc21b00..cbe1be55 100644 --- a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm @@ -232,14 +232,15 @@ mksection .text ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 - ; Expand to 8 bytes and write + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out mov tmp3, 0x3f3f3f3f3f3f3f3f pdep tmp2, tmp2, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read @@ -795,14 +796,17 @@ FLUSH_JOB_ZUC256_EEA3: ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 + ; Read next 6 bytes and write as 8 bytes + + ; Read last 8 bytes and keep only 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; Expand to 8 bytes and write mov tmp3, 0x3f3f3f3f3f3f3f3f pdep tmp2, tmp2, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index de7dd907..f7484ec6 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -177,14 +177,15 @@ mksection .text ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 - ; Expand to 8 bytes and write + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out mov tmp3, 0x3f3f3f3f3f3f3f3f pdep tmp2, tmp2, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read @@ -801,14 +802,15 @@ FLUSH_JOB_ZUC256_EEA3: ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 - ; Expand to 8 bytes and write + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out mov tmp3, 0x3f3f3f3f3f3f3f3f pdep tmp2, tmp2, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read diff --git a/lib/sse/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse/mb_mgr_zuc_submit_flush_sse.asm index 731250e5..eb39bfde 100644 --- a/lib/sse/mb_mgr_zuc_submit_flush_sse.asm +++ b/lib/sse/mb_mgr_zuc_submit_flush_sse.asm @@ -266,12 +266,14 @@ mksection .text ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes and write as 8 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out EXPAND_FROM_6_TO_8_BYTES tmp2, tmp, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read @@ -755,12 +757,14 @@ FLUSH_JOB_ZUC256_EEA3: ; Read and write next byte mov al, [tmp + 16] mov [state + _zuc_args_IV + lane + 16], al - ; Read next 6 bytes and write as 8 bytes - movzx DWORD(tmp2), word [tmp + 17] - mov DWORD(tmp3), [tmp + 19] - shl tmp2, 32 - or tmp2, tmp3 + ; Read last 8 bytes and keep only the last 6 bytes + mov tmp2, [tmp + 15] + mov tmp3, 0x0000ffffffffffff + bswap tmp2 + and tmp2, tmp3 ; last 6 bytes of IV + ; Expand 6 bytes to 8 bytes and write out EXPAND_FROM_6_TO_8_BYTES tmp2, tmp, tmp3 + bswap tmp2 mov [state + _zuc_args_IV + lane + 17], tmp2 jmp %%_iv_read -- GitLab From e70fb2ff5c47afa2bb7baad8eef0392c2b67d7e7 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Sat, 19 Mar 2022 09:12:40 +0000 Subject: [PATCH 033/369] test: [ZUC-256] Add more test vectors Add more ZUC-256 test vectors with 23-byte IVs for ciphering and authentication. --- test/zuc_test_vectors.h | 357 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 342 insertions(+), 15 deletions(-) diff --git a/test/zuc_test_vectors.h b/test/zuc_test_vectors.h index 74fad44c..8ab1b343 100644 --- a/test/zuc_test_vectors.h +++ b/test/zuc_test_vectors.h @@ -32,8 +32,8 @@ #define MAX_BUFFER_LENGTH_IN_BYTES ((MAX_BUFFER_LENGTH_IN_BITS) + 7)/8 #define NUM_ZUC_ALG_TESTS 3 #define NUM_ZUC_EEA3_TESTS 5 -#define NUM_ZUC_256_EEA3_TESTS 6 -#define NUM_ZUC_256_EIA3_TESTS 8 +#define NUM_ZUC_256_EEA3_TESTS 10 +#define NUM_ZUC_256_EIA3_TESTS 12 #define NUM_ZUC_EIA3_TESTS 10 #define ZUC_KEY_LEN_IN_BYTES 16 #define ZUC_IV_LEN_IN_BYTES 16 @@ -519,7 +519,191 @@ const struct test256EEA3_vectors_t test256EEA3_vectors[] = { 0x49, 0x43, 0xC6, 0xBB, 0xE8, 0xAD, 0x8A, 0xFD } }, - /* TestSet 2 */ + /* TestSet 2 */ + { + /* Key */ + {0x8f, 0x8e, 0xf9, 0xd8, 0xfb, 0x0a, 0xce, 0x2b, + 0x23, 0x19, 0x48, 0x42, 0xcb, 0x5c, 0x6d, 0x98, + 0x1e, 0x71, 0x68, 0x74, 0xe1, 0xdf, 0xeb, 0xe0, + 0xf2, 0x46, 0x02, 0x71, 0xbb, 0x69, 0x0d, 0x9e + }, + /* IV */ + {0x2c, 0xe8, 0x87, 0x0f, 0x8c, 0x7f, 0x47, 0x2a, + 0x02, 0x2d, 0x24, 0xcd, 0x23, 0x3f, 0x4d, 0x0a, + 0x40, 0x0d, 0x12, 0xdd, 0xc4, 0x16, 0x26 + }, + /* Message length */ + 1536, + /* IV length */ + 23, + /* Plaintext */ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }, + /* Ciphertext */ + {0xc1, 0xce, 0x46, 0xd2, 0x4e, 0x78, 0x6f, 0x97, + 0xcf, 0xc0, 0xa5, 0x3c, 0xec, 0x50, 0x6e, 0x17, + 0xe0, 0x8c, 0x7e, 0x33, 0x84, 0x98, 0x2b, 0xc1, + 0x68, 0x97, 0x24, 0x83, 0x03, 0x7c, 0x0d, 0xc5, + 0x19, 0xa1, 0xe8, 0xb1, 0xb7, 0x53, 0x4f, 0x3b, + 0x8a, 0xa3, 0xce, 0x9b, 0x3d, 0xd0, 0x1a, 0xf7, + 0x7d, 0xae, 0x4c, 0x6b, 0xe4, 0xe3, 0x12, 0x70, + 0x63, 0xc9, 0x4c, 0xcf, 0x1f, 0xf7, 0x18, 0xdf, + 0xf1, 0x7d, 0x96, 0xe4, 0x60, 0xa8, 0x3b, 0xf5, + 0x71, 0x7d, 0x2a, 0x87, 0x1d, 0x82, 0xed, 0x92, + 0xc5, 0xe7, 0x6e, 0xd9, 0x3c, 0x01, 0x0d, 0x87, + 0x13, 0x3b, 0x1a, 0x92, 0xa2, 0x83, 0x1a, 0x5b, + 0x9a, 0xfb, 0x81, 0x1d, 0xdb, 0xbd, 0x82, 0x01, + 0x3b, 0x32, 0x0e, 0x2c, 0x67, 0x3c, 0x14, 0x13, + 0x9d, 0x58, 0xf1, 0x88, 0x9d, 0xe5, 0xd6, 0xe3, + 0x48, 0xaa, 0x43, 0xc2, 0x08, 0xa6, 0x64, 0xa8, + 0xad, 0x71, 0x26, 0x7e, 0xe7, 0xed, 0x0c, 0x58, + 0xd3, 0x27, 0x42, 0x5e, 0x10, 0xb0, 0x03, 0x62, + 0x18, 0x30, 0xde, 0xdb, 0x45, 0xcd, 0x78, 0xdd, + 0xee, 0x4f, 0xa1, 0x45, 0xa6, 0xbf, 0xc1, 0x37, + 0x3e, 0x47, 0x5c, 0x1b, 0xb6, 0x8b, 0x63, 0x87, + 0x49, 0xc4, 0x1b, 0x9e, 0xea, 0x01, 0x62, 0x2a, + 0x44, 0x65, 0x17, 0x0f, 0xee, 0xcc, 0x7f, 0xe6, + 0xab, 0x05, 0x46, 0x25, 0x7d, 0xdc, 0x40, 0x1e + } + }, + /* TestSet 3 */ + { + /* Key */ + {0x92, 0xf9, 0x27, 0xe8, 0xab, 0x48, 0x46, 0xdb, + 0x2f, 0xa3, 0x61, 0x36, 0x7e, 0x89, 0xe1, 0x17, + 0xc9, 0x99, 0x57, 0x63, 0xe0, 0xe4, 0x4c, 0xce, + 0x20, 0x03, 0x8a, 0x9c, 0x9a, 0x44, 0xca, 0x64 + }, + /* IV */ + {0x7d, 0x51, 0xfb, 0x42, 0xf8, 0x7e, 0x62, 0xfa, + 0x60, 0x25, 0xb9, 0x2b, 0x4e, 0xd6, 0x1c, 0x2e, + 0xcc, 0x6c, 0x65, 0x18, 0x1e, 0x9d, 0x04 + }, + /* Message length */ + 768, + /* IV length */ + 23, + /* Plaintext */ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + /* Ciphertext */ + {0xe4, 0x14, 0xf9, 0x26, 0x45, 0xc6, 0x2e, 0x12, + 0xb0, 0xe1, 0x33, 0xf6, 0xa7, 0x96, 0x22, 0xfb, + 0x0f, 0xe0, 0x07, 0x5c, 0x6e, 0xbe, 0x10, 0x1b, + 0x37, 0xe4, 0xf7, 0x1c, 0x94, 0xcf, 0xf1, 0x73, + 0x02, 0x64, 0x61, 0x40, 0xb4, 0xf1, 0xfb, 0xcf, + 0x8c, 0xc6, 0xa2, 0xda, 0xd7, 0x3f, 0xb4, 0xcc, + 0xa8, 0x7b, 0x13, 0xaa, 0xd2, 0x6e, 0x2a, 0x1b, + 0x0a, 0x07, 0xef, 0x88, 0x41, 0xfb, 0x6c, 0x10, + 0x3f, 0x41, 0x60, 0xb3, 0x4c, 0x7d, 0x00, 0x9c, + 0x72, 0x2f, 0x4a, 0xa2, 0xc1, 0x0c, 0xf4, 0x6f, + 0xbf, 0xed, 0xec, 0xaf, 0xeb, 0xbc, 0xc8, 0x2a, + 0x54, 0x60, 0x48, 0x7d, 0xfe, 0x20, 0x38, 0x6a + } + }, + /* TestSet 4 */ + { + /* Key */ + {0xa7, 0x72, 0xf5, 0xfe, 0x9d, 0x81, 0xd1, 0xcf, + 0x22, 0x8e, 0x45, 0x53, 0x67, 0x75, 0xac, 0xc9, + 0x04, 0x19, 0x57, 0x55, 0x0f, 0x6c, 0x39, 0xf9, + 0xc5, 0x1b, 0x1e, 0x9e, 0xbb, 0x22, 0xa2, 0xf5 + }, + /* IV */ + {0xce, 0x51, 0x11, 0x83, 0x9b, 0x64, 0x4d, 0x20, + 0x51, 0x92, 0x71, 0x3b, 0x43, 0x47, 0xf9, 0x38, + 0x79, 0x0f, 0xd2, 0x59, 0xbc, 0x35, 0xd3 + }, + /* Message length */ + 384, + /* IV length */ + 23, + /* Plaintext */ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + /* Ciphertext */ + {0x89, 0x63, 0x69, 0xca, 0x77, 0xd3, 0x05, 0xf7, + 0xa3, 0xcb, 0xf6, 0xba, 0xb3, 0x8c, 0x14, 0x4f, + 0xc3, 0x73, 0xf4, 0xf0, 0xf5, 0x0c, 0xf1, 0xad, + 0x0f, 0x41, 0x65, 0x48, 0x40, 0x47, 0x5e, 0xef, + 0xad, 0xea, 0x1c, 0x3c, 0x15, 0xa0, 0xd2, 0x72, + 0x25, 0x14, 0x1d, 0x6f, 0xa0, 0x0f, 0xe8, 0x9a + } + }, + /* TestSet 5 */ + { + /* Key */ + {0xf8, 0xa0, 0x45, 0x4f, 0x6d, 0xea, 0x74, 0x6e, + 0x4c, 0xd1, 0x6e, 0xb0, 0xc3, 0xa2, 0x1f, 0x57, + 0xeb, 0x6f, 0x35, 0x2d, 0x6a, 0x02, 0x5b, 0x35, + 0x32, 0xba, 0x47, 0x3f, 0x1f, 0x0e, 0xdd, 0xc9 + }, + /* IV */ + {0x07, 0x90, 0xeb, 0x7d, 0x09, 0x6d, 0xc1, 0xf1, + 0x86, 0x47, 0xea, 0x57, 0xe4, 0xb8, 0x92, 0xb1, + 0x4e, 0x3b, 0x2d, 0x62, 0xaa, 0x53, 0x6f + }, + /* Message length */ + 384, + /* IV length */ + 23, + /* Plaintext */ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, + /* Ciphertext */ + {0xc6, 0x02, 0x3c, 0x58, 0x53, 0xc9, 0xae, 0xba, + 0x0a, 0x47, 0x10, 0xdc, 0x85, 0x76, 0x13, 0x82, + 0x08, 0x66, 0xbc, 0x3e, 0x9c, 0x2d, 0xb2, 0x42, + 0x20, 0x3a, 0x0a, 0x0c, 0x49, 0x1d, 0xe9, 0x44, + 0x73, 0x87, 0xe5, 0x60, 0x9d, 0x98, 0xff, 0xf1, + 0x33, 0xc3, 0xd0, 0x3d, 0x49, 0xfc, 0x77, 0x07 + } + }, + /* TestSet 6 */ { /* Key */ {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, @@ -562,7 +746,7 @@ const struct test256EEA3_vectors_t test256EEA3_vectors[] = { 0x68, 0x98, 0x94, 0x16, 0xb8, 0xfa, 0xc8, 0xc2 } }, - /* TestSet 3 */ + /* TestSet 7 */ { /* Key */ {0x67, 0xc6, 0x69, 0x73, 0x51, 0xff, 0x4a, 0xec, @@ -605,7 +789,7 @@ const struct test256EEA3_vectors_t test256EEA3_vectors[] = { 0x34, 0xFF, 0x9F, 0xF1, 0x3D, 0x80, 0xE8, 0x54 } }, - /* TestSet 4 */ + /* TestSet 8 */ { /* Key */ {0xC2, 0x54, 0xF8, 0x1B, 0xE8, 0xE7, 0x8D, 0x76, @@ -648,7 +832,7 @@ const struct test256EEA3_vectors_t test256EEA3_vectors[] = { 0x49, 0x37, 0xF9, 0x8A, 0xC6, 0x90, 0x7C, 0x36 } }, - /* TestSet 5 */ + /* TestSet 9 */ { /* Key */ {0x8D, 0x76, 0x5A, 0x2E, 0x63, 0x33, 0x9F, 0xC9, @@ -691,7 +875,7 @@ const struct test256EEA3_vectors_t test256EEA3_vectors[] = { 0xB4, 0x34, 0x5C, 0x5C, 0x0B, 0x48, 0x67, 0x37 } }, - /* TestSet 6 */ + /* TestSet 10 */ { /* Key */ {0x67, 0xc6, 0x69, 0x73, 0x51, 0xff, 0x4a, 0xec, @@ -995,7 +1179,7 @@ const struct test128EIA3_vectors_t testEIA3_vectors[] = { const struct test256EIA3_vectors_t test256EIA3_vectors[] = { { - /*Test 1*/ + /* Test 1 */ /* Key */ {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1025,8 +1209,151 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { 0x70, 0x84, 0xc9, 0x52, 0xa1, 0x65, 0x4b, 0x26 }, }, + /* Test 2 */ + { + /* Key */ + {0x8f, 0x8e, 0xf9, 0xd8, 0xfb, 0x0a, 0xce, 0x2b, + 0x23, 0x19, 0x48, 0x42, 0xcb, 0x5c, 0x6d, 0x98, + 0x1e, 0x71, 0x68, 0x74, 0xe1, 0xdf, 0xeb, 0xe0, + 0xf2, 0x46, 0x02, 0x71, 0xbb, 0x69, 0x0d, 0x9e + }, + /* IV */ + {0x2c, 0xe8, 0x87, 0x0f, 0x8c, 0x7f, 0x47, 0x2a, + 0x02, 0x2d, 0x24, 0xcd, 0x23, 0x3f, 0x4d, 0x0a, + 0x40, 0x0d, 0x12, 0xdd, 0xc4, 0x16, 0x26 + }, + 1600, + /* IV length */ + 23, + {0x05, 0xa8, 0xc3, 0x4b, 0x70, 0x9c, 0x97, 0x71, + 0x67, 0x70, 0xa5, 0xa3, 0x08, 0x60, 0xca, 0x25, + 0x0a, 0x8b, 0xb5, 0xc1, 0xc9, 0xd5, 0x8c, 0x7d, + 0xfb, 0x00, 0x3b, 0xc0, 0x9d, 0xe1, 0x09, 0x9f, + 0xcc, 0x22, 0x8c, 0xf6, 0x12, 0x6f, 0xb9, 0x1e, + 0xc9, 0x45, 0x43, 0x43, 0x25, 0x7a, 0x2b, 0xba, + 0x64, 0x4b, 0x8c, 0x91, 0x77, 0xc8, 0xfd, 0xce, + 0x01, 0xcf, 0xab, 0x6b, 0xe6, 0xc2, 0x48, 0x80, + 0x82, 0x77, 0xad, 0xb8, 0xb9, 0x8d, 0x1f, 0xd7, + 0x48, 0x0b, 0x73, 0x4d, 0x98, 0x96, 0x12, 0xd5, + 0xf1, 0x86, 0xfd, 0xa1, 0x12, 0x50, 0x9a, 0x38, + 0x07, 0x37, 0xd5, 0xa3, 0xd0, 0x21, 0xfe, 0x55, + 0x7a, 0x8f, 0xff, 0xe0, 0x4f, 0x25, 0x9c, 0x73, + 0x01, 0x06, 0x66, 0xff, 0x10, 0xa4, 0xdd, 0xd4, + 0x2a, 0xbf, 0x0f, 0x5a, 0xa2, 0x29, 0x64, 0xd9, + 0x99, 0xc8, 0x46, 0xe6, 0x46, 0x48, 0x4d, 0x56, + 0xe9, 0x02, 0x17, 0xa8, 0x14, 0x28, 0x13, 0x22, + 0xf0, 0xd4, 0x43, 0xbe, 0xa0, 0x64, 0xd5, 0x28, + 0x99, 0x27, 0x24, 0x5d, 0x7c, 0x25, 0x46, 0xd6, + 0xdf, 0x2c, 0x05, 0x70, 0x5a, 0x55, 0xcd, 0xf6, + 0xe7, 0xdb, 0x3d, 0x94, 0x67, 0xfa, 0x67, 0x15, + 0xe3, 0x84, 0x96, 0x26, 0xee, 0xf4, 0x22, 0xaf, + 0x2f, 0xa4, 0x6e, 0xda, 0x2f, 0x4a, 0xa0, 0xcd, + 0x10, 0x72, 0x85, 0xb6, 0x45, 0x3b, 0x22, 0xb8, + 0x1f, 0xe0, 0x3c, 0xf9, 0x64, 0x29, 0xb4, 0x46 + }, + {0x8d, 0x74, 0x8b, 0x71}, + {0xe5, 0x1d, 0xf2, 0x0a, + 0x9e, 0x74, 0x06, 0xac}, + {0x4d, 0x40, 0x5d, 0x6e, + 0xf7, 0xf8, 0xaf, 0xad, + 0xd6, 0x71, 0x71, 0x03, + 0xdf, 0x92, 0x28, 0x20}, + }, + { + /* Test 3 */ + /* Key */ + {0x92, 0xf9, 0x27, 0xe8, 0xab, 0x48, 0x46, 0xdb, + 0x2f, 0xa3, 0x61, 0x36, 0x7e, 0x89, 0xe1, 0x17, + 0xc9, 0x99, 0x57, 0x63, 0xe0, 0xe4, 0x4c, 0xce, + 0x20, 0x03, 0x8a, 0x9c, 0x9a, 0x44, 0xca, 0x64 + }, + /* IV */ + {0x7d, 0x51, 0xfb, 0x42, 0xf8, 0x7e, 0x62, 0xfa, + 0x60, 0x25, 0xb9, 0x2b, 0x4e, 0xd6, 0x1c, 0x2e, + 0xcc, 0x6c, 0x65, 0x18, 0x1e, 0x9d, 0x04 + }, + 800, + /* IV length */ + 23, + {0xf6, 0x6e, 0x21, 0x54, 0xb9, 0x60, 0xb5, 0x90, + 0xdc, 0x35, 0xaf, 0xb9, 0x9d, 0x03, 0xf9, 0xbe, + 0x58, 0xf8, 0x7c, 0x5c, 0x03, 0xdb, 0x72, 0x2e, + 0xa6, 0x34, 0xff, 0x43, 0x8d, 0xcf, 0xd4, 0xa7, + 0x2a, 0x52, 0xae, 0x3a, 0xb8, 0xc7, 0x11, 0xd8, + 0x19, 0xd3, 0x94, 0x66, 0x84, 0x10, 0xf0, 0x81, + 0x45, 0xc5, 0x0b, 0x05, 0xe6, 0x89, 0xc6, 0xc9, + 0xb4, 0x25, 0x7b, 0xb7, 0x89, 0x42, 0xd4, 0x6c, + 0x1a, 0xfd, 0x00, 0x42, 0x80, 0x9d, 0x10, 0x5e, + 0x68, 0xd6, 0x02, 0x13, 0x07, 0x55, 0x08, 0x24, + 0xe5, 0x9e, 0x6b, 0xf3, 0xea, 0x04, 0xd7, 0xd7, + 0x8f, 0x0a, 0x48, 0x02, 0x8c, 0x98, 0xd5, 0x68, + 0xee, 0x11, 0x93, 0x22 + }, + {0xf2, 0xef, 0xfb, 0xf7}, + {0x5a, 0x99, 0xe9, 0x23, 0xfa, 0xf1, 0xec, 0xbb}, + {0x0b, 0xfb, 0x8f, 0xff, 0x65, 0xaf, 0x6a, 0x69, + 0xea, 0xde, 0xbd, 0x94, 0x79, 0x7b, 0x08, 0xa5}, + }, { - /*Test 2*/ + /* Test 4 */ + /* Key */ + {0xa7, 0x72, 0xf5, 0xfe, 0x9d, 0x81, 0xd1, 0xcf, + 0x22, 0x8e, 0x45, 0x53, 0x67, 0x75, 0xac, 0xc9, + 0x04, 0x19, 0x57, 0x55, 0x0f, 0x6c, 0x39, 0xf9, + 0xc5, 0x1b, 0x1e, 0x9e, 0xbb, 0x22, 0xa2, 0xf5 + }, + /* IV */ + {0xce, 0x51, 0x11, 0x83, 0x9b, 0x64, 0x4d, 0x20, + 0x51, 0x92, 0x71, 0x3b, 0x43, 0x47, 0xf9, 0x38, + 0x79, 0x0f, 0xd2, 0x59, 0xbc, 0x35, 0xd3 + }, + 400, + /* IV length */ + 23, + {0xad, 0x2a, 0x9a, 0x9a, 0x7e, 0xb8, 0xb9, 0x72, + 0xc5, 0x50, 0xe8, 0x28, 0x5e, 0x17, 0xda, 0xa2, + 0x4c, 0x9a, 0xeb, 0x61, 0x72, 0xc6, 0xa7, 0x06, + 0x54, 0x32, 0xa6, 0x5b, 0xc5, 0x8b, 0xd7, 0xa2, + 0xe0, 0x5e, 0x18, 0xf5, 0x41, 0x8e, 0xa8, 0x6e, + 0x50, 0xd9, 0xde, 0x67, 0x28, 0x04, 0xee, 0x22, + 0xa5, 0x72 + }, + {0x18, 0x3d, 0xf5, 0xf7}, + {0x50, 0xa0, 0xc4, 0x24, 0xa9, 0x25, 0xf4, 0x5b}, + {0x4b, 0xde, 0xd3, 0x15, 0xa4, 0x80, 0xa7, 0xe4, + 0x02, 0xe4, 0xc4, 0x48, 0x90, 0xfe, 0x62, 0xf5}, + }, + { + /* Test 5 */ + /* Key */ + {0xf8, 0xa0, 0x45, 0x4f, 0x6d, 0xea, 0x74, 0x6e, + 0x4c, 0xd1, 0x6e, 0xb0, 0xc3, 0xa2, 0x1f, 0x57, + 0xeb, 0x6f, 0x35, 0x2d, 0x6a, 0x02, 0x5b, 0x35, + 0x32, 0xba, 0x47, 0x3f, 0x1f, 0x0e, 0xdd, 0xc9 + }, + /* IV */ + {0x07, 0x90, 0xeb, 0x7d, 0x09, 0x6d, 0xc1, 0xf1, + 0x86, 0x47, 0xea, 0x57, 0xe4, 0xb8, 0x92, 0xb1, + 0x4e, 0x3b, 0x2d, 0x62, 0xaa, 0x53, 0x6f + }, + 400, + /* IV length */ + 23, + {0xf7, 0xc3, 0xc4, 0x82, 0xe7, 0x20, 0x76, 0xa2, + 0x78, 0x5d, 0xe1, 0xcb, 0xa5, 0x3f, 0x7d, 0x7c, + 0xa3, 0x84, 0x0b, 0x69, 0xff, 0x3b, 0x19, 0xb5, + 0x6b, 0x9f, 0x25, 0x04, 0x35, 0xad, 0x89, 0x3b, + 0xad, 0xba, 0xa5, 0xe1, 0xe8, 0x4e, 0xa4, 0xf5, + 0x49, 0x84, 0x9a, 0x2c, 0x71, 0xb1, 0xd6, 0xc1, + 0x1b, 0xdc + }, + {0xb8, 0xbf, 0x02, 0x13}, + {0x2c, 0xfc, 0xa3, 0x59, 0x33, 0x50, 0xd8, 0xee}, + {0x9c, 0x6e, 0xd5, 0xf9, 0x63, 0x95, 0xa7, 0x28, + 0xe1, 0x6b, 0xb2, 0x8a, 0x0e, 0x5b, 0x90, 0x72} + }, + { + /* Test 6 */ /* Key */ {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1113,7 +1440,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { }, }, { - /*Test 3*/ + /* Test 7 */ /* Key */ {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -1144,7 +1471,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { }, }, { - /*Test 4*/ + /* Test 8 */ /* Key */ {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -1231,7 +1558,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { }, }, { - /*Test 5*/ + /* Test 9 */ /* Key */ {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -1261,7 +1588,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { }, }, { - /*Test 6*/ + /* Test 10 */ /* Key */ {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1290,7 +1617,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { } }, { - /*Test 7*/ + /* Test 11 */ /* Key */ {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -1319,7 +1646,7 @@ const struct test256EIA3_vectors_t test256EIA3_vectors[] = { } }, { - /*Test 8*/ + /* Test 12 */ /* Key */ {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -- GitLab From dfb62484a0e0464fb9b501d47e03d7c98e494942 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Sat, 19 Mar 2022 09:13:52 +0000 Subject: [PATCH 034/369] test: [ZUC-EIA3-256] Test mixed vectors for tag sizes of 8 and 16 bytes Test several ZUC-EIA3-256 test vectors simulteanously, for tag sizes of 8 and 16 bytes, apart from the previous 4-byte tags. --- test/zuc_test.c | 74 ++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/test/zuc_test.c b/test/zuc_test.c index 82892a4a..1b150da6 100644 --- a/test/zuc_test.c +++ b/test/zuc_test.c @@ -1017,7 +1017,8 @@ int validate_zuc_EIA_n_block(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, static int verify_tag_256(void *mac, const struct test256EIA3_vectors_t *vector, - const unsigned tag_sz, const uint32_t i, const uint32_t j) + const unsigned tag_sz, const uint32_t test_idx, + const uint32_t vector_idx, const int multi_vector) { const void *ref_mac = NULL; int ret; @@ -1031,19 +1032,34 @@ verify_tag_256(void *mac, const struct test256EIA3_vectors_t *vector, ret = memcmp(mac, ref_mac, tag_sz); if (ret) { - printf("Validate ZUC-256 n block test %u, " - "index %u (Int - %u bytes): FAIL\n", - i + 1, j, tag_sz); + if (multi_vector) { + printf("Validate ZUC-256 n block multi-vector test " + "# jobs = %u, index %u (Int - %u bytes): FAIL\n", + test_idx, vector_idx, tag_sz); + + } else { + printf("Validate ZUC-256 n block test %u, " + "index %u (Int - %u bytes): FAIL\n", + test_idx + 1, vector_idx, tag_sz); + } byte_hexdump("Expected", (const uint8_t *)ref_mac, tag_sz); byte_hexdump("Found", mac, tag_sz); } #ifdef DEBUG - else - printf("Validate ZUC-256 n block test %u, " - "index %u (Int - %u bytes): PASS\n", - i + 1, j, tag_sz); + else { + if (multi_vector) { + printf("Validate ZUC-256 n block multi-vector test " + "# jobs = %u, index %u (Int - %u bytes): PASS\n", + test_idx, vector_idx, tag_sz); + + } else { + printf("Validate ZUC-256 n block test %u, " + "index %u (Int - %u bytes): PASS\n", + test_idx + 1, vector_idx, tag_sz); + } + } #endif fflush(stdout); @@ -1081,7 +1097,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, for (j = 0; j < numBuffs; j++) { retTmp = verify_tag_256(pDstData[j], vector, - tag_sz, i, j); + tag_sz, i, j, 0); if (retTmp) ret = retTmp; } @@ -1101,34 +1117,22 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, iv_lens[i] = vector->iv_length; } - submit_eia3_jobs(mb_mgr, pKeys, pIV, - pSrcData, pDstData, - bitLength, numBuffs, - ZUC256_KEY_LEN_IN_BYTES, ZUC_DIGEST_LEN, - iv_lens); + for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { + submit_eia3_jobs(mb_mgr, pKeys, pIV, + pSrcData, pDstData, + bitLength, numBuffs, + ZUC256_KEY_LEN_IN_BYTES, tag_sz, + iv_lens); - for (i = 0; i < numBuffs; i++) { - vector = &test256EIA3_vectors[i % NUM_ZUC_256_EIA3_TESTS]; - retTmp = - memcmp(pDstData[i], &vector->mac4, - sizeof(((struct test256EIA3_vectors_t *)0)->mac4)); - if (retTmp) { - printf("Validate ZUC-256 n block multi-vector test " - "# jobs = %u, index %u (Int): FAIL\n", - numBuffs, i); - byte_hexdump("Expected", - (const uint8_t *)&vector->mac4, - ZUC_DIGEST_LEN); - byte_hexdump("Found", pDstData[i], ZUC_DIGEST_LEN); - ret = retTmp; + for (i = 0; i < numBuffs; i++) { + const uint32_t vector_idx = i % NUM_ZUC_256_EIA3_TESTS; + + vector = &test256EIA3_vectors[vector_idx]; + retTmp = verify_tag_256(pDstData[i], vector, + tag_sz, numBuffs, i, 1); + if (retTmp) + ret = retTmp; } -#ifdef DEBUG - else - printf("Validate ZUC-256 n block multi-vector test, " - "# jobs = %u, index %u (Int): PASS\n", - numBuffs, i); -#endif - fflush(stdout); } return ret; }; -- GitLab From 3481f2a541ffdb9f8468b48b2cb586fbbae384fe Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 11 Mar 2022 15:02:18 +0000 Subject: [PATCH 035/369] lib: [ghash] change key pointer to const in GHASH function prototype --- lib/include/gcm.h | 12 ++++++------ lib/intel-ipsec-mb.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/include/gcm.h b/lib/include/gcm.h index f5a9618f..2cbc9d42 100644 --- a/lib/include/gcm.h +++ b/lib/include/gcm.h @@ -844,25 +844,25 @@ aes_gcm_dec_var_iv_256_sse_no_aesni(const struct gcm_key_data *key_data, */ IMB_DLL_EXPORT void -ghash_sse_no_aesni(struct gcm_key_data *key_data, const void *in, +ghash_sse_no_aesni(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); IMB_DLL_EXPORT void -ghash_sse(struct gcm_key_data *key_data, const void *in, +ghash_sse(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); IMB_DLL_EXPORT void -ghash_avx_gen2(struct gcm_key_data *key_data, const void *in, +ghash_avx_gen2(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); IMB_DLL_EXPORT void -ghash_avx_gen4(struct gcm_key_data *key_data, const void *in, +ghash_avx_gen4(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); IMB_DLL_EXPORT void -ghash_avx512(struct gcm_key_data *key_data, const void *in, +ghash_avx512(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); IMB_DLL_EXPORT void -ghash_vaes_avx512(struct gcm_key_data *key_data, const void *in, +ghash_vaes_avx512(const struct gcm_key_data *key_data, const void *in, const uint64_t in_len, void *io_tag, const uint64_t tag_len); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 06c2529d..1a80ac7f 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -807,7 +807,7 @@ typedef void (*chacha_poly_enc_dec_update_t)(const void *, void *, const void *, const uint64_t); typedef void (*chacha_poly_finalize_t)(struct chacha20_poly1305_context_data *, void *, const uint64_t); -typedef void (*ghash_t)(struct gcm_key_data *, const void *, +typedef void (*ghash_t)(const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t); typedef void (*zuc_eea3_1_buffer_t)(const void *, const void *, const void *, -- GitLab From 4b94b2baf58b8fca4a70472b84f30811ce9834e0 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 11 Mar 2022 15:04:14 +0000 Subject: [PATCH 036/369] lib: [ghash] add GHASH support in JOB API - add authentication enum - add authentication structure on IMB_JOB - add GHASH dispatch code - add GHASH specific error codes - update API test to cover GHASH JOB API checks --- ReleaseNotes.txt | 1 + lib/include/mb_mgr_code.h | 46 +++++++++++++++++++++++++++++++++++++-- lib/intel-ipsec-mb.h | 7 ++++++ lib/x86_64/error.c | 5 ++++- test/api_test.c | 40 ++++++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 3 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 2fa9d34f..e6e5ada9 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -7,6 +7,7 @@ Unreleased Library - ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE and AVX - AES-ECB AVX512-VAES implementation added +- JOB API GHASH support added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index f7ff69ad..0955f0c4 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -973,6 +973,22 @@ process_gmac(IMB_MGR *state, IMB_JOB *job, const IMB_KEY_SIZE_BYTES key_size) } } +__forceinline IMB_JOB *process_ghash(IMB_MGR *state, IMB_JOB *job) +{ + /* copy initial tag value to the destination */ + memcpy(job->auth_tag_output, job->u.GHASH._init_tag, + job->auth_tag_output_len_in_bytes); + + /* compute new tag value */ + IMB_GHASH(state, job->u.GHASH._key, + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bytes, + job->auth_tag_output, job->auth_tag_output_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; +} + __forceinline IMB_JOB * SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) @@ -1166,11 +1182,13 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) CRC(IMB_CRC6_IUUP_HEADER, state, job); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; + case IMB_AUTH_GHASH: + return process_ghash(state, job); + default: /** * assume IMB_AUTH_GCM, IMB_AUTH_PON_CRC_BIP, - * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL + * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL */ - default: job->status |= IMB_STATUS_COMPLETED_AUTH; return job; } @@ -2265,6 +2283,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->u.GMAC._key == NULL) { + /* @todo change to IMB_ERR_JOB_NULL_AUTH_KEY */ imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } @@ -2281,6 +2300,29 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } break; + case IMB_AUTH_GHASH: + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->u.GHASH._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); + return 1; + } + if (job->u.GHASH._init_tag == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_GHASH_INIT_TAG); + return 1; + } + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + break; case IMB_AUTH_CUSTOM: if (job->hash_func == NULL) { imb_set_errno(state, EFAULT); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 1a80ac7f..22ab598e 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -223,6 +223,7 @@ typedef enum { IMB_ERR_JOB_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K3, IMB_ERR_JOB_CIPH_DIR, + IMB_ERR_JOB_NULL_GHASH_INIT_TAG, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; @@ -429,6 +430,7 @@ typedef enum { IMB_AUTH_CRC8_WIMAX_OFDMA_HCS, /**< CRC8-WIMAX-OFDMA-HCS */ IMB_AUTH_CRC7_FP_HEADER, /**< CRC7-FP-HEADER */ IMB_AUTH_CRC6_IUUP_HEADER, /**< CRC6-IUUP-HEADER */ + IMB_AUTH_GHASH, /**< GHASH */ IMB_AUTH_NUM } IMB_HASH_ALG; @@ -565,6 +567,11 @@ typedef struct IMB_JOB { uint64_t iv_len_in_bytes; /**< Authentication IV length in bytes */ } GMAC; /**< AES-GMAC specific fields */ + struct _GHASH_specific_fields { + const struct gcm_key_data *_key; + /**< Expanded GHASH key */ + const void *_init_tag; /**< initial tag value */ + } GHASH; /**< GHASH specific fields */ struct _POLY1305_specific_fields { const void *_key; /**< Poly1305 key */ diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index 96f4e8ee..8ae46d27 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -84,7 +84,8 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_JOB_NULL_XCBC_K1_EXP, IMB_ERR_JOB_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K3, - IMB_ERR_JOB_CIPH_DIR + IMB_ERR_JOB_CIPH_DIR, + IMB_ERR_JOB_NULL_GHASH_INIT_TAG }; #ifdef DEBUG @@ -163,6 +164,8 @@ imb_get_strerror(int errnum) return "Null pointer to XCBC K2"; case IMB_ERR_JOB_NULL_XCBC_K3: return "Null pointer to XCBC K3"; + case IMB_ERR_JOB_NULL_GHASH_INIT_TAG: + return "Null pointer to GHASH initial tag value"; case IMB_ERR_NULL_SRC: return "Null source pointer (direct API)"; case IMB_ERR_NULL_DST: diff --git a/test/api_test.c b/test/api_test.c index 184a831b..844c1f50 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -54,6 +54,8 @@ enum { TEST_AUTH_NULL_XCBC_K1_EXP, TEST_AUTH_NULL_XCBC_K2, TEST_AUTH_NULL_XCBC_K3, + TEST_AUTH_NULL_GHASH_KEY, + TEST_AUTH_NULL_GHASH_INIT_TAG, TEST_CIPH_SRC_NULL = 200, TEST_CIPH_DST_NULL, TEST_CIPH_IV_NULL, @@ -228,6 +230,7 @@ fill_in_job(struct IMB_JOB *job, 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ 4, /* IMB_AUTH_CRC7_FP_HEADER */ 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; static DECLARE_ALIGNED(uint8_t dust_bin[2048], 64); static void *ks_ptrs[3]; @@ -484,6 +487,11 @@ fill_in_job(struct IMB_JOB *job, job->u.GMAC.iv_len_in_bytes = 12; job->auth_tag_output_len_in_bytes = 16; break; + case IMB_AUTH_GHASH: + job->u.GHASH._key = (struct gcm_key_data *) dust_bin; + job->u.GHASH._init_tag = dust_bin; + job->auth_tag_output_len_in_bytes = 16; + break; case IMB_AUTH_POLY1305: job->u.POLY1305._key = dust_bin; job->auth_tag_output_len_in_bytes = 16; @@ -792,6 +800,7 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) hash == IMB_AUTH_CRC8_WIMAX_OFDMA_HCS || hash == IMB_AUTH_CRC7_FP_HEADER || hash == IMB_AUTH_CRC6_IUUP_HEADER || + hash == IMB_AUTH_GHASH || hash == IMB_AUTH_POLY1305) continue; @@ -1022,6 +1031,37 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) printf("."); } + /* + * Invalid GHASH parameters + */ + for (order = IMB_ORDER_CIPHER_HASH; order <= IMB_ORDER_HASH_CIPHER; + order++) + for (dir = IMB_DIR_ENCRYPT; dir <= IMB_DIR_DECRYPT; dir++) { + IMB_JOB *job = &template_job; + + hash = IMB_AUTH_GHASH; + + fill_in_job(job, cipher, dir, + hash, order, &chacha_ctx, + &gcm_ctx); + job->u.GHASH._key = NULL; + if (!is_submit_invalid(mb_mgr, job, + TEST_AUTH_NULL_GHASH_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) + return 1; + printf("."); + + fill_in_job(job, cipher, dir, + hash, order, &chacha_ctx, + &gcm_ctx); + job->u.GHASH._init_tag = NULL; + if (!is_submit_invalid(mb_mgr, job, + TEST_AUTH_NULL_GHASH_INIT_TAG, + IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) + return 1; + printf("."); + } + /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; -- GitLab From 173bc1bb48cbfee9d2901f9f9e04982cd295d6f5 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 11 Mar 2022 15:06:31 +0000 Subject: [PATCH 037/369] test: [ghash] enable JOB API GHASH tests --- ReleaseNotes.txt | 3 +++ test/api_test.c | 8 +++++--- test/gcm_test.c | 48 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index e6e5ada9..f6b8fe64 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -12,6 +12,9 @@ Library Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) +Test Applications +- GHASH JOB API support added in the test application + v1.2 February 2022 ======================================================================== diff --git a/test/api_test.c b/test/api_test.c index 844c1f50..e6d4ecdb 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -800,8 +800,8 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) hash == IMB_AUTH_CRC8_WIMAX_OFDMA_HCS || hash == IMB_AUTH_CRC7_FP_HEADER || hash == IMB_AUTH_CRC6_IUUP_HEADER || - hash == IMB_AUTH_GHASH || - hash == IMB_AUTH_POLY1305) + hash == IMB_AUTH_POLY1305 || + hash == IMB_AUTH_GHASH) continue; /* @@ -1759,7 +1759,9 @@ submit_reset_check_job(struct IMB_MGR *mb_mgr, if (next_job->status != IMB_STATUS_COMPLETED) { printf("Returned job's status is not completed\n"); printf("cipher = %u\n", cipher); - printf("imb errno = %u\n", mb_mgr->imb_errno); + printf("imb errno = %u (%s)\n", + mb_mgr->imb_errno, + imb_get_strerror(mb_mgr->imb_errno)); exit(0); } diff --git a/test/gcm_test.c b/test/gcm_test.c index 7268c9e0..2dccd934 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -2014,12 +2014,13 @@ test_gcm_std_vectors(struct test_suite_context *ts128, } static void -test_ghash(struct test_suite_context *ts) +test_ghash(struct test_suite_context *ts, const int use_job_api) { const int vectors_cnt = DIM(ghash_vectors); int vect; - printf("GHASH test vectors:\n"); + printf("GHASH test vectors (%s API):\n", + use_job_api ? "job" : "direct"); for (vect = 0; vect < vectors_cnt; vect++) { struct gcm_key_data gdata_key; struct gcm_ctr_vector const *vector = &ghash_vectors[vect]; @@ -2028,8 +2029,44 @@ test_ghash(struct test_suite_context *ts) memset(&gdata_key, 0, sizeof(struct gcm_key_data)); memset(T_test, 0, sizeof(T_test)); IMB_GHASH_PRE(p_gcm_mgr, vector->K, &gdata_key); - IMB_GHASH(p_gcm_mgr, &gdata_key, vector->P, vector->Plen, - T_test, vector->Tlen); + + if (!use_job_api) { + IMB_GHASH(p_gcm_mgr, &gdata_key, vector->P, + vector->Plen, T_test, vector->Tlen); + } else { + IMB_JOB *job = IMB_GET_NEXT_JOB(p_gcm_mgr); + + if (!job) { + fprintf(stderr, + "failed to get job for ghash\n"); + return; + } + + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_GHASH; + job->u.GHASH._key = &gdata_key; + job->u.GHASH._init_tag = T_test; + job->src = vector->P; + job->msg_len_to_hash_in_bytes = vector->Plen; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->auth_tag_output = T_test; + job->auth_tag_output_len_in_bytes = vector->Tlen; + + job = IMB_SUBMIT_JOB(p_gcm_mgr); + while (job) { + if (job->status != IMB_STATUS_COMPLETED) + fprintf(stderr, + "failed job, status:%d\n", + job->status); + job = IMB_GET_COMPLETED_JOB(p_gcm_mgr); + } + while ((job = IMB_FLUSH_JOB(p_gcm_mgr)) != NULL) { + if (job->status != IMB_STATUS_COMPLETED) + fprintf(stderr, + "failed job, status:%d\n", + job->status); + } + } if (check_data(T_test, vector->T, vector->Tlen, "generated tag (T)")) @@ -2524,7 +2561,8 @@ int gcm_test(IMB_MGR *p_mgr) errors += test_suite_end(&ts256); test_suite_start(&ts128, "GHASH"); - test_ghash(&ts128); + test_ghash(&ts128, 0); + test_ghash(&ts128, 1); errors += test_suite_end(&ts128); return errors; -- GitLab From 0e3aa644e813aea1ad884f7422f6461406e7d558 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 11 Mar 2022 15:08:36 +0000 Subject: [PATCH 038/369] perf: [ghash] add support for GHASH performance testing - fix missing algorithm names in h_alg_names[] --- ReleaseNotes.txt | 3 +++ perf/ipsec_perf.c | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index f6b8fe64..68a35042 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -15,6 +15,9 @@ Fixes Test Applications - GHASH JOB API support added in the test application +Performance Application +- GHASH support added (through JOB API) + v1.2 February 2022 ======================================================================== diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 190a56b4..4f1a41e1 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -187,6 +187,7 @@ enum test_hash_alg_e { TEST_CRC8_WIMAX_OFDMA_HCS, TEST_CRC7_FP_HEADER, TEST_CRC6_IUUP_HEADER, + TEST_AUTH_GHASH, TEST_NUM_HASH_TESTS }; @@ -654,6 +655,12 @@ const struct str_value_mapping hash_algo_str_map[] = { .hash_alg = TEST_CRC6_IUUP_HEADER, } }, + { + .name = "ghash", + .values.job_params = { + .hash_alg = TEST_AUTH_GHASH, + } + }, }; const struct str_value_mapping aead_algo_str_map[] = { @@ -835,6 +842,7 @@ const uint32_t auth_tag_length_bytes[] = { 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ 4, /* IMB_AUTH_CRC7_FP_HEADER */ 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; uint32_t index_limit; uint32_t key_idxs[NUM_OFFSETS]; @@ -1705,6 +1713,12 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.u.GMAC._iv = (uint8_t *) &auth_iv; job_template.u.GMAC.iv_len_in_bytes = 12; break; + case TEST_AUTH_GHASH: + job_template.hash_alg = IMB_AUTH_GHASH; + IMB_GHASH_PRE(mb_mgr, gcm_key, &gdata_key); + job_template.u.GHASH._key = &gdata_key; + job_template.u.GHASH._init_tag = (uint8_t *) &auth_iv; + break; case TEST_AUTH_SNOW_V_AEAD: job_template.hash_alg = IMB_AUTH_SNOW_V_AEAD; break; @@ -2291,7 +2305,11 @@ print_times(struct variant_s *variant_list, struct params_s *params, "BIP-CRC32", "ZUC_EIA3_BITLEN", "SNOW3G_UIA2_BITLEN", "KASUMI_UIA1", "GMAC-128", "GMAC-192", "GMAC-256", "POLY1305", "POLY1305_AEAD", "ZUC256_EIA3", - "SNOW_V_AEAD" + "SNOW_V_AEAD", "CRC32_ETH_FCS", "CRC32_SCTP", + "CRC32_WIMAX_DATA", "CRC24_LTE_A", "CR24_LTE_B", + "CR16_X25", "CRC16_FP_DATA", "CRC11_FP_HEADER", + "CRC10_IUUP_DATA", "CRC8_WIMAX_HCS", "CRC7_FP_HEADER", + "CRC6_IUUP_HEADER", "GHASH" }; struct params_s par; -- GitLab From 3655442a3847fb986990fdeb84a7338a742fc462 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 11 Mar 2022 15:09:35 +0000 Subject: [PATCH 039/369] test: [ghash] support for fuzzing GHASH JOB API --- ReleaseNotes.txt | 2 +- test/job_api_fuzz_test.c | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 68a35042..545f253d 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -13,7 +13,7 @@ Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) Test Applications -- GHASH JOB API support added in the test application +- GHASH JOB API support added in the test application and fuzzing tool Performance Application - GHASH support added (through JOB API) diff --git a/test/job_api_fuzz_test.c b/test/job_api_fuzz_test.c index 280a56b6..606097a3 100644 --- a/test/job_api_fuzz_test.c +++ b/test/job_api_fuzz_test.c @@ -225,6 +225,12 @@ static void fill_additional_hash_data(struct IMB_JOB *job, if (job->u.GMAC.iv_len_in_bytes > buffsize) job->u.GMAC.iv_len_in_bytes = buffsize; break; + case IMB_AUTH_GHASH: + if (job->u.GHASH._key != NULL) + job->u.GHASH._key = buff; + if (job->u.GHASH._init_tag != NULL) + job->u.GHASH._init_tag = buff; + break; case IMB_AUTH_POLY1305: if (job->u.POLY1305._key != NULL) job->u.POLY1305._key = buff; @@ -373,6 +379,8 @@ static IMB_HASH_ALG hash_selection(void) return IMB_AUTH_CRC7_FP_HEADER; else if (strcmp(a, "IMB_AUTH_CRC6_IUUP_HEADER") == 0) return IMB_AUTH_CRC6_IUUP_HEADER; + else if (strcmp(a, "IMB_AUTH_GHASH") == 0) + return IMB_AUTH_GHASH; else return 0; } -- GitLab From 3076d7da348425f02e570be7ce7389ca564c58ff Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 14 Mar 2022 08:28:09 +0000 Subject: [PATCH 040/369] test: [ghash] add ghash support in xvalid tool --- ReleaseNotes.txt | 2 +- test/ipsec_xvalid.c | 20 +++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 545f253d..f0915de0 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -13,7 +13,7 @@ Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) Test Applications -- GHASH JOB API support added in the test application and fuzzing tool +- GHASH JOB API support added in the test application, fuzzing and xvalid tools Performance Application - GHASH support added (through JOB API) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index f590cfa1..7d194178 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -488,6 +488,12 @@ struct str_value_mapping hash_algo_str_map[] = { .hash_alg = IMB_AUTH_ZUC256_EIA3_BITLEN, } }, + { + .name = "GHASH", + .values.job_params = { + .hash_alg = IMB_AUTH_GHASH, + } + }, }; struct str_value_mapping aead_algo_str_map[] = { @@ -616,6 +622,7 @@ const uint8_t auth_tag_length_bytes[] = { 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ 4, /* IMB_AUTH_CRC7_FP_HEADER */ 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; /* Minimum, maximum and step values of key sizes */ @@ -990,6 +997,10 @@ fill_job(IMB_JOB *job, const struct params_s *params, job->u.GMAC._iv = auth_iv; job->u.GMAC.iv_len_in_bytes = 12; break; + case IMB_AUTH_GHASH: + job->u.GHASH._key = gdata_key; + job->u.GHASH._init_tag = auth_iv; + break; case IMB_AUTH_PON_CRC_BIP: case IMB_AUTH_NULL: case IMB_AUTH_AES_GMAC: @@ -1248,6 +1259,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_AES_GMAC_128: case IMB_AUTH_AES_GMAC_192: case IMB_AUTH_AES_GMAC_256: + case IMB_AUTH_GHASH: memset(gdata_key, pattern_auth_key, sizeof(keys->gdata_key)); break; @@ -1417,6 +1429,9 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_AES_GMAC_256: IMB_AES256_GCM_PRE(mb_mgr, auth_key, gdata_key); break; + case IMB_AUTH_GHASH: + IMB_GHASH_PRE(mb_mgr, auth_key, gdata_key); + break; case IMB_AUTH_AES_CCM: case IMB_AUTH_AES_GMAC: case IMB_AUTH_NULL: @@ -2456,11 +2471,6 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, if (c_mode != IMB_CIPHER_DOCSIS_SEC_BPI && hash_alg == IMB_AUTH_DOCSIS_CRC32) continue; - if (c_mode == IMB_CIPHER_GCM && - (hash_alg == IMB_AUTH_AES_GMAC_128 || - hash_alg == IMB_AUTH_AES_GMAC_192 || - hash_alg == IMB_AUTH_AES_GMAC_256)) - continue; if ((c_mode == IMB_CIPHER_CHACHA20_POLY1305 && hash_alg != IMB_AUTH_CHACHA20_POLY1305) || (c_mode != IMB_CIPHER_CHACHA20_POLY1305 && -- GitLab From b945500a9b27738d826ae2937c5667d713a8869e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 14 Mar 2022 12:10:04 +0000 Subject: [PATCH 041/369] lib: [gmac] change IMB_ERR_JOB_NULL_KEY error code to IMB_ERR_JOB_NULL_AUTH_KEY --- lib/include/mb_mgr_code.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 0955f0c4..cd584fe7 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2283,8 +2283,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->u.GMAC._key == NULL) { - /* @todo change to IMB_ERR_JOB_NULL_AUTH_KEY */ - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); return 1; } if (job->u.GMAC._iv == NULL) { -- GitLab From 186daa85922a2cf217529b6a012928a7d860ca2e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 14 Mar 2022 12:10:39 +0000 Subject: [PATCH 042/369] test: [gmac] add GMAC specific fields API tests --- test/api_test.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/api_test.c b/test/api_test.c index e6d4ecdb..7baf5b55 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -56,6 +56,9 @@ enum { TEST_AUTH_NULL_XCBC_K3, TEST_AUTH_NULL_GHASH_KEY, TEST_AUTH_NULL_GHASH_INIT_TAG, + TEST_AUTH_NULL_GMAC_KEY, + TEST_AUTH_NULL_GMAC_IV, + TEST_AUTH_GMAC_IV_LEN, TEST_CIPH_SRC_NULL = 200, TEST_CIPH_DST_NULL, TEST_CIPH_IV_NULL, @@ -1062,6 +1065,49 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) printf("."); } + /* + * Invalid GMAC parameters + */ + for (order = IMB_ORDER_CIPHER_HASH; order <= IMB_ORDER_HASH_CIPHER; + order++) + for (dir = IMB_DIR_ENCRYPT; dir <= IMB_DIR_DECRYPT; dir++) { + for (hash = IMB_AUTH_AES_GMAC_128; + hash <= IMB_AUTH_AES_GMAC_256; hash++) { + IMB_JOB *job = &template_job; + + fill_in_job(job, cipher, dir, + hash, order, &chacha_ctx, + &gcm_ctx); + job->u.GMAC._key = NULL; + + if (!is_submit_invalid(mb_mgr, job, + TEST_AUTH_NULL_GMAC_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) + return 1; + printf("."); + + fill_in_job(job, cipher, dir, + hash, order, &chacha_ctx, + &gcm_ctx); + job->u.GMAC._iv = NULL; + if (!is_submit_invalid(mb_mgr, job, + TEST_AUTH_NULL_GMAC_IV, + IMB_ERR_JOB_NULL_IV)) + return 1; + printf("."); + + fill_in_job(job, cipher, dir, + hash, order, &chacha_ctx, + &gcm_ctx); + job->u.GMAC.iv_len_in_bytes = 0; + if (!is_submit_invalid(mb_mgr, job, + TEST_AUTH_GMAC_IV_LEN, + IMB_ERR_JOB_IV_LEN)) + return 1; + printf("."); + } + } + /* clean up */ while (IMB_FLUSH_JOB(mb_mgr) != NULL) ; -- GitLab From 91395c652e372c93395a141c8f81999bb063a116 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Tue, 5 Apr 2022 13:57:20 +0000 Subject: [PATCH 043/369] avx512:[ECB] add endbranch for cet --- lib/avx512/aes_ecb_vaes_avx512.asm | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/avx512/aes_ecb_vaes_avx512.asm b/lib/avx512/aes_ecb_vaes_avx512.asm index c0e8bca9..aec0951a 100644 --- a/lib/avx512/aes_ecb_vaes_avx512.asm +++ b/lib/avx512/aes_ecb_vaes_avx512.asm @@ -45,6 +45,7 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/aes_common.asm" +%include "include/cet.inc" %define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx512 %define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx512 %define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx512 @@ -202,14 +203,14 @@ align 16 align 16 MKGLOBAL(AES_ECB_ENC_128,function,internal) AES_ECB_ENC_128: - + endbranch64 AES_ECB 10, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_128,function,internal) AES_ECB_DEC_128: - + endbranch64 AES_ECB 10, DEC ret @@ -219,14 +220,14 @@ AES_ECB_DEC_128: align 16 MKGLOBAL(AES_ECB_ENC_192,function,internal) AES_ECB_ENC_192: - + endbranch64 AES_ECB 12, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_192,function,internal) AES_ECB_DEC_192: - + endbranch64 AES_ECB 12, DEC ret @@ -236,14 +237,14 @@ AES_ECB_DEC_192: align 16 MKGLOBAL(AES_ECB_ENC_256,function,internal) AES_ECB_ENC_256: - + endbranch64 AES_ECB 14, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_256,function,internal) AES_ECB_DEC_256: - + endbranch64 AES_ECB 14, DEC ret -- GitLab From 8d055cbf22ea145ac1d353aa0ecba55e972e684b Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 30 Mar 2022 09:57:56 +0000 Subject: [PATCH 044/369] lib: check for BMI2 on the AVX2 interface --- lib/x86_64/mb_mgr_auto.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/x86_64/mb_mgr_auto.c b/lib/x86_64/mb_mgr_auto.c index 029545db..d4b63b49 100644 --- a/lib/x86_64/mb_mgr_auto.c +++ b/lib/x86_64/mb_mgr_auto.c @@ -50,7 +50,8 @@ init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch) IMB_FEATURE_AESNI | IMB_FEATURE_PCLMULQDQ; const uint64_t detect_avx = IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; - const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | detect_avx; + const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | IMB_FEATURE_BMI2 | + detect_avx; const uint64_t detect_avx512 = IMB_FEATURE_AVX512_SKX | detect_avx2; /* reset error status */ -- GitLab From 1380ab6147143a99efedc7ac7a7cc2b67b4ff8d5 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 30 Mar 2022 10:19:25 +0000 Subject: [PATCH 045/369] lib: check CPU flags when initializing IMB_MGR Check for CPU flags needed for every interface (SSE, AVX...), when initializing IMB_MGR. If any of them is missing, return and IMB_ERR_MISSING_CPUFLAGS_INIT_MGR. --- lib/avx/mb_mgr_avx.c | 6 ++++++ lib/avx2/mb_mgr_avx2.c | 7 +++++++ lib/avx512/mb_mgr_avx512.c | 6 ++++++ lib/include/error.inc | 4 +++- lib/intel-ipsec-mb.h | 12 ++++++++++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 8 ++++++++ lib/sse/mb_mgr_sse.c | 6 ++++++ lib/x86_64/error.c | 6 +++++- lib/x86_64/mb_mgr_auto.c | 23 +++++------------------ 9 files changed, 58 insertions(+), 20 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 112c3e92..28cbaeb4 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -883,6 +883,12 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) return; } + /* Check if CPU flags needed for AVX interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX) != IMB_CPUFLAGS_AVX) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX; diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 65a5be92..ac7a0c2d 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -825,6 +825,13 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) return; } + + /* Check if CPU flags needed for AVX2 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX2) != IMB_CPUFLAGS_AVX2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 56645912..794774b4 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -1798,6 +1798,12 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) return; } + /* Check if CPU flags needed for AVX512 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX512) != IMB_CPUFLAGS_AVX512) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX512; diff --git a/lib/include/error.inc b/lib/include/error.inc index 75859798..b290ece5 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -90,7 +90,9 @@ SET_ERRNO_TYPES \ IMB_ERR_JOB_NULL_XCBC_K1_EXP, \ IMB_ERR_JOB_NULL_XCBC_K2, \ IMB_ERR_JOB_NULL_XCBC_K3, \ - IMB_ERR_JOB_CIPH_DIR + IMB_ERR_JOB_CIPH_DIR, \ + IMB_ERR_JOB_NULL_GHASH_INIT_TAG, \ + IMB_ERR_MISSING_CPUFLAGS_INIT_MGR ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 22ab598e..c73e1233 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -224,6 +224,7 @@ typedef enum { IMB_ERR_JOB_NULL_XCBC_K3, IMB_ERR_JOB_CIPH_DIR, IMB_ERR_JOB_NULL_GHASH_INIT_TAG, + IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; @@ -992,6 +993,17 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_FEATURE_BMI2 (1ULL << 18) #define IMB_FEATURE_AESNI_EMU (1ULL << 19) +/** + * CPU flags needed for each implementation + */ +#define IMB_CPUFLAGS_NO_AESNI (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV) +#define IMB_CPUFLAGS_SSE (IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \ + IMB_FEATURE_PCLMULQDQ) +#define IMB_CPUFLAGS_AVX (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX) +#define IMB_CPUFLAGS_AVX2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \ + IMB_FEATURE_BMI2) +#define IMB_CPUFLAGS_AVX512 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX) + /* TOP LEVEL (IMB_MGR) Data structure fields */ #define IMB_MAX_JOBS 128 diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index fa51b4af..b73e1856 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -876,6 +876,14 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) /* reset error status */ imb_set_errno(state, 0); + + /* Check if CPU flags needed for NO_AESNI interface are present */ + if ((state->features & IMB_CPUFLAGS_NO_AESNI) != + IMB_CPUFLAGS_NO_AESNI) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_NOAESNI; diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 0ce508ed..5fed5e1d 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -1143,6 +1143,12 @@ init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) return; } + /* Check if CPU flags needed for SSE interface are present */ + if ((state->features & IMB_CPUFLAGS_SSE) != IMB_CPUFLAGS_SSE) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_SSE; diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index 8ae46d27..bcdbe9df 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -85,7 +85,8 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_JOB_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K3, IMB_ERR_JOB_CIPH_DIR, - IMB_ERR_JOB_NULL_GHASH_INIT_TAG + IMB_ERR_JOB_NULL_GHASH_INIT_TAG, + IMB_ERR_MISSING_CPUFLAGS_INIT_MGR }; #ifdef DEBUG @@ -204,6 +205,9 @@ imb_get_strerror(int errnum) return "No AESNI emulation support"; case IMB_ERR_JOB_CIPH_DIR: return "Invalid cipher direction"; + case IMB_ERR_MISSING_CPUFLAGS_INIT_MGR: + return "Failed to initialize IMB_MGR due to missing " + "required CPU flags"; default: return strerror(errnum); } diff --git a/lib/x86_64/mb_mgr_auto.c b/lib/x86_64/mb_mgr_auto.c index d4b63b49..60e11dc4 100644 --- a/lib/x86_64/mb_mgr_auto.c +++ b/lib/x86_64/mb_mgr_auto.c @@ -41,19 +41,6 @@ void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch) { IMB_ARCH arch_detected = IMB_ARCH_NONE; -#ifdef AESNI_EMU - const uint64_t detect_no_aesni = - IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV; -#endif - const uint64_t detect_sse = - IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV | - IMB_FEATURE_AESNI | IMB_FEATURE_PCLMULQDQ; - const uint64_t detect_avx = - IMB_FEATURE_AVX | IMB_FEATURE_CMOV | IMB_FEATURE_AESNI; - const uint64_t detect_avx2 = IMB_FEATURE_AVX2 | IMB_FEATURE_BMI2 | - detect_avx; - const uint64_t detect_avx512 = IMB_FEATURE_AVX512_SKX | detect_avx2; - /* reset error status */ imb_set_errno(state, 0); @@ -63,31 +50,31 @@ init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch) return; } #endif - if ((state->features & detect_avx512) == detect_avx512) { + if ((state->features & IMB_CPUFLAGS_AVX512) == IMB_CPUFLAGS_AVX512) { init_mb_mgr_avx512(state); arch_detected = IMB_ARCH_AVX512; goto init_mb_mgr_auto_ret; } - if ((state->features & detect_avx2) == detect_avx2) { + if ((state->features & IMB_CPUFLAGS_AVX2) == IMB_CPUFLAGS_AVX2) { init_mb_mgr_avx2(state); arch_detected = IMB_ARCH_AVX2; goto init_mb_mgr_auto_ret; } - if ((state->features & detect_avx) == detect_avx) { + if ((state->features & IMB_CPUFLAGS_AVX) == IMB_CPUFLAGS_AVX) { init_mb_mgr_avx(state); arch_detected = IMB_ARCH_AVX; goto init_mb_mgr_auto_ret; } - if ((state->features & detect_sse) == detect_sse) { + if ((state->features & IMB_CPUFLAGS_SSE) == IMB_CPUFLAGS_SSE) { init_mb_mgr_sse(state); arch_detected = IMB_ARCH_SSE; goto init_mb_mgr_auto_ret; } #ifdef AESNI_EMU - if ((state->features & detect_no_aesni) == detect_no_aesni) { + if ((state->features & IMB_CPUFLAGS_NO_AESNI) == IMB_CPUFLAGS_NO_AESNI) { init_mb_mgr_sse_no_aesni(state); arch_detected = IMB_ARCH_NOAESNI; goto init_mb_mgr_auto_ret; -- GitLab From b46fd22ecfda99ec25c99c9475d0083aa987a443 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 25 Mar 2022 15:46:19 +0000 Subject: [PATCH 046/369] lib: add xmm versions of macros in aes_common.asm --- lib/include/aes_common.asm | 529 +++++++++++++++++++++++++++++++++++++ 1 file changed, 529 insertions(+) diff --git a/lib/include/aes_common.asm b/lib/include/aes_common.asm index cfb11289..afb035fb 100644 --- a/lib/include/aes_common.asm +++ b/lib/include/aes_common.asm @@ -747,4 +747,533 @@ %endmacro +;; ============================================================================= +;; Generic macro to produce code that executes %%OPCODE instruction with 3 +;; operands on selected number of AES blocks (16 bytes long) between 0 and 8. +;; All three operands of the instruction come from registers. +%macro XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 26 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%OPCODE %2 ; [in] instruction name +%define %%DST0 %3 ; [out] destination XMM register +%define %%DST1 %4 ; [out] destination XMM register +%define %%DST2 %5 ; [out] destination XMM register +%define %%DST3 %6 ; [out] destination XMM register +%define %%DST4 %7 ; [out] destination XMM register +%define %%DST5 %8 ; [out] destination XMM register +%define %%DST6 %9 ; [out] destination XMM register +%define %%DST7 %10 ; [out] destination XMM register +%define %%SRC1_0 %11 ; [in] source 1 XMM register +%define %%SRC1_1 %12 ; [in] source 1 XMM register +%define %%SRC1_2 %13 ; [in] source 1 XMM register +%define %%SRC1_3 %14 ; [in] source 1 XMM register +%define %%SRC1_4 %15 ; [in] source 1 XMM register +%define %%SRC1_5 %16 ; [in] source 1 XMM register +%define %%SRC1_6 %17 ; [in] source 1 XMM register +%define %%SRC1_7 %18 ; [in] source 1 XMM register +%define %%SRC2_0 %19 ; [in] source 2 XMM register +%define %%SRC2_1 %20 ; [in] source 2 XMM register +%define %%SRC2_2 %21 ; [in] source 2 XMM register +%define %%SRC2_3 %22 ; [in] source 2 XMM register +%define %%SRC2_4 %23 ; [in] source 2 XMM register +%define %%SRC2_5 %24 ; [in] source 2 XMM register +%define %%SRC2_6 %25 ; [in] source 2 XMM register +%define %%SRC2_7 %26 ; [in] source 2 XMM register + +%assign _reg_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ _reg_idx +%xdefine %%SRC1REG %%SRC1_ %+ _reg_idx +%xdefine %%SRC2REG %%SRC2_ %+ _reg_idx + %%OPCODE %%DSTREG, %%SRC1REG, %%SRC2REG +%undef %%DSTREG +%undef %%SRC1REG +%undef %%SRC2REG +%assign _reg_idx (_reg_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Generic macro to produce code that executes %%OPCODE instruction with 2 +;; operands on selected number of AES blocks (16 bytes long) between 0 and 8. +;; Both operands of the instruction come from registers. +%macro XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 18 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%OPCODE %2 ; [in] instruction name +%define %%DST0 %3 ; [out] destination YMM register +%define %%DST1 %4 ; [out] destination YMM register +%define %%DST2 %5 ; [out] destination YMM register +%define %%DST3 %6 ; [out] destination YMM register +%define %%DST4 %7 ; [out] destination YMM register +%define %%DST5 %8 ; [out] destination YMM register +%define %%DST6 %9 ; [out] destination YMM register +%define %%DST7 %10 ; [out] destination YMM register +%define %%SRC0 %11 ; [in] source YMM register +%define %%SRC1 %12 ; [in] source YMM register +%define %%SRC2 %13 ; [in] source YMM register +%define %%SRC3 %14 ; [in] source YMM register +%define %%SRC4 %15 ; [in] source YMM register +%define %%SRC5 %16 ; [in] source YMM register +%define %%SRC6 %17 ; [in] source YMM register +%define %%SRC7 %18 ; [in] source YMM register + +%assign _reg_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ _reg_idx +%xdefine %%SRCREG %%SRC %+ _reg_idx + %%OPCODE %%DSTREG, %%SRCREG +%undef %%DSTREG +%undef %%SRCREG +%assign _reg_idx (_reg_idx + 1) +%endrep + +%endmacro + +;;; =========================================================================== +;;; Handles AES encryption rounds for 0 to 8 blocks on AVX +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro XMM_AESENC_ROUND_BLOCKS_AVX_0_8 20 +%define %%L0B0 %1 ; [in/out] xmm; ciphered blocks +%define %%L0B1 %2 ; [in/out] xmm; ciphered blocks +%define %%L0B2 %3 ; [in/out] xmm; ciphered blocks +%define %%L0B3 %4 ; [in/out] xmm; ciphered blocks +%define %%L0B4 %5 ; [in/out] xmm; ciphered blocks +%define %%L0B5 %6 ; [in/out] xmm; ciphered blocks +%define %%L0B6 %7 ; [in/out] xmm; ciphered blocks +%define %%L0B7 %8 ; [in/out] xmm; ciphered blocks +%define %%KEY %9 ; [in] xmm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0 %11 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D1 %12 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D2 %13 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D3 %14 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D4 %15 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D5 %16 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D6 %17 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D7 %18 ; [in] xmm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vpxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vaesenc, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vaesenclast, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0, no_data +%ifnidn %%D1, no_data +%ifnidn %%D2, no_data +%ifnidn %%D3, no_data +%ifnidn %%D4, no_data +%ifnidn %%D5, no_data +%ifnidn %%D6, no_data +%ifnidn %%D7, no_data + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vpxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%D0, %%D1, %%D2, %%D3, \ + %%D4, %%D5, %%D6, %%D7 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +;;; =========================================================================== +;;; Handles AES encryption rounds for 0 to 8 blocks on SSE +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro XMM_AESENC_ROUND_BLOCKS_SSE_0_8 20 +%define %%L0B0 %1 ; [in/out] xmm; ciphered blocks +%define %%L0B1 %2 ; [in/out] xmm; ciphered blocks +%define %%L0B2 %3 ; [in/out] xmm; ciphered blocks +%define %%L0B3 %4 ; [in/out] xmm; ciphered blocks +%define %%L0B4 %5 ; [in/out] xmm; ciphered blocks +%define %%L0B5 %6 ; [in/out] xmm; ciphered blocks +%define %%L0B6 %7 ; [in/out] xmm; ciphered blocks +%define %%L0B7 %8 ; [in/out] xmm; ciphered blocks +%define %%KEY %9 ; [in] xmm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0 %11 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D1 %12 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D2 %13 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D3 %14 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D4 %15 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D5 %16 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D6 %17 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D7 %18 ; [in] xmm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, pxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, aesenc, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, aesenclast, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0, no_data +%ifnidn %%D1, no_data +%ifnidn %%D2, no_data +%ifnidn %%D3, no_data +%ifnidn %%D4, no_data +%ifnidn %%D5, no_data +%ifnidn %%D6, no_data +%ifnidn %%D7, no_data + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, pxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%D0, %%D1, %%D2, %%D3, \ + %%D4, %%D5, %%D6, %%D7 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + + +;;; =========================================================================== +;;; Handles AES decryption rounds for 0 to 8 blocks on AVX +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro XMM_AESDEC_ROUND_BLOCKS_AVX_0_8 20 +%define %%L0B0 %1 ; [in/out] xmm; ciphered blocks +%define %%L0B1 %2 ; [in/out] xmm; ciphered blocks +%define %%L0B2 %3 ; [in/out] xmm; ciphered blocks +%define %%L0B3 %4 ; [in/out] xmm; ciphered blocks +%define %%L0B4 %5 ; [in/out] xmm; ciphered blocks +%define %%L0B5 %6 ; [in/out] xmm; ciphered blocks +%define %%L0B6 %7 ; [in/out] xmm; ciphered blocks +%define %%L0B7 %8 ; [in/out] xmm; ciphered blocks +%define %%KEY %9 ; [in] xmm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0 %11 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D1 %12 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D2 %13 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D3 %14 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D4 %15 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D5 %16 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D6 %17 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D7 %18 ; [in] xmm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vpxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vaesdec, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vaesdeclast, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0, no_data +%ifnidn %%D1, no_data +%ifnidn %%D2, no_data +%ifnidn %%D3, no_data +%ifnidn %%D4, no_data +%ifnidn %%D5, no_data +%ifnidn %%D6, no_data +%ifnidn %%D7, no_data + XMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_8 %%NUMBL, vpxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%D0, %%D1, %%D2, %%D3, \ + %%D4, %%D5, %%D6, %%D7 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + %endif ;; _AES_COMMON_ASM + +;;; =========================================================================== +;;; Handles AES decryption rounds for 0 to 8 blocks on SSE +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro XMM_AESDEC_ROUND_BLOCKS_SSE_0_8 20 +%define %%L0B0 %1 ; [in/out] xmm; ciphered blocks +%define %%L0B1 %2 ; [in/out] xmm; ciphered blocks +%define %%L0B2 %3 ; [in/out] xmm; ciphered blocks +%define %%L0B3 %4 ; [in/out] xmm; ciphered blocks +%define %%L0B4 %5 ; [in/out] xmm; ciphered blocks +%define %%L0B5 %6 ; [in/out] xmm; ciphered blocks +%define %%L0B6 %7 ; [in/out] xmm; ciphered blocks +%define %%L0B7 %8 ; [in/out] xmm; ciphered blocks +%define %%KEY %9 ; [in] xmm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0 %11 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D1 %12 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D2 %13 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D3 %14 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D4 %15 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D5 %16 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D6 %17 ; [in] xmm or no_data; plain/cipher text blocks +%define %%D7 %18 ; [in] xmm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, pxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, aesdec, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, aesdeclast, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0, no_data +%ifnidn %%D1, no_data +%ifnidn %%D2, no_data +%ifnidn %%D3, no_data +%ifnidn %%D4, no_data +%ifnidn %%D5, no_data +%ifnidn %%D6, no_data +%ifnidn %%D7, no_data + XMM_OPCODE2_DSTR_SRCR_BLOCKS_0_8 %%NUMBL, pxor, \ + %%L0B0, %%L0B1, %%L0B2, %%L0B3, \ + %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ + %%D0, %%D1, %%D2, %%D3, \ + %%D4, %%D5, %%D6, %%D7 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +;; ============================================================================= +;; Loads up to 8 blocks into XMM registers on AVX +%macro XMM_LOAD_BLOCKS_AVX_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] XMM register with loaded data +%define %%DST1 %5 ; [out] XMM register with loaded data +%define %%DST2 %6 ; [out] XMM register with loaded data +%define %%DST3 %7 ; [out] XMM register with loaded data +%define %%DST4 %8 ; [out] XMM register with loaded data +%define %%DST5 %9 ; [out] XMM register with loaded data +%define %%DST6 %10 ; [out] XMM register with loaded data +%define %%DST7 %11 ; [out] XMM register with loaded data + +%assign src_offset 0 +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ dst_idx + vmovdqu %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 16) +%assign dst_idx (dst_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Loads up to 8 AES blocks into XMM registers on SSE +%macro XMM_LOAD_BLOCKS_SSE_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] XMM register with loaded data +%define %%DST1 %5 ; [out] XMM register with loaded data +%define %%DST2 %6 ; [out] XMM register with loaded data +%define %%DST3 %7 ; [out] XMM register with loaded data +%define %%DST4 %8 ; [out] XMM register with loaded data +%define %%DST5 %9 ; [out] XMM register with loaded data +%define %%DST6 %10 ; [out] XMM register with loaded data +%define %%DST7 %11 ; [out] XMM register with loaded data + +%assign src_offset 0 +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%DSTREG %%DST %+ dst_idx + movdqu %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 16) +%assign dst_idx (dst_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Stores up to 8 AES blocks from XMM registers on AVX +%macro XMM_STORE_BLOCKS_AVX_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] XMM register with data to store +%define %%SRC1 %5 ; [in] XMM register with data to store +%define %%SRC2 %6 ; [in] XMM register with data to store +%define %%SRC3 %7 ; [in] XMM register with data to store +%define %%SRC4 %8 ; [in] XMM register with data to store +%define %%SRC5 %9 ; [in] XMM register with data to store +%define %%SRC6 %10 ; [in] XMM register with data to store +%define %%SRC7 %11 ; [in] XMM register with data to store + +%assign dst_offset 0 +%assign src_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%SRCREG %%SRC %+ src_idx + vmovdqu [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 16) +%assign src_idx (src_idx + 1) +%endrep + +%endmacro + +;; ============================================================================= +;; Stores up to 8 AES blocks from XMM registers on SSE +%macro XMM_STORE_BLOCKS_SSE_0_8 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 8) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] XMM register with data to store +%define %%SRC1 %5 ; [in] XMM register with data to store +%define %%SRC2 %6 ; [in] XMM register with data to store +%define %%SRC3 %7 ; [in] XMM register with data to store +%define %%SRC4 %8 ; [in] XMM register with data to store +%define %%SRC5 %9 ; [in] XMM register with data to store +%define %%SRC6 %10 ; [in] XMM register with data to store +%define %%SRC7 %11 ; [in] XMM register with data to store + +%assign dst_offset 0 +%assign src_idx 0 + +%rep (%%NUM_BLOCKS) +%xdefine %%SRCREG %%SRC %+ src_idx + movdqu [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 16) +%assign src_idx (src_idx + 1) +%endrep + +%endmacro -- GitLab From 84e68855cbc7f81d5011f1357ad2b1198e73b142 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 31 Mar 2022 12:41:18 +0000 Subject: [PATCH 047/369] lib: add by8 ECB implementation for SSE --- README | 6 +- README.md | 6 +- lib/Makefile | 3 + lib/include/asm.h | 26 +++-- lib/sse/aes128_ecb_by4_sse.asm | 6 +- lib/sse/aes128_ecb_by8_sse.asm | 168 +++++++++++++++++++++++++++++++++ lib/sse/aes192_ecb_by4_sse.asm | 4 +- lib/sse/aes192_ecb_by8_sse.asm | 33 +++++++ lib/sse/aes256_ecb_by4_sse.asm | 4 +- lib/sse/aes256_ecb_by8_sse.asm | 33 +++++++ lib/sse/mb_mgr_sse.c | 46 +++++++++ lib/win_x64.mak | 3 + 12 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 lib/sse/aes128_ecb_by8_sse.asm create mode 100644 lib/sse/aes192_ecb_by8_sse.asm create mode 100644 lib/sse/aes256_ecb_by8_sse.asm diff --git a/README b/README index bbb43105..90a29399 100644 --- a/README +++ b/README @@ -75,9 +75,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES128-ECB | N | Y by4 | Y by4 | N | N | Y by16 | - | AES192-ECB | N | Y by4 | Y by4 | N | N | Y by16 | - | AES256-ECB | N | Y by4 | Y by4 | N | N | Y by16 | + | AES128-ECB | N | Y(1) | Y by8 | N | N | Y by16 | + | AES192-ECB | N | Y(1) | Y by8 | N | N | Y by16 | + | AES256-ECB | N | Y(1) | Y by8 | N | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | diff --git a/README.md b/README.md index 87753547..1ba8bad0 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | -| AES128-ECB | N | Y by4 | Y by4 | N | N | Y by16 | -| AES192-ECB | N | Y by4 | Y by4 | N | N | Y by16 | -| AES256-ECB | N | Y by4 | Y by4 | N | N | Y by16 | +| AES128-ECB | N | Y(1) | Y by4 | N | N | Y by16 | +| AES192-ECB | N | Y(1) | Y by4 | N | N | Y by16 | +| AES256-ECB | N | Y(1) | Y by4 | N | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | diff --git a/lib/Makefile b/lib/Makefile index ea4d5c89..81fde438 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -375,6 +375,9 @@ asm_sse_lib_objs := \ aes128_ecb_by4_sse.o \ aes192_ecb_by4_sse.o \ aes256_ecb_by4_sse.o \ + aes128_ecb_by8_sse.o \ + aes192_ecb_by8_sse.o \ + aes256_ecb_by8_sse.o \ aes128_cntr_ccm_by8_sse.o \ aes256_cntr_ccm_by8_sse.o \ aes_cfb_sse.o \ diff --git a/lib/include/asm.h b/lib/include/asm.h index 34814bbb..4739e012 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -151,11 +151,18 @@ IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); /* AES-ECB */ -void aes_ecb_enc_256_sse(const void *in, const void *keys, +void aes_ecb_enc_256_by4_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_enc_192_sse(const void *in, const void *keys, +void aes_ecb_enc_192_by4_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_enc_128_sse(const void *in, const void *keys, +void aes_ecb_enc_128_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_enc_256_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_by8_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_256_sse_no_aesni(const void *in, const void *keys, @@ -180,11 +187,18 @@ void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_sse(const void *in, const void *keys, +void aes_ecb_dec_256_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_by8_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_192_sse(const void *in, const void *keys, +void aes_ecb_dec_192_by8_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_128_sse(const void *in, const void *keys, +void aes_ecb_dec_128_by8_sse(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_256_sse_no_aesni(const void *in, const void *keys, diff --git a/lib/sse/aes128_ecb_by4_sse.asm b/lib/sse/aes128_ecb_by4_sse.asm index 62567387..f1fafa27 100644 --- a/lib/sse/aes128_ecb_by4_sse.asm +++ b/lib/sse/aes128_ecb_by4_sse.asm @@ -25,7 +25,7 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -; routine to do AES ECB encrypt/decrypt on 16n bytes doing AES by 4 +; routine to do AES ECB 128 encrypt/decrypt on 16n bytes doing AES by 4 ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -48,8 +48,8 @@ %ifndef AES_ECB_ENC_256 %ifndef AES_ECB_ENC_192 %ifndef AES_ECB_ENC_128 -%define AES_ECB_ENC_128 aes_ecb_enc_128_sse -%define AES_ECB_DEC_128 aes_ecb_dec_128_sse +%define AES_ECB_ENC_128 aes_ecb_enc_128_by4_sse +%define AES_ECB_DEC_128 aes_ecb_dec_128_by4_sse %endif %endif %endif diff --git a/lib/sse/aes128_ecb_by8_sse.asm b/lib/sse/aes128_ecb_by8_sse.asm new file mode 100644 index 00000000..84ff93e7 --- /dev/null +++ b/lib/sse/aes128_ecb_by8_sse.asm @@ -0,0 +1,168 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 128 encrypt/decrypt on 16n bytes doing AES by 8 + +%include "include/os.asm" +%include "include/clear_regs.asm" +%include "include/aes_common.asm" +%include "include/cet.inc" + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define LEN rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define LEN r9 +%endif +%define IDX rax +%define TMP r11 +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XDATA4 xmm4 +%define XDATA5 xmm5 +%define XDATA6 xmm6 +%define XDATA7 xmm7 +%define XKEY1 xmm8 + +%ifndef AES_ECB_NROUNDS +%define AES_ECB_NROUNDS 10 +%endif + +%if AES_ECB_NROUNDS == 10 +%define KEYSIZE 128 +%elif AES_ECB_NROUNDS == 12 +%define KEYSIZE 192 +%else +%define KEYSIZE 256 +%endif + +%define AES_ECB_ENC aes_ecb_enc_ %+ KEYSIZE %+ _by8_sse +%define AES_ECB_DEC aes_ecb_dec_ %+ KEYSIZE %+ _by8_sse + +%macro AES_ECB 1 +%define %%DIR %1 ; [in] Direction (ENC/DEC) +%ifidn %%DIR, ENC +%define AES XMM_AESENC_ROUND_BLOCKS_SSE_0_8 +%else ; DIR = DEC +%define AES XMM_AESDEC_ROUND_BLOCKS_SSE_0_8 +%endif + or LEN, LEN + jz %%done + xor IDX, IDX + mov TMP, LEN + and TMP, 127 ; number of initial bytes (0 to 7 AES blocks) + jz %%main_loop + ; branch to different code block based on remainder + cmp TMP, 4*16 + je %%initial_num_blocks_is_4 + jb %%initial_num_blocks_is_3_1 + cmp TMP, 6*16 + je %%initial_num_blocks_is_6 + jb %%initial_num_blocks_is_5 + ja %%initial_num_blocks_is_7 +%%initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2*16 + ja %%initial_num_blocks_is_3 + je %%initial_num_blocks_is_2 + ;; fall through for `jmp %%initial_num_blocks_is_1` +%assign num_blocks 1 +%rep 7 +%%initial_num_blocks_is_ %+ num_blocks : + ; load initial blocks + XMM_LOAD_BLOCKS_SSE_0_8 num_blocks, IN, 0, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption/decryption on initial blocks +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + movdqu XKEY1, [KEYS + %%I*16] + AES XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_blocks, (AES_ECB_NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store initial blocks + XMM_STORE_BLOCKS_SSE_0_8 num_blocks, OUT, 0, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7 + add IDX, num_blocks*16 + cmp IDX, LEN + je %%done +%assign num_blocks (num_blocks + 1) + jmp %%main_loop +%endrep +align 16 +%%main_loop: + ; load next 8 blocks + XMM_LOAD_BLOCKS_SSE_0_8 8, {IN + IDX}, 0, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption/decryption on 8 blocks +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + movdqu XKEY1, [KEYS + %%I*16] + AES XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, 8, (AES_ECB_NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store 8 blocks + XMM_STORE_BLOCKS_SSE_0_8 8, {OUT + IDX}, 0, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7 + add IDX, 8*16 + cmp IDX, LEN + jne %%main_loop +%%done: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif +%endmacro + +mksection .text +align 16 +MKGLOBAL(AES_ECB_ENC,function,internal) +AES_ECB_ENC: + endbranch64 + AES_ECB ENC + ret +align 16 +MKGLOBAL(AES_ECB_DEC,function,internal) +AES_ECB_DEC: + endbranch64 + AES_ECB DEC + ret + +mksection stack-noexec diff --git a/lib/sse/aes192_ecb_by4_sse.asm b/lib/sse/aes192_ecb_by4_sse.asm index b66db012..a5386e1f 100644 --- a/lib/sse/aes192_ecb_by4_sse.asm +++ b/lib/sse/aes192_ecb_by4_sse.asm @@ -27,7 +27,7 @@ ; routine to do AES ECB 192 encrypt/decrypt on 16n bytes doing AES by 4 -%define AES_ECB_ENC_192 aes_ecb_enc_192_sse -%define AES_ECB_DEC_192 aes_ecb_dec_192_sse +%define AES_ECB_ENC_192 aes_ecb_enc_192_by4_sse +%define AES_ECB_DEC_192 aes_ecb_dec_192_by4_sse %include "sse/aes128_ecb_by4_sse.asm" diff --git a/lib/sse/aes192_ecb_by8_sse.asm b/lib/sse/aes192_ecb_by8_sse.asm new file mode 100644 index 00000000..ce2794a5 --- /dev/null +++ b/lib/sse/aes192_ecb_by8_sse.asm @@ -0,0 +1,33 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 192 encrypt/decrypt on 16n bytes doing AES by 8 + +%define AES_ECB_NROUNDS 12 + +%include "include/os.asm" +%include "sse/aes128_ecb_by8_sse.asm" diff --git a/lib/sse/aes256_ecb_by4_sse.asm b/lib/sse/aes256_ecb_by4_sse.asm index 544f8ded..1ef5c49f 100644 --- a/lib/sse/aes256_ecb_by4_sse.asm +++ b/lib/sse/aes256_ecb_by4_sse.asm @@ -27,7 +27,7 @@ ; routine to do AES ECB 256 encrypt/decrypt on 16n bytes doing AES by 4 -%define AES_ECB_ENC_256 aes_ecb_enc_256_sse -%define AES_ECB_DEC_256 aes_ecb_dec_256_sse +%define AES_ECB_ENC_256 aes_ecb_enc_256_by4_sse +%define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse %include "sse/aes128_ecb_by4_sse.asm" diff --git a/lib/sse/aes256_ecb_by8_sse.asm b/lib/sse/aes256_ecb_by8_sse.asm new file mode 100644 index 00000000..fb7434a3 --- /dev/null +++ b/lib/sse/aes256_ecb_by8_sse.asm @@ -0,0 +1,33 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 256 encrypt/decrypt on 16n bytes doing AES by 8 + +%define AES_ECB_NROUNDS 14 + +%include "include/os.asm" +%include "sse/aes128_ecb_by8_sse.asm" diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 5fed5e1d..ab2113a8 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -453,6 +453,41 @@ static ccm_flush_job_t flush_job_aes256_ccm_auth_ptr = /* ====================================================================== */ +/* + * ECB function pointers + */ +static void +(*aes_ecb_enc_128_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_128_by4_sse; + +static void +(*aes_ecb_enc_192_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_192_by4_sse; + +static void +(*aes_ecb_enc_256_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_256_by4_sse; + +static void +(*aes_ecb_dec_128_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_128_by4_sse; + +static void +(*aes_ecb_dec_192_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_192_by4_sse; + +static void +(*aes_ecb_dec_256_sse) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_256_by4_sse; + +/* ====================================================================== */ + /* * GCM submit / flush API for SSE arch */ @@ -722,6 +757,17 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_cbc_dec_ptr = aes_cbc_dec_256_by8_sse; } + if (state->features & IMB_FEATURE_GFNI) { + /* change AES-ECB implementation */ + aes_ecb_enc_128_sse = aes_ecb_enc_128_by8_sse; + aes_ecb_enc_192_sse = aes_ecb_enc_192_by8_sse; + aes_ecb_enc_256_sse = aes_ecb_enc_256_by8_sse; + + aes_ecb_dec_128_sse = aes_ecb_dec_128_by8_sse; + aes_ecb_dec_192_sse = aes_ecb_dec_192_by8_sse; + aes_ecb_dec_256_sse = aes_ecb_dec_256_by8_sse; + } + /* DOCSIS SEC BPI uses same settings as AES CBC */ memset(docsis128_sec_ooo->lens, 0xFF, sizeof(docsis128_sec_ooo->lens)); diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 2f280e56..57f4c119 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -162,6 +162,9 @@ lib_objs1 = \ $(OBJ_DIR)\aes128_ecb_by4_sse.obj \ $(OBJ_DIR)\aes192_ecb_by4_sse.obj \ $(OBJ_DIR)\aes256_ecb_by4_sse.obj \ + $(OBJ_DIR)\aes128_ecb_by8_sse.obj \ + $(OBJ_DIR)\aes192_ecb_by8_sse.obj \ + $(OBJ_DIR)\aes256_ecb_by8_sse.obj \ $(OBJ_DIR)\aes128_ecb_by4_avx.obj \ $(OBJ_DIR)\aes192_ecb_by4_avx.obj \ $(OBJ_DIR)\aes256_ecb_by4_avx.obj \ -- GitLab From cc2de2a52ee401d5b967668347f7a75f0374d80c Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 31 Mar 2022 12:44:29 +0000 Subject: [PATCH 048/369] lib: add by8 ECB implementation for AVX --- ReleaseNotes.txt | 1 + lib/Makefile | 6 +- lib/avx/aes128_ecb_by4_avx.asm | 672 ------------------ lib/avx/aes128_ecb_by8_avx.asm | 168 +++++ ...ecb_by4_avx.asm => aes192_ecb_by8_avx.asm} | 10 +- ...ecb_by4_avx.asm => aes256_ecb_by8_avx.asm} | 10 +- lib/win_x64.mak | 6 +- 7 files changed, 185 insertions(+), 688 deletions(-) delete mode 100644 lib/avx/aes128_ecb_by4_avx.asm create mode 100644 lib/avx/aes128_ecb_by8_avx.asm rename lib/avx/{aes192_ecb_by4_avx.asm => aes192_ecb_by8_avx.asm} (89%) rename lib/avx/{aes256_ecb_by4_avx.asm => aes256_ecb_by8_avx.asm} (89%) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index f0915de0..f889336c 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -8,6 +8,7 @@ Library - ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE and AVX - AES-ECB AVX512-VAES implementation added - JOB API GHASH support added +- AES-ECB optimizations for AVX and SSE Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) diff --git a/lib/Makefile b/lib/Makefile index 81fde438..68f62e25 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -475,9 +475,9 @@ asm_avx_lib_objs := \ aes256_cntr_by8_avx.o \ aes128_cntr_ccm_by8_avx.o \ aes256_cntr_ccm_by8_avx.o \ - aes128_ecb_by4_avx.o \ - aes192_ecb_by4_avx.o \ - aes256_ecb_by4_avx.o \ + aes128_ecb_by8_avx.o \ + aes192_ecb_by8_avx.o \ + aes256_ecb_by8_avx.o \ aes_cfb_avx.o \ aes128_cbc_mac_x8_avx.o \ aes256_cbc_mac_x8_avx.o \ diff --git a/lib/avx/aes128_ecb_by4_avx.asm b/lib/avx/aes128_ecb_by4_avx.asm deleted file mode 100644 index fbe29765..00000000 --- a/lib/avx/aes128_ecb_by4_avx.asm +++ /dev/null @@ -1,672 +0,0 @@ -;; -;; Copyright (c) 2019-2022, Intel Corporation -;; -;; Redistribution and use in source and binary forms, with or without -;; modification, are permitted provided that the following conditions are met: -;; -;; * Redistributions of source code must retain the above copyright notice, -;; this list of conditions and the following disclaimer. -;; * Redistributions in binary form must reproduce the above copyright -;; notice, this list of conditions and the following disclaimer in the -;; documentation and/or other materials provided with the distribution. -;; * Neither the name of Intel Corporation nor the names of its contributors -;; may be used to endorse or promote products derived from this software -;; without specific prior written permission. -;; -;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -;; - -; routine to do AES ECB encrypt/decrypt on 16n bytes doing AES by 4 - -; XMM registers are clobbered. Saving/restoring must be done at a higher level - -; void aes_ecb_x_y_avx(void *in, -; UINT128 keys[], -; void *out, -; UINT64 len_bytes); -; -; x = direction (enc/dec) -; y = key size (128/192/256) -; arg 1: IN: pointer to input (cipher text) -; arg 2: KEYS: pointer to keys -; arg 3: OUT: pointer to output (plain text) -; arg 4: LEN: length in bytes (multiple of 16) -; - -%include "include/os.asm" -%include "include/clear_regs.asm" - -%ifndef AES_ECB_ENC_256 -%ifndef AES_ECB_ENC_192 -%ifndef AES_ECB_ENC_128 -%define AES_ECB_ENC_128 aes_ecb_enc_128_avx -%define AES_ECB_DEC_128 aes_ecb_dec_128_avx -%endif -%endif -%endif - -%ifdef LINUX -%define IN rdi -%define KEYS rsi -%define OUT rdx -%define LEN rcx -%else -%define IN rcx -%define KEYS rdx -%define OUT r8 -%define LEN r9 -%endif - -%define IDX rax -%define TMP IDX -%define XDATA0 xmm0 -%define XDATA1 xmm1 -%define XDATA2 xmm2 -%define XDATA3 xmm3 -%define XKEY0 xmm4 -%define XKEY2 xmm5 -%define XKEY4 xmm6 -%define XKEY6 xmm7 -%define XKEY10 xmm8 -%define XKEY_A xmm9 -%define XKEY_B xmm10 - -mksection .text - -%macro AES_ECB 2 -%define %%NROUNDS %1 ; [in] Number of AES rounds, numerical value -%define %%DIR %2 ; [in] Direction (encrypt/decrypt) - -%ifidn %%DIR, ENC -%define AES vaesenc -%define AES_LAST vaesenclast -%else ; DIR = DEC -%define AES vaesdec -%define AES_LAST vaesdeclast -%endif - mov TMP, LEN - and TMP, 3*16 - jz %%initial_4 - cmp TMP, 2*16 - jb %%initial_1 - ja %%initial_3 - -%%initial_2: - ; load plain/cipher text - vmovdqu XDATA0, [IN + 0*16] - vmovdqu XDATA1, [IN + 1*16] - - vmovdqa XKEY0, [KEYS + 0*16] - - vpxor XDATA0, XKEY0 ; 0. ARK - vpxor XDATA1, XKEY0 - - vmovdqa XKEY2, [KEYS + 2*16] - - AES XDATA0, [KEYS + 1*16] ; 1. ENC - AES XDATA1, [KEYS + 1*16] - - mov IDX, 2*16 - - AES XDATA0, XKEY2 ; 2. ENC - AES XDATA1, XKEY2 - - vmovdqa XKEY4, [KEYS + 4*16] - - AES XDATA0, [KEYS + 3*16] ; 3. ENC - AES XDATA1, [KEYS + 3*16] - - AES XDATA0, XKEY4 ; 4. ENC - AES XDATA1, XKEY4 - - vmovdqa XKEY6, [KEYS + 6*16] - - AES XDATA0, [KEYS + 5*16] ; 5. ENC - AES XDATA1, [KEYS + 5*16] - - AES XDATA0, XKEY6 ; 6. ENC - AES XDATA1, XKEY6 - - vmovdqa XKEY_B, [KEYS + 8*16] - - AES XDATA0, [KEYS + 7*16] ; 7. ENC - AES XDATA1, [KEYS + 7*16] - - AES XDATA0, XKEY_B ; 8. ENC - AES XDATA1, XKEY_B - - vmovdqa XKEY10, [KEYS + 10*16] - - AES XDATA0, [KEYS + 9*16] ; 9. ENC - AES XDATA1, [KEYS + 9*16] - -%if %%NROUNDS >= 12 - AES XDATA0, XKEY10 ; 10. ENC - AES XDATA1, XKEY10 - - AES XDATA0, [KEYS + 11*16] ; 11. ENC - AES XDATA1, [KEYS + 11*16] -%endif - -%if %%NROUNDS == 14 - AES XDATA0, [KEYS + 12*16] ; 12. ENC - AES XDATA1, [KEYS + 12*16] - - AES XDATA0, [KEYS + 13*16] ; 13. ENC - AES XDATA1, [KEYS + 13*16] -%endif - -%if %%NROUNDS == 10 - AES_LAST XDATA0, XKEY10 ; 10. ENC - AES_LAST XDATA1, XKEY10 -%elif %%NROUNDS == 12 - AES_LAST XDATA0, [KEYS + 12*16] ; 12. ENC - AES_LAST XDATA1, [KEYS + 12*16] -%else - AES_LAST XDATA0, [KEYS + 14*16] ; 14. ENC - AES_LAST XDATA1, [KEYS + 14*16] -%endif - vmovdqu [OUT + 0*16], XDATA0 - vmovdqu [OUT + 1*16], XDATA1 - - cmp LEN, 2*16 - je %%done - jmp %%main_loop - - align 16 -%%initial_1: - ; load plain/cipher text - vmovdqu XDATA0, [IN + 0*16] - - vmovdqa XKEY0, [KEYS + 0*16] - - vpxor XDATA0, XKEY0 ; 0. ARK - - vmovdqa XKEY2, [KEYS + 2*16] - - AES XDATA0, [KEYS + 1*16] ; 1. ENC - - mov IDX, 1*16 - - AES XDATA0, XKEY2 ; 2. ENC - - vmovdqa XKEY4, [KEYS + 4*16] - - AES XDATA0, [KEYS + 3*16] ; 3. ENC - - AES XDATA0, XKEY4 ; 4. ENC - - vmovdqa XKEY6, [KEYS + 6*16] - - AES XDATA0, [KEYS + 5*16] ; 5. ENC - - AES XDATA0, XKEY6 ; 6. ENC - - vmovdqa XKEY_B, [KEYS + 8*16] - - AES XDATA0, [KEYS + 7*16] ; 7. ENC - - AES XDATA0, XKEY_B ; 8. ENC - - vmovdqa XKEY10, [KEYS + 10*16] - - AES XDATA0, [KEYS + 9*16] ; 9. ENC - -%if %%NROUNDS >= 12 - AES XDATA0, XKEY10 ; 10. ENC - - AES XDATA0, [KEYS + 11*16] ; 11. ENC -%endif - -%if %%NROUNDS == 14 - AES XDATA0, [KEYS + 12*16] ; 12. ENC - - AES XDATA0, [KEYS + 13*16] ; 13. ENC -%endif - -%if %%NROUNDS == 10 - - AES_LAST XDATA0, XKEY10 ; 10. ENC -%elif %%NROUNDS == 12 - AES_LAST XDATA0, [KEYS + 12*16] ; 12. ENC -%else - AES_LAST XDATA0, [KEYS + 14*16] ; 14. ENC -%endif - - vmovdqu [OUT + 0*16], XDATA0 - - cmp LEN, 1*16 - je %%done - jmp %%main_loop - -%%initial_3: - ; load plain/cipher text - vmovdqu XDATA0, [IN + 0*16] - vmovdqu XDATA1, [IN + 1*16] - vmovdqu XDATA2, [IN + 2*16] - - vmovdqa XKEY0, [KEYS + 0*16] - - vmovdqa XKEY_A, [KEYS + 1*16] - - vpxor XDATA0, XKEY0 ; 0. ARK - vpxor XDATA1, XKEY0 - vpxor XDATA2, XKEY0 - - vmovdqa XKEY2, [KEYS + 2*16] - - AES XDATA0, XKEY_A ; 1. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - - vmovdqa XKEY_A, [KEYS + 3*16] - mov IDX, 3*16 - - AES XDATA0, XKEY2 ; 2. ENC - AES XDATA1, XKEY2 - AES XDATA2, XKEY2 - - vmovdqa XKEY4, [KEYS + 4*16] - - AES XDATA0, XKEY_A ; 3. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - - vmovdqa XKEY_A, [KEYS + 5*16] - - AES XDATA0, XKEY4 ; 4. ENC - AES XDATA1, XKEY4 - AES XDATA2, XKEY4 - - vmovdqa XKEY6, [KEYS + 6*16] - - AES XDATA0, XKEY_A ; 5. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - - vmovdqa XKEY_A, [KEYS + 7*16] - - AES XDATA0, XKEY6 ; 6. ENC - AES XDATA1, XKEY6 - AES XDATA2, XKEY6 - - vmovdqa XKEY_B, [KEYS + 8*16] - - AES XDATA0, XKEY_A ; 7. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - - vmovdqa XKEY_A, [KEYS + 9*16] - - AES XDATA0, XKEY_B ; 8. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - - vmovdqa XKEY_B, [KEYS + 10*16] - - AES XDATA0, XKEY_A ; 9. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - -%if %%NROUNDS >= 12 - vmovdqa XKEY_A, [KEYS + 11*16] - - AES XDATA0, XKEY_B ; 10. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - - vmovdqa XKEY_B, [KEYS + 12*16] - - AES XDATA0, XKEY_A ; 11. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - -%endif - -%if %%NROUNDS == 14 - vmovdqa XKEY_A, [KEYS + 13*16] - - AES XDATA0, XKEY_B ; 12. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - - vmovdqa XKEY_B, [KEYS + 14*16] - - AES XDATA0, XKEY_A ; 13. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A -%endif - - AES_LAST XDATA0, XKEY_B ; 10/12/14. ENC (depending on key size) - AES_LAST XDATA1, XKEY_B - AES_LAST XDATA2, XKEY_B - - vmovdqu [OUT + 0*16], XDATA0 - vmovdqu [OUT + 1*16], XDATA1 - vmovdqu [OUT + 2*16], XDATA2 - - cmp LEN, 3*16 - je %%done - jmp %%main_loop - - align 16 -%%initial_4: - ; load plain/cipher text - vmovdqu XDATA0, [IN + 0*16] - vmovdqu XDATA1, [IN + 1*16] - vmovdqu XDATA2, [IN + 2*16] - vmovdqu XDATA3, [IN + 3*16] - - vmovdqa XKEY0, [KEYS + 0*16] - - vmovdqa XKEY_A, [KEYS + 1*16] - - vpxor XDATA0, XKEY0 ; 0. ARK - vpxor XDATA1, XKEY0 - vpxor XDATA2, XKEY0 - vpxor XDATA3, XKEY0 - - vmovdqa XKEY2, [KEYS + 2*16] - - AES XDATA0, XKEY_A ; 1. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 3*16] - - mov IDX, 4*16 - - AES XDATA0, XKEY2 ; 2. ENC - AES XDATA1, XKEY2 - AES XDATA2, XKEY2 - AES XDATA3, XKEY2 - - vmovdqa XKEY4, [KEYS + 4*16] - - AES XDATA0, XKEY_A ; 3. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 5*16] - - AES XDATA0, XKEY4 ; 4. ENC - AES XDATA1, XKEY4 - AES XDATA2, XKEY4 - AES XDATA3, XKEY4 - - vmovdqa XKEY6, [KEYS + 6*16] - - AES XDATA0, XKEY_A ; 5. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 7*16] - - AES XDATA0, XKEY6 ; 6. ENC - AES XDATA1, XKEY6 - AES XDATA2, XKEY6 - AES XDATA3, XKEY6 - - vmovdqa XKEY_B, [KEYS + 8*16] - - AES XDATA0, XKEY_A ; 7. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 9*16] - - AES XDATA0, XKEY_B ; 8. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 10*16] - - AES XDATA0, XKEY_A ; 9. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - -%if %%NROUNDS >= 12 - vmovdqa XKEY_A, [KEYS + 11*16] - - AES XDATA0, XKEY_B ; 10. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 12*16] - - AES XDATA0, XKEY_A ; 11. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A -%endif - -%if %%NROUNDS == 14 - vmovdqa XKEY_A, [KEYS + 13*16] - - AES XDATA0, XKEY_B ; 12. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 14*16] - - AES XDATA0, XKEY_A ; 13. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A -%endif - - AES_LAST XDATA0, XKEY_B ; 10/12/14. ENC (depending on key size) - AES_LAST XDATA1, XKEY_B - AES_LAST XDATA2, XKEY_B - AES_LAST XDATA3, XKEY_B - - vmovdqu [OUT + 0*16], XDATA0 - vmovdqu [OUT + 1*16], XDATA1 - vmovdqu [OUT + 2*16], XDATA2 - vmovdqu [OUT + 3*16], XDATA3 - - cmp LEN, 4*16 - jz %%done - jmp %%main_loop - - align 16 -%%main_loop: - ; load plain/cipher text - vmovdqu XDATA0, [IN + IDX + 0*16] - vmovdqu XDATA1, [IN + IDX + 1*16] - vmovdqu XDATA2, [IN + IDX + 2*16] - vmovdqu XDATA3, [IN + IDX + 3*16] - - vmovdqa XKEY_A, [KEYS + 1*16] - - vpxor XDATA0, XKEY0 ; 0. ARK - vpxor XDATA1, XKEY0 - vpxor XDATA2, XKEY0 - vpxor XDATA3, XKEY0 - - add IDX, 4*16 - - AES XDATA0, XKEY_A ; 1. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 3*16] - - AES XDATA0, XKEY2 ; 2. ENC - AES XDATA1, XKEY2 - AES XDATA2, XKEY2 - AES XDATA3, XKEY2 - - AES XDATA0, XKEY_A ; 3. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 5*16] - - AES XDATA0, XKEY4 ; 4. ENC - AES XDATA1, XKEY4 - AES XDATA2, XKEY4 - AES XDATA3, XKEY4 - - AES XDATA0, XKEY_A ; 5. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 7*16] - - AES XDATA0, XKEY6 ; 6. ENC - AES XDATA1, XKEY6 - AES XDATA2, XKEY6 - AES XDATA3, XKEY6 - - vmovdqa XKEY_B, [KEYS + 8*16] - - AES XDATA0, XKEY_A ; 7. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - - vmovdqa XKEY_A, [KEYS + 9*16] - - AES XDATA0, XKEY_B ; 8. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 10*16] - - AES XDATA0, XKEY_A ; 9. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A - -%if %%NROUNDS >= 12 - vmovdqa XKEY_A, [KEYS + 11*16] - - AES XDATA0, XKEY_B ; 10. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 12*16] - - AES XDATA0, XKEY_A ; 11. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A -%endif - -%if %%NROUNDS == 14 - vmovdqa XKEY_A, [KEYS + 13*16] - - AES XDATA0, XKEY_B ; 12. ENC - AES XDATA1, XKEY_B - AES XDATA2, XKEY_B - AES XDATA3, XKEY_B - - vmovdqa XKEY_B, [KEYS + 14*16] - - AES XDATA0, XKEY_A ; 13. ENC - AES XDATA1, XKEY_A - AES XDATA2, XKEY_A - AES XDATA3, XKEY_A -%endif - - AES_LAST XDATA0, XKEY_B ; 10/12/14. ENC (depending on key size) - AES_LAST XDATA1, XKEY_B - AES_LAST XDATA2, XKEY_B - AES_LAST XDATA3, XKEY_B - - vmovdqu [OUT + IDX + 0*16 - 4*16], XDATA0 - vmovdqu [OUT + IDX + 1*16 - 4*16], XDATA1 - vmovdqu [OUT + IDX + 2*16 - 4*16], XDATA2 - vmovdqu [OUT + IDX + 3*16 - 4*16], XDATA3 - - cmp IDX, LEN - jne %%main_loop - -%%done: - -%ifdef SAFE_DATA - clear_all_xmms_avx_asm -%endif ;; SAFE_DATA - - ret - -%endmacro - -;; -;; AES-ECB 128 functions -;; -%ifdef AES_ECB_ENC_128 -align 16 -MKGLOBAL(AES_ECB_ENC_128,function,internal) -AES_ECB_ENC_128: - - AES_ECB 10, ENC - -align 16 -MKGLOBAL(AES_ECB_DEC_128,function,internal) -AES_ECB_DEC_128: - - AES_ECB 10, DEC - -%endif - -;; -;; AES-ECB 192 functions -;; -%ifdef AES_ECB_ENC_192 -align 16 -MKGLOBAL(AES_ECB_ENC_192,function,internal) -AES_ECB_ENC_192: - - AES_ECB 12, ENC - -align 16 -MKGLOBAL(AES_ECB_DEC_192,function,internal) -AES_ECB_DEC_192: - - AES_ECB 12, DEC - -%endif - -;; -;; AES-ECB 256 functions -;; -%ifdef AES_ECB_ENC_256 -align 16 -MKGLOBAL(AES_ECB_ENC_256,function,internal) -AES_ECB_ENC_256: - - AES_ECB 14, ENC - -align 16 -MKGLOBAL(AES_ECB_DEC_256,function,internal) -AES_ECB_DEC_256: - - AES_ECB 14, DEC - -%endif - -mksection stack-noexec diff --git a/lib/avx/aes128_ecb_by8_avx.asm b/lib/avx/aes128_ecb_by8_avx.asm new file mode 100644 index 00000000..97152c61 --- /dev/null +++ b/lib/avx/aes128_ecb_by8_avx.asm @@ -0,0 +1,168 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 128 encrypt/decrypt on 16n bytes doing AES by 8 + +%include "include/os.asm" +%include "include/clear_regs.asm" +%include "include/aes_common.asm" +%include "include/cet.inc" + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define LEN rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define LEN r9 +%endif +%define IDX rax +%define TMP r11 +%define XDATA0 xmm0 +%define XDATA1 xmm1 +%define XDATA2 xmm2 +%define XDATA3 xmm3 +%define XDATA4 xmm4 +%define XDATA5 xmm5 +%define XDATA6 xmm6 +%define XDATA7 xmm7 +%define XKEY1 xmm8 + +%ifndef AES_ECB_NROUNDS +%define AES_ECB_NROUNDS 10 +%endif + +%if AES_ECB_NROUNDS == 10 +%define KEYSIZE 128 +%elif AES_ECB_NROUNDS == 12 +%define KEYSIZE 192 +%else +%define KEYSIZE 256 +%endif + +%define AES_ECB_ENC aes_ecb_enc_ %+ KEYSIZE %+ _avx +%define AES_ECB_DEC aes_ecb_dec_ %+ KEYSIZE %+ _avx + +%macro AES_ECB 1 +%define %%DIR %1 ; [in] Direction (ENC/DIR) +%ifidn %%DIR, ENC +%define AES XMM_AESENC_ROUND_BLOCKS_AVX_0_8 +%else ; DIR = DEC +%define AES XMM_AESDEC_ROUND_BLOCKS_AVX_0_8 +%endif + or LEN, LEN + jz %%done + xor IDX, IDX + mov TMP, LEN + and TMP, 127 ; number of initial bytes (0 to 7 AES blocks) + jz %%main_loop + ; branch to different code block based on remainder + cmp TMP, 4*16 + je %%initial_num_blocks_is_4 + jb %%initial_num_blocks_is_3_1 + cmp TMP, 6*16 + je %%initial_num_blocks_is_6 + jb %%initial_num_blocks_is_5 + ja %%initial_num_blocks_is_7 +%%initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2*16 + ja %%initial_num_blocks_is_3 + je %%initial_num_blocks_is_2 + ;; fall through for `jmp %%initial_num_blocks_is_1` +%assign num_blocks 1 +%rep 7 +%%initial_num_blocks_is_ %+ num_blocks : + ; load initial blocks + XMM_LOAD_BLOCKS_AVX_0_8 num_blocks, IN, 0, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption/decryption on initial blocks +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + vmovdqu XKEY1, [KEYS + %%I*16] + AES XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_blocks, (AES_ECB_NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store initial blocks + XMM_STORE_BLOCKS_AVX_0_8 num_blocks, OUT, 0, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7 + add IDX, num_blocks*16 + cmp IDX, LEN + je %%done +%assign num_blocks (num_blocks + 1) + jmp %%main_loop +%endrep +align 16 +%%main_loop: + ; load next 8 blocks + XMM_LOAD_BLOCKS_AVX_0_8 8, {IN + IDX}, 0, XDATA0,\ + XDATA1, XDATA2, XDATA3, XDATA4, XDATA5,\ + XDATA6, XDATA7 +%assign %%I 0 +; Perform AES encryption/decryption on 8 blocks +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + vmovdqu XKEY1, [KEYS + %%I*16] + AES XDATA0, XDATA1, XDATA2, XDATA3, XDATA4,\ + XDATA5, XDATA6, XDATA7, XKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, 8, (AES_ECB_NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + ; store 8 blocks + XMM_STORE_BLOCKS_AVX_0_8 8, {OUT + IDX}, 0, XDATA0, XDATA1,\ + XDATA2, XDATA3, XDATA4, XDATA5, XDATA6, XDATA7 + add IDX, 8*16 + cmp IDX, LEN + jne %%main_loop +%%done: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif +%endmacro + +mksection .text +align 16 +MKGLOBAL(AES_ECB_ENC,function,internal) +AES_ECB_ENC: + endbranch64 + AES_ECB ENC + ret +align 16 +MKGLOBAL(AES_ECB_DEC,function,internal) +AES_ECB_DEC: + endbranch64 + AES_ECB DEC + ret + +mksection stack-noexec diff --git a/lib/avx/aes192_ecb_by4_avx.asm b/lib/avx/aes192_ecb_by8_avx.asm similarity index 89% rename from lib/avx/aes192_ecb_by4_avx.asm rename to lib/avx/aes192_ecb_by8_avx.asm index d8272830..873538fe 100644 --- a/lib/avx/aes192_ecb_by4_avx.asm +++ b/lib/avx/aes192_ecb_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2022, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -; routine to do AES ECB 192 encrypt/decrypt on 16n bytes doing AES by 4 +; routine to do AES ECB 192 encrypt/decrypt on 16n bytes doing AES by 8 -%define AES_ECB_ENC_192 aes_ecb_enc_192_avx -%define AES_ECB_DEC_192 aes_ecb_dec_192_avx +%define AES_ECB_NROUNDS 12 -%include "avx/aes128_ecb_by4_avx.asm" +%include "include/os.asm" +%include "avx/aes128_ecb_by8_avx.asm" diff --git a/lib/avx/aes256_ecb_by4_avx.asm b/lib/avx/aes256_ecb_by8_avx.asm similarity index 89% rename from lib/avx/aes256_ecb_by4_avx.asm rename to lib/avx/aes256_ecb_by8_avx.asm index 9bc75902..44423419 100644 --- a/lib/avx/aes256_ecb_by4_avx.asm +++ b/lib/avx/aes256_ecb_by8_avx.asm @@ -1,5 +1,5 @@ ;; -;; Copyright (c) 2021-2022, Intel Corporation +;; Copyright (c) 2022, Intel Corporation ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are met: @@ -25,9 +25,9 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; -; routine to do AES ECB 256 encrypt/decrypt on 16n bytes doing AES by 4 +; routine to do AES ECB 256 encrypt/decrypt on 16n bytes doing AES by 8 -%define AES_ECB_ENC_256 aes_ecb_enc_256_avx -%define AES_ECB_DEC_256 aes_ecb_dec_256_avx +%define AES_ECB_NROUNDS 14 -%include "avx/aes128_ecb_by4_avx.asm" +%include "include/os.asm" +%include "avx/aes128_ecb_by8_avx.asm" diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 57f4c119..1598a4d7 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -165,9 +165,9 @@ lib_objs1 = \ $(OBJ_DIR)\aes128_ecb_by8_sse.obj \ $(OBJ_DIR)\aes192_ecb_by8_sse.obj \ $(OBJ_DIR)\aes256_ecb_by8_sse.obj \ - $(OBJ_DIR)\aes128_ecb_by4_avx.obj \ - $(OBJ_DIR)\aes192_ecb_by4_avx.obj \ - $(OBJ_DIR)\aes256_ecb_by4_avx.obj \ + $(OBJ_DIR)\aes128_ecb_by8_avx.obj \ + $(OBJ_DIR)\aes192_ecb_by8_avx.obj \ + $(OBJ_DIR)\aes256_ecb_by8_avx.obj \ $(OBJ_DIR)\aes_ecb_vaes_avx512.obj \ $(OBJ_DIR)\pon_by8_sse.obj \ $(OBJ_DIR)\aes128_cntr_by8_sse.obj \ -- GitLab From 8c71d036f1bc5b17583304680bc71a0deab58247 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Mon, 4 Apr 2022 16:16:44 +0000 Subject: [PATCH 049/369] avx2: [AES-ECB] add new VAES implementation --- README | 8 +- README.md | 10 +- ReleaseNotes.txt | 1 + lib/Makefile | 3 + lib/avx2/aes128_ecb_vaes_avx2.asm | 221 ++++++++++++++++++++++++++ lib/avx2/aes192_ecb_vaes_avx2.asm | 33 ++++ lib/avx2/aes256_ecb_vaes_avx2.asm | 33 ++++ lib/avx2/mb_mgr_avx2.c | 68 ++++++-- lib/include/aes_common.asm | 247 +++++++++++++++++++++++++++++- lib/include/asm.h | 14 ++ lib/win_x64.mak | 3 + 11 files changed, 621 insertions(+), 20 deletions(-) create mode 100644 lib/avx2/aes128_ecb_vaes_avx2.asm create mode 100644 lib/avx2/aes192_ecb_vaes_avx2.asm create mode 100644 lib/avx2/aes256_ecb_vaes_avx2.asm diff --git a/README b/README index 90a29399..3499ac1a 100644 --- a/README +++ b/README @@ -75,9 +75,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES128-ECB | N | Y(1) | Y by8 | N | N | Y by16 | - | AES192-ECB | N | Y(1) | Y by8 | N | N | Y by16 | - | AES256-ECB | N | Y(1) | Y by8 | N | N | Y by16 | + | AES128-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | + | AES192-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | + | AES256-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | @@ -105,6 +105,8 @@ On CPU’s supporting GFNI, decryption is by8 and encryption is x8. (7) - same as AES128-CBC for AVX, combines cipher and CRC32 (8) - decryption is by16 and encryption is x16 (9) - currently 1:9 crypt:skip pattern supported +(10) - by default, decryption and encryption are AVX by8. +On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16. Legend: byY - single buffer Y blocks at a time diff --git a/README.md b/README.md index 1ba8bad0..a928cf5b 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | -| AES128-ECB | N | Y(1) | Y by4 | N | N | Y by16 | -| AES192-ECB | N | Y(1) | Y by4 | N | N | Y by16 | -| AES256-ECB | N | Y(1) | Y by4 | N | N | Y by16 | +| AES128-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | +| AES192-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | +| AES256-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | @@ -95,7 +95,9 @@ Notes: (6) - decryption is by16 and encryption is x16 (7) - same as AES128-CBC for AVX, combines cipher and CRC32 (8) - decryption is by16 and encryption is x16 -(9) - currently 1:9 crypt:skip pattern supported +(9) - currently 1:9 crypt:skip pattern supported +(10) - by default, decryption and encryption are AVX by8. + On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16. Legend: ` byY` - single buffer Y blocks at a time diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index f889336c..004711fc 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -9,6 +9,7 @@ Library - AES-ECB AVX512-VAES implementation added - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE +- AES-ECB AVX2-VAES implementation added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) diff --git a/lib/Makefile b/lib/Makefile index 68f62e25..eafa09d3 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -544,6 +544,9 @@ asm_avx2_lib_objs := \ sha256_oct_avx2.o \ sha512_x4_avx2.o \ zuc_x8_avx2.o \ + aes128_ecb_vaes_avx2.o \ + aes192_ecb_vaes_avx2.o \ + aes256_ecb_vaes_avx2.o \ mb_mgr_hmac_md5_flush_avx2.o \ mb_mgr_hmac_md5_submit_avx2.o \ mb_mgr_hmac_sha1_flush_avx2.o \ diff --git a/lib/avx2/aes128_ecb_vaes_avx2.asm b/lib/avx2/aes128_ecb_vaes_avx2.asm new file mode 100644 index 00000000..43726df2 --- /dev/null +++ b/lib/avx2/aes128_ecb_vaes_avx2.asm @@ -0,0 +1,221 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB encrypt/decrypt on 16n bytes doing AES by 16 + +; YMM registers are clobbered. Saving/restoring must be done at a higher level + +; void aes_ecb_x_y_vaes_avx2(void *in, +; UINT128 keys[], +; void *out, +; UINT64 len_bytes); +; +; x = direction (enc/dec) +; y = key size (128/192/256) +; arg 1: IN: pointer to input (cipher text) +; arg 2: KEYS: pointer to keys +; arg 3: OUT: pointer to output (plain text) +; arg 4: LEN: length in bytes (multiple of 16) +; + +%include "include/os.asm" +%include "include/clear_regs.asm" +%include "include/aes_common.asm" +%include "include/cet.inc" + +%ifdef LINUX +%define IN rdi +%define KEYS rsi +%define OUT rdx +%define LEN rcx +%else +%define IN rcx +%define KEYS rdx +%define OUT r8 +%define LEN r9 +%endif +%define IDX rax +%define TMP r11 + +%define YKEY1 ymm1 +%define YDATA0 ymm2 +%define YDATA1 ymm3 +%define YDATA2 ymm4 +%define YDATA3 ymm5 +%define YDATA4 ymm6 +%define YDATA5 ymm7 +%define YDATA6 ymm8 +%define YDATA7 ymm9 + +%ifndef AES_ECB_NROUNDS +%define AES_ECB_NROUNDS 10 +%endif + +%if AES_ECB_NROUNDS == 10 +%define KEYSIZE 128 +%elif AES_ECB_NROUNDS == 12 +%define KEYSIZE 192 +%else +%define KEYSIZE 256 +%endif + +%define AES_ECB_ENC aes_ecb_enc_ %+ KEYSIZE %+ _vaes_avx2 +%define AES_ECB_DEC aes_ecb_dec_ %+ KEYSIZE %+ _vaes_avx2 + +%macro AES_ECB 1 +%define %%DIR %1 ; [in] Direction (ENC/DIR) +%ifidn %%DIR, ENC +%define AES YMM_AESENC_ROUND_BLOCKS_0_16 +%else ; DIR = DEC +%define AES YMM_AESDEC_ROUND_BLOCKS_0_16 +%endif + + or LEN, LEN + jz %%done + + xor IDX, IDX + mov TMP, LEN + and TMP, 255 ; number of initial bytes (0 to 15 AES blocks) + jz %%main_loop + + ; branch to different code block based on remainder + cmp TMP, 8*16 + je %%initial_num_blocks_is_8 + jb %%initial_num_blocks_is_7_1 + cmp TMP, 12*16 + je %%initial_num_blocks_is_12 + jb %%initial_num_blocks_is_11_9 + ;; 15, 14 or 13 + cmp TMP, 14*16 + ja %%initial_num_blocks_is_15 + je %%initial_num_blocks_is_14 + jmp %%initial_num_blocks_is_13 +%%initial_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp TMP, 10*16 + ja %%initial_num_blocks_is_11 + je %%initial_num_blocks_is_10 + jmp %%initial_num_blocks_is_9 +%%initial_num_blocks_is_7_1: + cmp TMP, 4*16 + je %%initial_num_blocks_is_4 + jb %%initial_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp TMP, 6*16 + ja %%initial_num_blocks_is_7 + je %%initial_num_blocks_is_6 + jmp %%initial_num_blocks_is_5 +%%initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2*16 + ja %%initial_num_blocks_is_3 + je %%initial_num_blocks_is_2 + ;; fall through for `jmp %%initial_num_blocks_is_1` + +%assign num_blocks 1 +%rep 15 + + %%initial_num_blocks_is_ %+ num_blocks : +%assign %%I 0 + ; load initial blocks + YMM_LOAD_BLOCKS_AVX2_0_16 num_blocks, IN, 0, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + +; Perform AES encryption/decryption on initial blocks +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + vbroadcasti128 YKEY1, [KEYS + %%I*16] + AES YDATA0, YDATA1, YDATA2, YDATA3, YDATA4,\ + YDATA5, YDATA6, YDATA7, YKEY1, %%I, no_data,\ + no_data, no_data, no_data, no_data, no_data,\ + no_data, no_data, num_blocks, (AES_ECB_NROUNDS - 1) +%assign %%I (%%I + 1) +%endrep + + ; store initial blocks + YMM_STORE_BLOCKS_0_16 num_blocks, OUT, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + add IDX, num_blocks*16 + cmp IDX, LEN + je %%done + +%assign num_blocks (num_blocks + 1) + jmp %%main_loop +%endrep + +align 16 +%%main_loop: + ; load the next 16 blocks into ymm registers + YMM_LOAD_BLOCKS_AVX2_0_16 16, {IN + IDX}, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + ; Perform AES encryption/decryption on 16 blocks +%assign %%ROUNDNO 0 ; current key number +%rep (AES_ECB_NROUNDS + 1) ; 10/12/14 + vbroadcasti128 YKEY1, [KEYS + %%ROUNDNO*16] + + AES YDATA0, YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7, YKEY1, %%ROUNDNO, no_data, no_data,\ + no_data, no_data, no_data, no_data, no_data, no_data,\ + 16, (AES_ECB_NROUNDS - 1) + +%assign %%ROUNDNO (%%ROUNDNO + 1) +%endrep + + ; store 16 blocks + YMM_STORE_BLOCKS_0_16 16, {OUT + IDX}, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + add IDX, 16*16 + cmp IDX, LEN + jne %%main_loop + +%%done: + +%ifdef SAFE_DATA + clear_all_ymms_asm +%else + vzeroupper +%endif +%endmacro + +mksection .text +align 16 +MKGLOBAL(AES_ECB_ENC,function,internal) +AES_ECB_ENC: + endbranch64 + AES_ECB ENC + ret +align 16 +MKGLOBAL(AES_ECB_DEC,function,internal) +AES_ECB_DEC: + endbranch64 + AES_ECB DEC + ret + +mksection stack-noexec diff --git a/lib/avx2/aes192_ecb_vaes_avx2.asm b/lib/avx2/aes192_ecb_vaes_avx2.asm new file mode 100644 index 00000000..9f8dbdb5 --- /dev/null +++ b/lib/avx2/aes192_ecb_vaes_avx2.asm @@ -0,0 +1,33 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 192 encrypt/decrypt on 16n bytes doing AES by 8 + +%define AES_ECB_NROUNDS 12 + +%include "include/os.asm" +%include "avx2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/avx2/aes256_ecb_vaes_avx2.asm b/lib/avx2/aes256_ecb_vaes_avx2.asm new file mode 100644 index 00000000..448293c3 --- /dev/null +++ b/lib/avx2/aes256_ecb_vaes_avx2.asm @@ -0,0 +1,33 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +; routine to do AES ECB 256 encrypt/decrypt on 16n bytes doing AES by 8 + +%define AES_ECB_NROUNDS 14 + +%include "include/os.asm" +%include "avx2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index ac7a0c2d..433ff08c 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -118,12 +118,12 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx #define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx2 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx2 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx2 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx2 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx2 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx2 #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx @@ -148,12 +148,12 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx2 +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx2 +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx2 +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx2 +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx2 +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx2 #define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx #define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx @@ -319,6 +319,41 @@ uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); /* ====================================================================== */ +/* + * ECB function pointers + */ +static void +(*aes_ecb_enc_128_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_128_avx; + +static void +(*aes_ecb_enc_192_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_192_avx; + +static void +(*aes_ecb_enc_256_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_enc_256_avx; + +static void +(*aes_ecb_dec_128_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_128_avx; + +static void +(*aes_ecb_dec_192_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_192_avx; + +static void +(*aes_ecb_dec_256_avx2) (const void *in, const void *keys, + void *out, uint64_t len_bytes)= + aes_ecb_dec_256_avx; + +/* ====================================================================== */ + /* * GCM submit / flush API for AVX2 arch */ @@ -835,6 +870,15 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) /* Set architecture for future checks */ state->used_arch = (uint32_t) IMB_ARCH_AVX2; + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + aes_ecb_enc_128_avx2 = aes_ecb_enc_128_vaes_avx2; + aes_ecb_enc_192_avx2 = aes_ecb_enc_192_vaes_avx2; + aes_ecb_enc_256_avx2 = aes_ecb_enc_256_vaes_avx2; + aes_ecb_dec_128_avx2 = aes_ecb_dec_128_vaes_avx2; + aes_ecb_dec_192_avx2 = aes_ecb_dec_192_vaes_avx2; + aes_ecb_dec_256_avx2 = aes_ecb_dec_256_vaes_avx2; + } + if (reset_mgrs) { reset_ooo_mgrs(state); diff --git a/lib/include/aes_common.asm b/lib/include/aes_common.asm index afb035fb..f0410dea 100644 --- a/lib/include/aes_common.asm +++ b/lib/include/aes_common.asm @@ -997,7 +997,6 @@ %endmacro - ;;; =========================================================================== ;;; Handles AES decryption rounds for 0 to 8 blocks on AVX ;;; It handles special cases: the last and first rounds @@ -1153,6 +1152,7 @@ %%L0B4, %%L0B5, %%L0B6, %%L0B7, \ %%D0, %%D1, %%D2, %%D3, \ %%D4, %%D5, %%D6, %%D7 + %endif ; !no_data %endif ; !no_data %endif ; !no_data @@ -1277,3 +1277,248 @@ %endrep %endmacro + +;; ============================================================================= +;; Loads specified number of AES blocks into YMM registers +%macro YMM_LOAD_BLOCKS_AVX2_0_16 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%INP %2 ; [in] input data pointer to read from +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%DST0 %4 ; [out] YMM register with loaded data +%define %%DST1 %5 ; [out] YMM register with loaded data +%define %%DST2 %6 ; [out] YMM register with loaded data +%define %%DST3 %7 ; [out] YMM register with loaded data +%define %%DST4 %8 ; [out] YMM register with loaded data +%define %%DST5 %9 ; [out] YMM register with loaded data +%define %%DST6 %10 ; [out] YMM register with loaded data +%define %%DST7 %11 ; [out] YMM register with loaded data + +%assign src_offset 0 +%assign dst_idx 0 + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%DSTREG %%DST %+ dst_idx + vmovdqu %%DSTREG, [%%INP + %%DATA_OFFSET + src_offset] +%undef %%DSTREG +%assign src_offset (src_offset + 32) +%assign dst_idx (dst_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 2) +%xdefine %%DSTREG %%DST %+ dst_idx + +%if blocks_left == 1 + vmovdqu XWORD(%%DSTREG), [%%INP + %%DATA_OFFSET + src_offset] +%endif + +%endmacro + +;; ============================================================================= +;; Stores specified number of AES blocks from YMM registers +%macro YMM_STORE_BLOCKS_AVX2_0_16 11 +%define %%NUM_BLOCKS %1 ; [in] numerical value, number of AES blocks (0 to 16) +%define %%OUTP %2 ; [in] output data pointer to write to +%define %%DATA_OFFSET %3 ; [in] offset to the output pointer (GP or numerical) +%define %%SRC0 %4 ; [in] YMM register with data to store +%define %%SRC1 %5 ; [in] YMM register with data to store +%define %%SRC2 %6 ; [in] YMM register with data to store +%define %%SRC3 %7 ; [in] YMM register with data to store +%define %%SRC4 %8 ; [in] YMM register with data to store +%define %%SRC5 %9 ; [in] YMM register with data to store +%define %%SRC6 %10 ; [in] YMM register with data to store +%define %%SRC7 %11 ; [in] YMM register with data to store + +%assign dst_offset 0 +%assign src_idx 0 + +%rep (%%NUM_BLOCKS / 2) +%xdefine %%SRCREG %%SRC %+ src_idx + vmovdqu [%%OUTP + %%DATA_OFFSET + dst_offset], %%SRCREG +%undef %%SRCREG +%assign dst_offset (dst_offset + 32) +%assign src_idx (src_idx + 1) +%endrep + +%assign blocks_left (%%NUM_BLOCKS % 2) +%xdefine %%SRCREG %%SRC %+ src_idx + +%if blocks_left == 1 + vmovdqu [%%OUTP + %%DATA_OFFSET + dst_offset], XWORD(%%SRCREG) +%endif + +%endmacro + +;;; =========================================================================== +;;; Handles AES encryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameter to check what needs to be done for the current round. +%macro YMM_AESENC_ROUND_BLOCKS_AVX2_0_16 20 +%define %%L0B0_1 %1 ; [in/out] ymm; ciphered blocks +%define %%L0B2_3 %2 ; [in/out] ymm; ciphered blocks +%define %%L0B4_5 %3 ; [in/out] ymm; ciphered blocks +%define %%L0B6_7 %4 ; [in/out] ymm; ciphered blocks +%define %%L0B8_9 %5 ; [in/out] ymm; ciphered blocks +%define %%L0B10_11 %6 ; [in/out] ymm; ciphered blocks +%define %%L0B12_13 %7 ; [in/out] ymm; ciphered blocks +%define %%L0B14_15 %8 ; [in/out] ymm; ciphered blocks +%define %%KEY %9 ; [in] ymm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0_1 %11 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D2_3 %12 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D4_5 %13 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D6_7 %14 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D8_9 %15 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D10_11 %16 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D12_13 %17 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D14_15 %18 ; [in] ymm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxor, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenc, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesenclast, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +%ifnidn %%D0_1, no_data +%ifnidn %%D2_3, no_data +%ifnidn %%D4_5, no_data +%ifnidn %%D6_7, no_data +%ifnidn %%D8_9, no_data +%ifnidn %%D10_11, no_data +%ifnidn %%D12_13, no_data +%ifnidn %%D14_15, no_data + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxor, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%D0_1, %%D2_3, %%D4_5, %%D6_7, \ + %%D8_9, %%D10_11, %%D12_13, %%D14_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro + +;;; =========================================================================== +;;; Handles AES decryption rounds +;;; It handles special cases: the last and first rounds +;;; Optionally, it performs XOR with data after the last AES round. +;;; Uses NROUNDS parameterto check what needs to be done for the current round. +%macro YMM_AESDEC_ROUND_BLOCKS_AVX2_0_16 20 +%define %%L0B0_1 %1 ; [in/out] ymm; ciphered blocks +%define %%L0B2_3 %2 ; [in/out] ymm; ciphered blocks +%define %%L0B4_5 %3 ; [in/out] ymm; ciphered blocks +%define %%L0B6_7 %4 ; [in/out] ymm; ciphered blocks +%define %%L0B8_9 %5 ; [in/out] ymm; ciphered blocks +%define %%L0B10_11 %6 ; [in/out] ymm; ciphered blocks +%define %%L0B12_13 %7 ; [in/out] ymm; ciphered blocks +%define %%L0B14_15 %8 ; [in/out] ymm; ciphered blocks +%define %%KEY %9 ; [in] ymm containing round key +%define %%ROUND %10 ; [in] round number +%define %%D0_1 %11 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D2_3 %12 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D4_5 %13 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D6_7 %14 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D8_9 %15 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D10_11 %16 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D12_13 %17 ; [in] ymm or no_data; plain/cipher text blocks +%define %%D14_15 %18 ; [in] ymm or no_data; plain/cipher text blocks +%define %%NUMBL %19 ; [in] number of blocks; numerical value +%define %%NROUNDS %20 ; [in] number of rounds; numerical value + +;;; === first AES round +%if (%%ROUND < 1) + ;; round 0 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxor, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; ROUND 0 + +;;; === middle AES rounds +%if (%%ROUND >= 1 && %%ROUND <= %%NROUNDS) + ;; rounds 1 to 9/11/13 + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdec, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY +%endif ; rounds 1 to 9/11/13 + +;;; === last AES round +%if (%%ROUND > %%NROUNDS) + ;; the last round - mix enclast with text xor's + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vaesdeclast, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY, %%KEY + +;;; === XOR with data +%ifnidn %%D0_1, no_data +%ifnidn %%D2_3, no_data +%ifnidn %%D4_5, no_data +%ifnidn %%D6_7, no_data +%ifnidn %%D8_9, no_data +%ifnidn %%D10_11, no_data +%ifnidn %%D12_13, no_data +%ifnidn %%D14_15, no_data + YMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 %%NUMBL, vpxor, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%L0B0_1, %%L0B2_3, %%L0B4_5, %%L0B6_7, \ + %%L0B8_9, %%L0B10_11, %%L0B12_13, %%L0B14_15, \ + %%D0_1, %%D2_3, %%D4_5, %%D6_7, \ + %%D8_9, %%D10_11, %%D12_13, %%D14_15 +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data +%endif ; !no_data + +%endif ; The last round + +%endmacro diff --git a/lib/include/asm.h b/lib/include/asm.h index 4739e012..f46bf955 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -180,6 +180,13 @@ void aes_ecb_enc_192_avx(const void *in, const void *keys, void aes_ecb_enc_128_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_enc_256_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); + void aes_ecb_enc_256_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, @@ -216,6 +223,13 @@ void aes_ecb_dec_192_avx(const void *in, const void *keys, void aes_ecb_dec_128_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); +void aes_ecb_dec_256_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); + void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 1598a4d7..ca8cb180 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -168,6 +168,9 @@ lib_objs1 = \ $(OBJ_DIR)\aes128_ecb_by8_avx.obj \ $(OBJ_DIR)\aes192_ecb_by8_avx.obj \ $(OBJ_DIR)\aes256_ecb_by8_avx.obj \ + $(OBJ_DIR)\aes128_ecb_vaes_avx2.obj \ + $(OBJ_DIR)\aes192_ecb_vaes_avx2.obj \ + $(OBJ_DIR)\aes256_ecb_vaes_avx2.obj \ $(OBJ_DIR)\aes_ecb_vaes_avx512.obj \ $(OBJ_DIR)\pon_by8_sse.obj \ $(OBJ_DIR)\aes128_cntr_by8_sse.obj \ -- GitLab From 3726fae1494c4d05d70120bc9fa3120687eea38c Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 14 Mar 2022 14:10:44 +0000 Subject: [PATCH 050/369] avx2: [ZUC-EIA3-256] Add tag size parameter to EIA3-256 C function Add tag size parameter to ZUC-EIA3-256 C function (in top file), in preparation for other tag sizes. It also makes pMacI parameter of void* type, instead of uint32_t *, as tag will be of different sizes. --- lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm | 28 +++++++++++++++++++++-- lib/avx2/zuc_top_avx2.c | 16 +++++++------ lib/include/zuc_internal.h | 5 ++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm index cbe1be55..86c40630 100644 --- a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm @@ -92,13 +92,15 @@ extern asm_ZucCipher_8_avx2 %define arg4 rcx %define arg5 r8 %define arg6 r9 +%define arg7 qword [rsp] %else %define arg1 rcx %define arg2 rdx %define arg3 r8 %define arg4 r9 -%define arg5 [rsp + 32] -%define arg6 [rsp + 40] +%define arg5 qword [rsp + 32] +%define arg6 qword [rsp + 40] +%define arg7 qword [rsp + 48] %endif %define state arg1 @@ -853,7 +855,11 @@ FLUSH_JOB_ZUC256_EEA3: ; to pass parameter to next function mov r11, state +%if %%KEY_SIZE == 128 RESERVE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESERVE_STACK_SPACE 7 +%endif lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] @@ -868,6 +874,9 @@ FLUSH_JOB_ZUC256_EEA3: lea r12, [r11 + _zuc_job_in_lane] mov arg6, r12 %endif +%if %%KEY_SIZE == 256 + mov arg7, 4 +%endif %if %%KEY_SIZE == 128 call zuc_eia3_8_buffer_job_avx2 @@ -875,7 +884,11 @@ FLUSH_JOB_ZUC256_EEA3: call zuc256_eia3_8_buffer_job_avx2 %endif +%if %%KEY_SIZE == 128 RESTORE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 +%endif mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -1002,7 +1015,11 @@ APPEND(%%skip_eia3_,I): ; to pass parameter to next function mov r11, state +%if %%KEY_SIZE == 128 RESERVE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESERVE_STACK_SPACE 7 +%endif lea arg1, [r11 + _zuc_args_keys] lea arg2, [r11 + _zuc_args_IV] @@ -1017,6 +1034,9 @@ APPEND(%%skip_eia3_,I): lea r12, [r11 + _zuc_job_in_lane] mov arg6, r12 %endif +%if %%KEY_SIZE == 256 + mov arg7, 4 +%endif %if %%KEY_SIZE == 128 call zuc_eia3_8_buffer_job_avx2 @@ -1024,7 +1044,11 @@ APPEND(%%skip_eia3_,I): call zuc256_eia3_8_buffer_job_avx2 %endif +%if %%KEY_SIZE == 128 RESTORE_STACK_SPACE 6 +%else ; %%KEY_SIZE == 256 + RESTORE_STACK_SPACE 7 +%endif vmovdqa xmm2, [rsp + _null_len_save] diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index 62baae7d..a2b2ba96 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -846,9 +846,10 @@ void zuc_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], const uint8_t *ivs, const void * const pBufferIn[NUM_AVX2_BUFS], - uint32_t *pMacI[NUM_AVX2_BUFS], + void *pMacI[NUM_AVX2_BUFS], const uint16_t lengthInBits[NUM_AVX2_BUFS], - const void * const job_in_lane[NUM_AVX2_BUFS]) + const void * const job_in_lane[NUM_AVX2_BUFS], + const uint64_t tag_size) { unsigned int i = 0; DECLARE_ALIGNED(ZucState8_t state, 64); @@ -873,13 +874,13 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], } /* TODO: Handle 8 and 16-byte digest cases */ - asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, 4); + asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, tag_size); asm_ZucGenKeystream4B_8_avx2(&state, pKeyStrArr); /* Initialize the tag with the first 4 bytes of the keystream */ for (i = 0; i < NUM_AVX2_BUFS; i++) - memcpy(&T[i], pKeyStrArr[i], 4); + memcpy(&T[i], pKeyStrArr[i], tag_size); /* Generate 32 bytes at a time */ asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); @@ -955,7 +956,8 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], else asm_ZucGenKeystream32B_avx(&keyStr32[8], &singlePktState); - asm_Eia3Round32B_avx(tag, &keyStr32[0], pIn8[i], 4); + asm_Eia3Round32B_avx(tag, &keyStr32[0], pIn8[i], + tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } @@ -969,10 +971,10 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], &singlePktState, 1); asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits, - 256, 4); + 256, tag_size); /* save the final MAC-I result */ - memcpy(pMacI[i], tag, 4); + memcpy(pMacI[i], tag, tag_size); } #ifdef SAFE_DATA diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 5428fdaa..8132e518 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1317,9 +1317,10 @@ IMB_DLL_LOCAL void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[8], const uint8_t *ivs, const void * const pBufferIn[8], - uint32_t *pMacI[8], + void *pMacI[8], const uint16_t lengthInBits[8], - const void * const job_in_lane[8]); + const void * const job_in_lane[8], + const uint64_t tag_size); /* the s-boxes */ extern const uint8_t S0[256]; -- GitLab From 2c6a66bc769c68fcfffa8f0392e960326340b04c Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 14 Mar 2022 15:11:57 +0000 Subject: [PATCH 051/369] avx2: [ZUC-EIA3-256] Initialize tags at LFSR init Initialize the authentication tags for ZUC-256 in initialization function, also modifying how the constant tables are loaded (for cipher, it is no longer required to pass a "dummy" value of 2. --- lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm | 21 ++---- lib/avx2/zuc_top_avx2.c | 8 +- lib/avx2/zuc_x8_avx2.asm | 90 ++++++++++++++++++----- lib/include/zuc_internal.h | 9 ++- 4 files changed, 87 insertions(+), 41 deletions(-) diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm index 86c40630..9a9583bd 100644 --- a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm @@ -305,24 +305,19 @@ mksection .text %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 4 + RESERVE_STACK_SPACE 5 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] -%if %%KEY_SIZE == 256 - ;; Setting "tag size" to 2 in case of ciphering - ;; (dummy size, just for constant selecion at Initialization) - mov arg4, 2 -%endif - %if %%KEY_SIZE == 128 call ZUC128_INIT_8 %else + mov arg5, 0 ; Tag size = 0, arg4 not used call ZUC256_INIT_8 %endif - RESTORE_STACK_SPACE 4 + RESTORE_STACK_SPACE 5 cmp word [r12 + _zuc_init_not_done], 0xff ; Init done for all lanes je %%skip_submit_restoring_state @@ -525,24 +520,20 @@ APPEND(%%skip_eea3_,I): %assign I (I + 1) %endrep - RESERVE_STACK_SPACE 4 + RESERVE_STACK_SPACE 5 lea arg1, [r12 + _zuc_args_keys] lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] -%if %%KEY_SIZE == 256 - ;; Setting "tag size" to 2 in case of ciphering - ;; (dummy size, just for constant selecion at Initialization) - mov arg4, 2 -%endif %if %%KEY_SIZE == 128 call ZUC128_INIT_8 %else + mov arg5, 0 ; Tag size = 0, arg4 not used call ZUC256_INIT_8 %endif - RESTORE_STACK_SPACE 4 + RESTORE_STACK_SPACE 5 cmp word [r12 + _zuc_init_not_done], 0xff ; Init done for all lanes je %%skip_flush_restoring_state diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index a2b2ba96..c346f898 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -874,13 +874,7 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], } /* TODO: Handle 8 and 16-byte digest cases */ - asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, tag_size); - - asm_ZucGenKeystream4B_8_avx2(&state, pKeyStrArr); - - /* Initialize the tag with the first 4 bytes of the keystream */ - for (i = 0; i < NUM_AVX2_BUFS; i++) - memcpy(&T[i], pKeyStrArr[i], tag_size); + asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, T, tag_size); /* Generate 32 bytes at a time */ asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 5ab2cea1..6be6b993 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -57,6 +57,12 @@ Ek_d: dd 0x0044D700, 0x0026BC00, 0x00626B00, 0x00135E00, 0x00578900, 0x0035E200, 0x00713500, 0x0009AF00 dd 0x004D7800, 0x002F1300, 0x006BC400, 0x001AF100, 0x005E2600, 0x003C4D00, 0x00789A00, 0x0047AC00 +; Constants to be used to initialize the LFSR registers +; The tables contain four different sets of constants: +; 0-63 bytes: Encryption +; 64-127 bytes: Authentication with tag size = 4 +; 128-191 bytes: Authentication with tag size = 8 +; 192-255 bytes: Authentication with tag size = 16 align 16 EK256_d64: dd 0x00220000, 0x002F0000, 0x00240000, 0x002A0000, @@ -618,7 +624,13 @@ align 64 %define %%XTMP %5 ;; [clobbered] XMM temporary register %define %%XTMP2 %6 ;; [clobbered] XMM temporary register %define %%TMP %7 ;; [clobbered] GP temporary register -%define %%CONSTANTS %8 ;; [in] Address to constants +%define %%TAG_SIZE %8 ;; [in] Tag size (0, 4, 8 or 16 bytes) + +%if %%TAG_SIZE == 0 +%define %%CONSTANTS rel EK256_d64 +%elif %%TAG_SIZE == 4 +%define %%CONSTANTS rel EK256_EIA3_4 +%endif ; s0 - s7 vpxor %%LFSR0_7, %%LFSR0_7 @@ -743,13 +755,14 @@ align 64 vinserti128 %%LFSR8_15, %%XTMP, 1 %endmacro -%macro ZUC_INIT_8 1 +%macro ZUC_INIT_8 2-3 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [in] Tag size (0 (for cipher), 4, 8 or 16) +%define %%TAGS %3 ; [in] Array of temporary tags %define pKe arg1 %define pIv arg2 %define pState arg3 -%define tag_sz arg4 ; Only used in ZUC-256 %define %%YTMP1 ymm0 %define %%YTMP2 ymm1 @@ -770,6 +783,10 @@ align 64 %define %%W %%YTMP10 %define %%X3 %%YTMP11 +%define %%KSTR1 %%YTMP12 +%define %%KSTR2 %%YTMP13 +%define %%KSTR3 %%YTMP14 +%define %%KSTR4 %%YTMP15 %define %%MASK_31 %%YTMP16 FUNC_SAVE @@ -815,14 +832,6 @@ align 64 %endrep %else ;; %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea r13, [rel EK256_d64] - bsf DWORD(tag_sz), DWORD(tag_sz) - dec DWORD(tag_sz) - shl DWORD(tag_sz), 6 - add r13, tag_sz - ;;; Initialize all LFSR registers %assign %%OFF 0 %rep 8 @@ -831,7 +840,7 @@ align 64 lea r10, [pIv + 4*%%OFF] ; Load IV N pointer ; Initialize S0-15 for each packet - INIT_LFSR_256 r15, r10, %%YTMP1, %%YTMP2, XWORD(%%YTMP3), XWORD(%%YTMP4), r11, r13 + INIT_LFSR_256 r15, r10, %%YTMP1, %%YTMP2, XWORD(%%YTMP3), XWORD(%%YTMP4), r11, %%TAG_SIZE vmovdqa [pState + 4*%%OFF], %%YTMP1 vmovdqa [pState + 256 + 4*%%OFF], %%YTMP2 @@ -865,16 +874,16 @@ align 64 vmovdqa %%MASK_31, [rel mask31] ; Shift LFSR 32-times, update state variables -%assign N 0 +%assign %%N 0 %rep 32 - BITS_REORG8 pState, N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + BITS_REORG8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10 NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W vpsrld %%W, 1 ; Shift out LSB of W - LFSR_UPDT8 pState, N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + LFSR_UPDT8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ %%MASK_31, %%W, init ; W used in LFSR update -%assign N N+1 +%assign %%N %%N+1 %endrep ; And once more, initial round from keygen phase = 33 times @@ -885,6 +894,36 @@ align 64 LFSR_UPDT8 pState, 0, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ %%MASK_31, %%YTMP8, work + ; Generate extra 4, 8 or 16 bytes of KS for initial tags +%if %%TAG_SIZE == 4 +%define %%NUM_ROUNDS 1 +%elif %%TAG_SIZE == 8 +%define %%NUM_ROUNDS 2 +%elif %%TAG_SIZE == 16 +%define %%NUM_ROUNDS 4 +%else +%define %%NUM_ROUNDS 0 +%endif + +%assign %%N 1 +%rep %%NUM_ROUNDS + BITS_REORG8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, \ + %%YTMP6, %%YTMP7, %%YTMP8, %%YTMP9, %%YTMP10, APPEND(%%KSTR,%%N) + NONLIN_FUN8 pState, %%YTMP1, %%YTMP2, %%YTMP3, \ + %%YTMP4, %%YTMP5, %%YTMP6, %%YTMP7, %%W + ; OFS_X3 XOR W and store in stack + vpxor APPEND(%%KSTR, %%N), %%W + LFSR_UPDT8 pState, %%N, %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4, %%YTMP5, %%YTMP6, \ + %%MASK_31, %%YTMP8, work +%assign %%N %%N+1 +%endrep + +%if %%TAG_SIZE == 4 + vmovdqa [%%TAGS], %%KSTR1 + REORDER_LFSR pState, 1 +%elif %%TAG_SIZE == 8 ; TODO +%elif %%TAG_SIZE == 16 ; TODO +%endif FUNC_RESTORE ret @@ -893,12 +932,27 @@ align 64 MKGLOBAL(asm_ZucInitialization_8_avx2,function,internal) asm_ZucInitialization_8_avx2: endbranch64 - ZUC_INIT_8 128 + ZUC_INIT_8 128, 0 MKGLOBAL(asm_Zuc256Initialization_8_avx2,function,internal) asm_Zuc256Initialization_8_avx2: +%define tags arg4 +%define tag_sz arg5 + endbranch64 - ZUC_INIT_8 256 + + cmp tag_sz, 0 + je init_for_cipher + + ;; TODO: Check for 8B and 16B tags + cmp tag_sz, 4 + je init_for_auth_tag_4B + +init_for_cipher: + ZUC_INIT_8 256, 0 + +init_for_auth_tag_4B: + ZUC_INIT_8 256, 4, tags ; ; Generate N*4 bytes of keystream diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 8132e518..659d3696 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -361,6 +361,12 @@ IMB_DLL_LOCAL void asm_ZucInitialization_8_avx2(ZucKey8_t *pKeys, * @param[in,out] pState Pointer to a ZUC state structure of type * @ref ZucState8_t that will be populated * with the initialized ZUC state. + * @param[in,out] tags Array of 4 pointers to authentication tags + * (up to 16 bytes each, only for ZUC-EIA3) + * @param[in] tag_sz Tag size (0, 4, 8 or 16), to select the + * constants used to initialize the LFSR + * the LFSR registers (0 is used in case of + * encryption). * * @pre * None @@ -369,7 +375,8 @@ IMB_DLL_LOCAL void asm_ZucInitialization_8_avx2(ZucKey8_t *pKeys, IMB_DLL_LOCAL void asm_Zuc256Initialization_8_avx2(ZucKey8_t *pKeys, const uint8_t *ivs, ZucState8_t *pState, - const unsigned tag_sz); + void *tags, + const uint64_t tag_sz); /** ****************************************************************************** -- GitLab From e95316baa3e38a56de962ad4045f7a0b322aacfd Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 14 Mar 2022 15:32:45 +0000 Subject: [PATCH 052/369] avx2: [ZUC-EIA3-256] Initialize 8-byte and 16-byte digests --- lib/avx2/zuc_x8_avx2.asm | 84 ++++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 6be6b993..5f8133ad 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -249,6 +249,35 @@ align 64 mov rsp, [rsp + GP_OFFSET + 40] %endmacro +; +; Transpose 4 YMM registers, double word granularity +; +%macro TRANSPOSE4_U32 8 +%define %%R0 %1 ; [in/out] Input / Output row 0 +%define %%R1 %2 ; [in/out] Input / Output row 1 +%define %%R2 %3 ; [in/out] Input / Output row 2 +%define %%R3 %4 ; [in/out] Input / Output row 3 +%define %%T0 %5 ; [clobbered] Temporary YMM register +%define %%T1 %6 ; [clobbered] Temporary YMM register +%define %%T2 %7 ; [clobbered] Temporary YMM register +%define %%T3 %8 ; [clobbered] Temporary YMM register + + vshufps %%T0, %%R0, %%R1, 0x44 ; T0 = {b5 b4 a5 a4 b1 b0 a1 a0} + vshufps %%R0, %%R0, %%R1, 0xEE ; R0 = {b7 b6 a7 a6 b3 b2 a3 a2} + vshufps %%T1, %%R2, %%R3, 0x44 ; T1 = {d5 d4 c5 c4 d1 d0 c1 c0} + vshufps %%R2, %%R2, %%R3, 0xEE ; R2 = {d7 d6 c7 c6 d3 d2 c3 c2} + + vshufps %%T3, %%T0, %%T1, 0xDD ; T3 = {d5 c5 b5 a5 d1 c1 b1 a1} + vshufps %%T2, %%R0, %%R2, 0x88 ; T2 = {d6 c6 b6 a6 d2 c2 b2 a2} + vshufps %%R0, %%R0, %%R2, 0xDD ; R0 = {d7 c7 b7 a7 d3 c3 b3 a3} + vshufps %%T0, %%T0, %%T1, 0x88 ; T0 = {d4 c4 b4 a4 d0 c0 b0 a0} + + vperm2i128 %%R2, %%T0, %%T3, 0x31 ; {d5 c5 b5 a5 d4 c4 b4 a4} + vperm2i128 %%R1, %%T2, %%R0, 0x20 ; {d3 c3 b3 a3 d2 c2 b2 a2} + vperm2i128 %%R3, %%T2, %%R0, 0x31 ; {d7 c7 b7 a7 d6 c6 b6 a6} + vperm2i128 %%R0, %%T0, %%T3, 0x20 ; {d1 c1 b1 a1 d0 c0 b0 a0} +%endmacro + ; This macro reorder the LFSR registers ; after N rounds (1 <= N <= 15), since the registers ; are shifted every round @@ -256,8 +285,8 @@ align 64 ; The macro clobbers YMM0-15 ; %macro REORDER_LFSR 2 -%define %%STATE %1 -%define %%NUM_ROUNDS %2 +%define %%STATE %1 ; [in] Pointer to LFSR state +%define %%NUM_ROUNDS %2 ; [immediate] Number of key generation rounds %if %%NUM_ROUNDS != 16 %assign i 0 @@ -630,6 +659,10 @@ align 64 %define %%CONSTANTS rel EK256_d64 %elif %%TAG_SIZE == 4 %define %%CONSTANTS rel EK256_EIA3_4 +%elif %%TAG_SIZE == 8 +%define %%CONSTANTS rel EK256_EIA3_8 +%elif %%TAG_SIZE == 16 +%define %%CONSTANTS rel EK256_EIA3_16 %endif ; s0 - s7 @@ -921,12 +954,28 @@ align 64 %if %%TAG_SIZE == 4 vmovdqa [%%TAGS], %%KSTR1 REORDER_LFSR pState, 1 -%elif %%TAG_SIZE == 8 ; TODO -%elif %%TAG_SIZE == 16 ; TODO +%elif %%TAG_SIZE == 8 + ; Transpose the keystream and store the 8 bytes per buffer consecutively, + ; being the initial tag for each buffer + vpunpckldq %%YTMP1, %%KSTR1, %%KSTR2 + vpunpckhdq %%YTMP2, %%KSTR1, %%KSTR2 + vperm2i128 %%KSTR1, %%YTMP1, %%YTMP2, 0x20 + vperm2i128 %%KSTR2, %%YTMP1, %%YTMP2, 0x31 + + vmovdqa [%%TAGS], %%KSTR1 + vmovdqa [%%TAGS + 32], %%KSTR2 + REORDER_LFSR pState, 2 +%elif %%TAG_SIZE == 16 + TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, \ + %%YTMP1, %%YTMP2, %%YTMP3, %%YTMP4 + + vmovdqa [%%TAGS], %%KSTR1 + vmovdqa [%%TAGS + 32], %%KSTR2 + vmovdqa [%%TAGS + 32*2], %%KSTR3 + vmovdqa [%%TAGS + 32*3], %%KSTR4 + REORDER_LFSR pState, 4 %endif FUNC_RESTORE - - ret %endmacro MKGLOBAL(asm_ZucInitialization_8_avx2,function,internal) @@ -934,6 +983,8 @@ asm_ZucInitialization_8_avx2: endbranch64 ZUC_INIT_8 128, 0 + ret + MKGLOBAL(asm_Zuc256Initialization_8_avx2,function,internal) asm_Zuc256Initialization_8_avx2: %define tags arg4 @@ -944,15 +995,26 @@ asm_Zuc256Initialization_8_avx2: cmp tag_sz, 0 je init_for_cipher - ;; TODO: Check for 8B and 16B tags - cmp tag_sz, 4 - je init_for_auth_tag_4B + cmp tag_sz, 8 + je init_for_auth_tag_8B + jb init_for_auth_tag_4B -init_for_cipher: - ZUC_INIT_8 256, 0 + ; Fall-through for tag size = 16 bytes +init_for_auth_tag_16B: + ZUC_INIT_8 256, 16, tags + ret + +init_for_auth_tag_8B: + ZUC_INIT_8 256, 8, tags + ret init_for_auth_tag_4B: ZUC_INIT_8 256, 4, tags + ret + +init_for_cipher: + ZUC_INIT_8 256, 0 + ret ; ; Generate N*4 bytes of keystream -- GitLab From 7a59c8fa586ff71d54bab694f9d30fb9fd1da5f1 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 24 Mar 2022 18:39:09 +0000 Subject: [PATCH 053/369] avx2: [ZUC-EIA3-256] Pass tag size as argument to submit/flush --- ReleaseNotes.txt | 2 +- lib/avx2/mb_mgr_avx2.c | 18 +++---- lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm | 64 ++++++++++++++++++----- lib/avx2/zuc_top_avx2.c | 63 +++++++++++++--------- lib/avx2/zuc_x8_avx2.asm | 18 +++++++ lib/include/zuc_internal.h | 23 ++++++++ 6 files changed, 140 insertions(+), 48 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 004711fc..804fbd2e 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -5,7 +5,7 @@ Unreleased ======================================================================== Library -- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE and AVX +- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE, AVX and AVX2 - AES-ECB AVX512-VAES implementation added - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 433ff08c..0e446a5b 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -81,11 +81,11 @@ IMB_JOB *submit_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state); -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); +IMB_JOB *submit_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); void aes_cmac_256_subkey_gen_avx2(const void *key_exp, void *key1, void *key2); @@ -134,8 +134,8 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 #define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 #define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx2 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 #define AES_CBC_DEC_128 aes_cbc_dec_128_avx #define AES_CBC_DEC_192 aes_cbc_dec_192_avx @@ -586,12 +586,12 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(zuc256_eia3_ooo->lens)); memset(zuc256_eia3_ooo->job_in_lane, 0, sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; + zuc256_eia3_ooo->unused_lanes = 0xF76543210; zuc256_eia3_ooo->num_lanes_inuse = 0; memset(&zuc256_eia3_ooo->state, 0, sizeof(zuc256_eia3_ooo->state)); zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; + zuc256_eia3_ooo->unused_lane_bitmask = 0xff; /* Init HMAC/SHA1 out-of-order fields */ hmac_sha_1_ooo->lens[0] = 0; diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm index 9a9583bd..77b565fd 100644 --- a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm @@ -733,8 +733,9 @@ FLUSH_JOB_ZUC256_EEA3: endbranch64 FLUSH_JOB_ZUC_EEA3 256 -%macro SUBMIT_JOB_ZUC_EIA3 1 +%macro SUBMIT_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16) ; idx needs to be in rbp %define len rbp @@ -866,7 +867,7 @@ FLUSH_JOB_ZUC256_EEA3: mov arg6, r12 %endif %if %%KEY_SIZE == 256 - mov arg7, 4 + mov arg7, %%TAG_SIZE %endif %if %%KEY_SIZE == 128 @@ -914,15 +915,18 @@ FLUSH_JOB_ZUC256_EEA3: %endif mov rsp, [rsp + _rsp_save] ; original SP - ret + jmp %%exit_submit_eia3 %%return_null_submit_eia3: xor job_rax, job_rax jmp %%return_submit_eia3 + +%%exit_submit_eia3: %endmacro -%macro FLUSH_JOB_ZUC_EIA3 1 +%macro FLUSH_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16) %define unused_lanes rbx %define tmp1 rbx @@ -1026,7 +1030,7 @@ APPEND(%%skip_eia3_,I): mov arg6, r12 %endif %if %%KEY_SIZE == 256 - mov arg7, 4 + mov arg7, %%TAG_SIZE %endif %if %%KEY_SIZE == 128 @@ -1074,11 +1078,13 @@ APPEND(%%skip_eia3_,I): %endif mov rsp, [rsp + _rsp_save] ; original SP - ret + jmp %%exit_flush_eia3 %%return_null_flush_eia3: xor job_rax, job_rax jmp %%return_flush_eia3 + +%%exit_flush_eia3: %endmacro ; JOB* SUBMIT_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -1087,28 +1093,62 @@ APPEND(%%skip_eia3_,I): MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: endbranch64 - SUBMIT_JOB_ZUC_EIA3 128 + SUBMIT_JOB_ZUC_EIA3 128, 4 + ret -; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) +; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job, +; const uint64_t tag_sz) ; arg 1 : state ; arg 2 : job +; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: endbranch64 - SUBMIT_JOB_ZUC_EIA3 256 + cmp arg3, 8 + je submit_tag_8B + jb submit_tag_4B + + ; Fall-through for 16-byte tag +submit_tag_16B: + SUBMIT_JOB_ZUC_EIA3 256, 16 + ret +submit_tag_8B: + SUBMIT_JOB_ZUC_EIA3 256, 8 + ret +submit_tag_4B: + SUBMIT_JOB_ZUC_EIA3 256, 4 + ret ; JOB* FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: endbranch64 - FLUSH_JOB_ZUC_EIA3 128 + FLUSH_JOB_ZUC_EIA3 128, 4 + ret -; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state) +; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, +; const uint64_t tag_sz) ; arg 1 : state +; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: endbranch64 - FLUSH_JOB_ZUC_EIA3 256 + cmp arg2, 8 + je flush_tag_8B + jb flush_tag_4B + + ; Fall-through for 16-byte tag +flush_tag_16B: + FLUSH_JOB_ZUC_EIA3 256, 16 + ret + +flush_tag_8B: + FLUSH_JOB_ZUC_EIA3 256, 8 + ret + +flush_tag_4B: + FLUSH_JOB_ZUC_EIA3 256, 4 + ret mksection stack-noexec diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2/zuc_top_avx2.c index c346f898..ff66f494 100755 --- a/lib/avx2/zuc_top_avx2.c +++ b/lib/avx2/zuc_top_avx2.c @@ -859,7 +859,7 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], DECLARE_ALIGNED(ZucKey8_t keys, 64); const uint8_t *pIn8[NUM_AVX2_BUFS] = {NULL}; uint32_t numKeyStr = 0; - uint8_t T[NUM_AVX2_BUFS*4]; + DECLARE_ALIGNED(uint8_t T[NUM_AVX2_BUFS*16], 32); const uint32_t keyStreamLengthInBits = KEYSTR_ROUND_LEN * 8; DECLARE_ALIGNED(uint32_t *pKeyStrArr[NUM_AVX2_BUFS], 32) = {NULL}; unsigned int allCommonBits; @@ -873,7 +873,6 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], keys.pKeys[i] = pKey[i]; } - /* TODO: Handle 8 and 16-byte digest cases */ asm_Zuc256Initialization_8_avx2(&keys, ivs, &state, T, tag_size); /* Generate 32 bytes at a time */ @@ -886,27 +885,35 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], while (remainCommonBits >= keyStreamLengthInBits) { remainCommonBits -= keyStreamLengthInBits; numKeyStr++; - /* Generate the next key stream 4 bytes or 32 bytes */ - if (!remainCommonBits && allCommonBits) - asm_ZucGenKeystream4B_8_avx2(&state, - (uint32_t **)pKeyStrArr); - else + /* Generate the next key stream 4/8/16 bytes or 32 bytes */ + if (!remainCommonBits && allCommonBits) { + if (tag_size == 4) + asm_ZucGenKeystream4B_8_avx2(&state, + pKeyStrArr); + else if (tag_size == 8) + asm_ZucGenKeystream8B_8_avx2(&state, + pKeyStrArr); + else + asm_ZucGenKeystream16B_8_avx2(&state, + pKeyStrArr); + } else asm_ZucGenKeystream32B_8_avx2(&state, (uint32_t **)pKeyStrArr); for (i = 0; i < NUM_AVX2_BUFS; i++) { - uint32_t *tag = (uint32_t *) &T[i*4]; + void *tag = (void *) &T[i*tag_size]; if (job_in_lane[i] == NULL) continue; - asm_Eia3Round32B_avx(tag, &keyStr[i][0], pIn8[i], 4); + asm_Eia3Round32B_avx(tag, &keyStr[i][0], pIn8[i], + tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } } /* Process each packet separately for the remaining bits */ for (i = 0; i < NUM_AVX2_BUFS; i++) { - uint32_t *tag = (uint32_t *) &T[i*4]; + void *tag = (void *) &T[i*tag_size]; if (job_in_lane[i] == NULL) continue; @@ -915,10 +922,15 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], numKeyStr*keyStreamLengthInBits; uint32_t *keyStr32 = (uint32_t *) keyStr[i]; - /* If remaining bits are more than 160 bytes, we need to - * generate at least 4B more of keystream, so we need to copy - * the zuc state to single packet state first */ - if (remainBits > (5*32)) { + const uint32_t N = remainBits + ((uint32_t) tag_size << 3); + uint32_t L = ((N + 31) / ZUC_WORD_BITS); + + /* 8 KS words are generated already */ + L = (L > 8) ? (L - 8) : 0; + + /* Copy the ZUC state to single packet state, + * if more KS is needed */ + if (L > 0) { singlePktState.lfsrState[0] = state.lfsrState[0][i]; singlePktState.lfsrState[1] = state.lfsrState[1][i]; singlePktState.lfsrState[2] = state.lfsrState[2][i]; @@ -943,26 +955,25 @@ void zuc256_eia3_8_buffer_job_avx2(const void * const pKey[NUM_AVX2_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - /* Generate the next key stream 4 bytes or 32 bytes */ - if (!remainBits) - asm_ZucGenKeystream_avx(&keyStr32[8], - &singlePktState, 1); - else + /* Generate the next key stream (32 bytes max) */ + if (L > 7) { asm_ZucGenKeystream32B_avx(&keyStr32[8], &singlePktState); + L -= 8; + } else { + asm_ZucGenKeystream_avx(&keyStr32[8], + &singlePktState, L); + L = 0; + } asm_Eia3Round32B_avx(tag, &keyStr32[0], pIn8[i], tag_size); pIn8[i] = &pIn8[i][KEYSTR_ROUND_LEN]; } - /* - * If remaining bits has more than 5 ZUC WORDS (double words), - * keystream needs to have another ZUC WORD (4B) - */ - - if (remainBits > (5 * 32)) + /* Generate final keystream if needed */ + if (L > 0) asm_ZucGenKeystream_avx(&keyStr32[8], - &singlePktState, 1); + &singlePktState, L); asm_Eia3Remainder_avx(tag, keyStr32, pIn8[i], remainBits, 256, tag_size); diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2/zuc_x8_avx2.asm index 5f8133ad..20877dd0 100755 --- a/lib/avx2/zuc_x8_avx2.asm +++ b/lib/avx2/zuc_x8_avx2.asm @@ -1149,6 +1149,24 @@ asm_ZucGenKeystream32B_8_avx2: vzeroupper ret +;; +;; void asm_ZucGenKeystream16B_8_avx2(state8_t *pSta, u32* pKeyStr[8]) +;; +;; WIN64 +;; RCX - pSta +;; RDX - pKeyStr +;; +;; LIN64 +;; RDI - pSta +;; RSI - pKeyStr +;; +MKGLOBAL(asm_ZucGenKeystream16B_8_avx2,function,internal) +asm_ZucGenKeystream16B_8_avx2: + endbranch64 + KEYGEN_8_AVX2 4 + vzeroupper + ret + ;; ;; void asm_ZucGenKeystream8B_8_avx2(state8_t *pSta, u32* pKeyStr[8]) ;; diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 659d3696..548948e8 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -764,6 +764,29 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream4B_4_gfni_sse(ZucState4_t *pState, IMB_DLL_LOCAL void asm_ZucGenKeystream4B_4_avx(ZucState4_t *pState, uint32_t *pKeyStr[4]); +/** + ****************************************************************************** + * + * @description + * Definition of the external function that implements the working + * stage of the ZUC algorithm. The function will generate 16 bytes of + * keystream for eight packets in parallel. + * + * @param[in] pState Pointer to a ZUC state structure of type + * @ref ZucState8_t + * + * @param[in,out] pKeyStr Array of pointers to 8 input buffers that + * will contain the generated keystream for + * these 8 packets. + * + * @pre + * A successful call to @ref asm_ZucInitialization_8 to initialize the ZUC + * state. + * + *****************************************************************************/ +IMB_DLL_LOCAL void asm_ZucGenKeystream16B_8_avx2(ZucState8_t *pState, + uint32_t *pKeyStr[8]); + /** ****************************************************************************** * -- GitLab From 30cb6df2b04d69a224a5fcd3bb9f0c0c1a01fc91 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 6 Apr 2022 17:09:18 +0000 Subject: [PATCH 054/369] lib: [SNOW3G] fix 8-buffer keystream generation When generating keystream for 8 different buffers, using AVX2 or higher architecture, the keystream generated for buffer 1 is the output to be XOR'd to plaintext of buffer 0 and vice versa, on encryption. Same for buffers 2&3, 4&5 and 6&7. Fixes #104. --- ReleaseNotes.txt | 3 + lib/include/snow3g_common.h | 138 ++++++++++++++++++------------------ 2 files changed, 72 insertions(+), 69 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 804fbd2e..6abeee3f 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -20,6 +20,9 @@ Test Applications Performance Application - GHASH support added (through JOB API) +Fixes +- Fixed incorrect 8-buffer SNOW3G keystream generation + v1.2 February 2022 ======================================================================== diff --git a/lib/include/snow3g_common.h b/lib/include/snow3g_common.h index 7f569dec..11a30359 100644 --- a/lib/include/snow3g_common.h +++ b/lib/include/snow3g_common.h @@ -1689,6 +1689,63 @@ static inline __m256i snow3g_keystream_8_4(snow3gKeyState8_t *pCtx) return keyStream; } +/* + * @brief 8x8 uint32_t matrix tranpose. + * + * @param[in/clobbered] in Array of rows to transpose + * @param[out] out Array of transposed columns + */ +static inline void +transpose8xu32_avx2(__m256i in[8], __m256i out[8]) +{ + __m256i tmp[2]; + + tmp[0] = (__m256i) _mm256_shuffle_ps((__m256)in[0], (__m256)in[1], + 0x44); + in[0] = (__m256i) _mm256_shuffle_ps((__m256)in[0], (__m256)in[1], + 0xEE); + tmp[1] = (__m256i) _mm256_shuffle_ps((__m256)in[2], (__m256)in[3], + 0x44); + in[2] = (__m256i) _mm256_shuffle_ps((__m256)in[2], (__m256)in[3], + 0xEE); + + in[3] = (__m256i) _mm256_shuffle_ps((__m256)tmp[0],(__m256) tmp[1], + 0xDD); + in[1] = (__m256i) _mm256_shuffle_ps((__m256)in[0], (__m256) in[2], + 0x88); + in[0] = (__m256i) _mm256_shuffle_ps((__m256)in[0], (__m256) in[2], + 0xDD); + tmp[0] = (__m256i) _mm256_shuffle_ps((__m256)tmp[0],(__m256) tmp[1], + 0x88); + + in[2] = (__m256i) _mm256_shuffle_ps((__m256)in[4], (__m256)in[5], + 0x44); + in[4] = (__m256i) _mm256_shuffle_ps((__m256)in[4], (__m256)in[5], + 0xEE); + tmp[1] = (__m256i) _mm256_shuffle_ps((__m256)in[6], (__m256)in[7], + 0x44); + in[6] = (__m256i) _mm256_shuffle_ps((__m256)in[6], (__m256)in[7], + 0xEE); + + in[7] = (__m256i) _mm256_shuffle_ps((__m256)in[2],(__m256) tmp[1], + 0xDD); + in[5] = (__m256i) _mm256_shuffle_ps((__m256)in[4], (__m256) in[6], + 0x88); + in[4] = (__m256i) _mm256_shuffle_ps((__m256)in[4], (__m256) in[6], + 0xDD); + tmp[1] = (__m256i) _mm256_shuffle_ps((__m256)in[2],(__m256) tmp[1], + 0x88); + + out[6] = _mm256_permute2f128_si256(in[5], in[1], 0x13); + out[2] = _mm256_permute2f128_si256(in[5], in[1], 0x02); + out[5] = _mm256_permute2f128_si256(in[7], in[3], 0x13); + out[1] = _mm256_permute2f128_si256(in[7], in[3], 0x02); + out[7] = _mm256_permute2f128_si256(in[4], in[0], 0x13); + out[3] = _mm256_permute2f128_si256(in[4], in[0], 0x02); + out[4] = _mm256_permute2f128_si256(tmp[1], tmp[0], 0x13); + out[0] = _mm256_permute2f128_si256(tmp[1], tmp[0], 0x02); +} + /** * @brief Generates 32 bytes of key stream 8 buffers at a time * @@ -1698,81 +1755,24 @@ static inline __m256i snow3g_keystream_8_4(snow3gKeyState8_t *pCtx) static inline void snow3g_keystream_8_32(snow3gKeyState8_t *pCtx, __m256i *pKeyStream) { + __m256i in[8]; + unsigned int i; - __m256i temp[8]; + /** Byte reversal on each KS */ + static const __m256i mask = { + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL, + 0x0405060700010203ULL, 0x0c0d0e0f08090a0bULL + }; /** produces the next 4 bytes for each buffer */ - int i; + for (i = 0; i < 8; i++) + in[i] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask); - /** Byte reversal on each KS */ - static const __m256i mask1 = { - 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL, - 0x0001020304050607ULL, 0x08090a0b0c0d0e0fULL - }; - /** Reversal, shifted 4 bytes right */ - static const __m256i mask2 = { - 0x0405060708090a0bULL, 0x0c0d0e0f00010203ULL, - 0x0405060708090a0bULL, 0x0c0d0e0f00010203ULL - }; - /** Reversal, shifted 8 bytes right */ - static const __m256i mask3 = { - 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL, - 0x08090a0b0c0d0e0fULL, 0x0001020304050607ULL - }; - /** Reversal, shifted 12 bytes right */ - static const __m256i mask4 = { - 0x0c0d0e0f00010203ULL, 0x0405060708090a0bULL, - 0x0c0d0e0f00010203ULL, 0x0405060708090a0bULL - }; + /* Transposes the dwords of KS for all buffers into + * 32 consecutive KS bytes for each buffer */ + transpose8xu32_avx2(in, pKeyStream); - temp[0] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask1); - temp[1] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask2); - temp[2] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask3); - temp[3] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask4); - temp[4] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask1); - temp[5] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask2); - temp[6] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask3); - temp[7] = _mm256_shuffle_epi8(snow3g_keystream_8_4(pCtx), mask4); - - __m256i blended[8]; - /* blends KS together: 128bit slice consists - of 4 32-bit words for one packet */ - blended[0] = _mm256_blend_epi32(temp[0], temp[1], 0xaa); - blended[1] = _mm256_blend_epi32(temp[0], temp[1], 0x55); - blended[2] = _mm256_blend_epi32(temp[2], temp[3], 0xaa); - blended[3] = _mm256_blend_epi32(temp[2], temp[3], 0x55); - blended[4] = _mm256_blend_epi32(temp[4], temp[5], 0xaa); - blended[5] = _mm256_blend_epi32(temp[4], temp[5], 0x55); - blended[6] = _mm256_blend_epi32(temp[6], temp[7], 0xaa); - blended[7] = _mm256_blend_epi32(temp[6], temp[7], 0x55); - - temp[0] = _mm256_blend_epi32(blended[0], blended[2], 0xcc); - temp[1] = _mm256_blend_epi32(blended[1], blended[3], 0x99); - temp[2] = _mm256_blend_epi32(blended[0], blended[2], 0x33); - temp[3] = _mm256_blend_epi32(blended[1], blended[3], 0x66); - temp[4] = _mm256_blend_epi32(blended[4], blended[6], 0xcc); - temp[5] = _mm256_blend_epi32(blended[5], blended[7], 0x99); - temp[6] = _mm256_blend_epi32(blended[4], blended[6], 0x33); - temp[7] = _mm256_blend_epi32(blended[5], blended[7], 0x66); - - /** sorts 32 bit words back into order */ - blended[0] = temp[0]; - blended[1] = _mm256_shuffle_epi32(temp[1], 0x39); - blended[2] = _mm256_shuffle_epi32(temp[2], 0x4e); - blended[3] = _mm256_shuffle_epi32(temp[3], 0x93); - blended[4] = temp[4]; - blended[5] = _mm256_shuffle_epi32(temp[5], 0x39); - blended[6] = _mm256_shuffle_epi32(temp[6], 0x4e); - blended[7] = _mm256_shuffle_epi32(temp[7], 0x93); - for (i = 0; i < 4; i++) { - pKeyStream[i] = - _mm256_permute2x128_si256(blended[i], - blended[i + 4], 0x20); - pKeyStream[i + 4] = - _mm256_permute2x128_si256(blended[i], - blended[i + 4], 0x31); - } } #endif /* AVX2 */ -- GitLab From 1c6a6c93915611c2f94e16e1431403efc2796211 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 12 Apr 2022 14:45:43 +0000 Subject: [PATCH 055/369] test: [ipsec_xvalid_app] Add option to set safe check retries - 2 retries done by default --- test/ipsec_xvalid.c | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 7d194178..18f6a2cc 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -84,6 +84,10 @@ #define SEED 0xdeadcafe #define STACK_DEPTH 8192 +/* Max safe check retries to eliminate false positives */ +#define MAX_SAFE_RETRIES 100 +#define DEFAULT_SAFE_RETRIES 2 + static int pattern_auth_key; static int pattern_cipher_key; static int pattern_plain_text; @@ -653,6 +657,7 @@ const uint8_t key_sizes[][3] = { uint8_t custom_test = 0; uint8_t verbose = 0; +uint32_t safe_retries = DEFAULT_SAFE_RETRIES; enum range { RANGE_MIN = 0, @@ -2291,14 +2296,22 @@ process_variant(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, dec_arch, params, variant_data, 1, 0, 1); if (result < 0) { - printf("=== Issue found. " - "Checking again...\n"); - generate_patterns(); - result = do_test(enc_mgr, enc_arch, - dec_mgr, dec_arch, - params, variant_data, - 1, 0, 1); - + uint32_t j; + + for (j = 0; j < safe_retries; j++) { + printf("=== Issue found. " + "Checking again...\n"); + generate_patterns(); + result = do_test(enc_mgr, + enc_arch, + dec_mgr, + dec_arch, + params, + variant_data, + 1, 0, 1); + if (result == 0) + break; + } if (result < 0) { if (verbose) printf("FAIL\n"); @@ -2921,6 +2934,15 @@ int main(int argc, char *argv[]) } } else if (strcmp(argv[i], "--safe-check") == 0) { safe_check = 1; + } else if (strcmp(argv[i], "--safe-retries") == 0) { + i = get_next_num_arg((const char * const *)argv, i, + argc, &safe_retries, + sizeof(safe_retries)); + if (safe_retries > MAX_SAFE_RETRIES) { + fprintf(stderr, "Number of retries cannot be " + "higher than %d\n", MAX_SAFE_RETRIES); + return EXIT_FAILURE; + } } else if (strcmp(argv[i], "--imix") == 0) { imix_enabled = 1; } else { -- GitLab From 3722e73885ffaea9801c99783aaac9542fdccfb0 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 12 Apr 2022 15:47:39 +0000 Subject: [PATCH 056/369] lib: Extend auth tag len arrays for GHASH in is_job_invalid() --- lib/include/mb_mgr_code.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index cd584fe7..f198a343 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1336,6 +1336,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ 4, /* IMB_AUTH_CRC7_FP_HEADER */ 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; const uint64_t auth_tag_len_ipsec[] = { 0, /* INVALID selection */ @@ -1384,6 +1385,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ 4, /* IMB_AUTH_CRC7_FP_HEADER */ 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ }; /* Maximum length of buffer in PON is 2^14 + 8, since maximum -- GitLab From 3bc5a592259adbff9349926d2fd3aa1a5eb7fa12 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 15 Apr 2022 08:37:25 +0000 Subject: [PATCH 057/369] avx2: [AES-ECB] fix macro call --- lib/avx2/aes128_ecb_vaes_avx2.asm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/avx2/aes128_ecb_vaes_avx2.asm b/lib/avx2/aes128_ecb_vaes_avx2.asm index 43726df2..09a38fff 100644 --- a/lib/avx2/aes128_ecb_vaes_avx2.asm +++ b/lib/avx2/aes128_ecb_vaes_avx2.asm @@ -89,9 +89,9 @@ %macro AES_ECB 1 %define %%DIR %1 ; [in] Direction (ENC/DIR) %ifidn %%DIR, ENC -%define AES YMM_AESENC_ROUND_BLOCKS_0_16 +%define AES YMM_AESENC_ROUND_BLOCKS_AVX2_0_16 %else ; DIR = DEC -%define AES YMM_AESDEC_ROUND_BLOCKS_0_16 +%define AES YMM_AESDEC_ROUND_BLOCKS_AVX2_0_16 %endif or LEN, LEN @@ -157,7 +157,7 @@ %endrep ; store initial blocks - YMM_STORE_BLOCKS_0_16 num_blocks, OUT, 0, YDATA0, YDATA1,\ + YMM_STORE_BLOCKS_AVX2_0_16 num_blocks, OUT, 0, YDATA0, YDATA1,\ YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 add IDX, num_blocks*16 @@ -188,7 +188,7 @@ align 16 %endrep ; store 16 blocks - YMM_STORE_BLOCKS_0_16 16, {OUT + IDX}, 0, YDATA0, YDATA1,\ + YMM_STORE_BLOCKS_AVX2_0_16 16, {OUT + IDX}, 0, YDATA0, YDATA1,\ YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 add IDX, 16*16 -- GitLab From 0385e6bb6e41c6e2154b1e669a32831ccc3cfa7d Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 11 Mar 2022 09:53:38 +0000 Subject: [PATCH 058/369] CI: add freebsd to github CI --- .github/workflows/freebsd_make.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/freebsd_make.yml diff --git a/.github/workflows/freebsd_make.yml b/.github/workflows/freebsd_make.yml new file mode 100644 index 00000000..d821219a --- /dev/null +++ b/.github/workflows/freebsd_make.yml @@ -0,0 +1,27 @@ +name: FreeBSD make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + testfreebsd: + runs-on: macos-10.15 + name: FreeBSD run + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: Test in FreeBSD + id: tests + uses: vmactions/freebsd-vm@v0.1.5 + with: + usesh: true + mem: 8192 + prepare: pkg install -y curl nasm llvm gmake + run: | + freebsd-version + gmake CC=clang -j 4 -- GitLab From 66baa6d3230611a59f6b7d911742024790ca06b3 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 21 Apr 2022 10:28:40 +0000 Subject: [PATCH 059/369] build: clang doesn't support -fno-strict-overflow argument --- lib/Makefile | 7 ++++++- perf/Makefile | 7 ++++++- test/Makefile | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index eafa09d3..ad98bcc4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -96,7 +96,12 @@ CFLAGS := -DNO_COMPAT_IMB_API_053 $(EXTRA_CFLAGS) $(INCLUDES) \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -fno-strict-overflow -fno-delete-null-pointer-checks -fwrapv + -fno-delete-null-pointer-checks -fwrapv + +# -fno-strict-overflow is not supported by clang +ifneq ($(CC),clang) +CFLAGS += -fno-strict-overflow +endif ifeq ($(MINGW),0) CFLAGS += -DLINUX diff --git a/perf/Makefile b/perf/Makefile index 4d00aff2..97eb29e2 100644 --- a/perf/Makefile +++ b/perf/Makefile @@ -36,7 +36,12 @@ CFLAGS = -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 $(INCLUDES) \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -pthread -fno-strict-overflow -fno-delete-null-pointer-checks -fwrapv + -pthread -fno-delete-null-pointer-checks -fwrapv + +# -fno-strict-overflow is not supported by clang +ifneq ($(CC),clang) +CFLAGS += -fno-strict-overflow +endif ifeq ($(MINGW),0) CFLAGS += -DLINUX diff --git a/test/Makefile b/test/Makefile index fad4571a..b03700a5 100644 --- a/test/Makefile +++ b/test/Makefile @@ -44,7 +44,12 @@ CFLAGS = -MMD -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 \ -Wformat -Wformat-security \ -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ - -fno-strict-overflow -fno-delete-null-pointer-checks -fwrapv + -fno-delete-null-pointer-checks -fwrapv + +# -fno-strict-overflow is not supported by clang +ifneq ($(CC),clang) +CFLAGS += -fno-strict-overflow +endif # if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported # "-fcf-protection" needs to be checked separately -- GitLab From bb124d699ff76e799338687a36e3c97b3fe78ff2 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Thu, 21 Apr 2022 15:33:33 +0000 Subject: [PATCH 060/369] CI: add freebsd gcc build to github CI --- .github/workflows/freebsd_gcc_make.yml | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/freebsd_gcc_make.yml diff --git a/.github/workflows/freebsd_gcc_make.yml b/.github/workflows/freebsd_gcc_make.yml new file mode 100644 index 00000000..106c7d27 --- /dev/null +++ b/.github/workflows/freebsd_gcc_make.yml @@ -0,0 +1,27 @@ +name: FreeBSD GCC make + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + testfreebsd: + runs-on: macos-10.15 + name: FreeBSD GCC run + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: intel/intel-ipsec-mb + - name: Test FreeBSD GCC build + id: tests + uses: vmactions/freebsd-vm@v0.1.5 + with: + usesh: true + mem: 8192 + prepare: pkg install -y curl nasm gmake gcc + run: | + freebsd-version + gmake CC=gcc -j 4 -- GitLab From fa05c5403f24c9ee20bd8e133cb68e6d5dfe74a2 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 4 Mar 2022 11:14:52 +0000 Subject: [PATCH 061/369] lib: Initial burst API support --- lib/avx/mb_mgr_avx.c | 4 ++ lib/avx2/mb_mgr_avx2.c | 4 ++ lib/avx512/mb_mgr_avx512.c | 4 ++ lib/include/mb_mgr_code.h | 101 +++++++++++++++++++++++++++++ lib/include/noaesni.h | 8 +++ lib/intel-ipsec-mb.h | 30 +++++++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 4 ++ lib/sse/mb_mgr_sse.c | 4 ++ 8 files changed, 159 insertions(+) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 28cbaeb4..d2c74be7 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -258,6 +258,8 @@ void aes_cmac_256_subkey_gen_avx(const void *key_exp, #define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx #define GET_NEXT_JOB get_next_job_avx #define GET_COMPLETED_JOB get_completed_job_avx +#define SUBMIT_BURST submit_burst_avx +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx /* ====================================================================== */ @@ -903,6 +905,8 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) /* set AVX handlers */ state->get_next_job = get_next_job_avx; state->submit_job = submit_job_avx; + state->submit_burst = submit_burst_avx; + state->submit_burst_nocheck= submit_burst_nocheck_avx; state->submit_job_nocheck = submit_job_nocheck_avx; state->get_completed_job = get_completed_job_avx; state->flush_job = flush_job_avx; diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 0e446a5b..4ecaf022 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -263,6 +263,8 @@ IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); #define QUEUE_SIZE queue_size_avx2 #define GET_NEXT_JOB get_next_job_avx2 #define GET_COMPLETED_JOB get_completed_job_avx2 +#define SUBMIT_BURST submit_burst_avx2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2 /* ====================================================================== */ @@ -890,6 +892,8 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) /* set handlers */ state->get_next_job = get_next_job_avx2; state->submit_job = submit_job_avx2; + state->submit_burst = submit_burst_avx2; + state->submit_burst_nocheck= submit_burst_nocheck_avx2; state->submit_job_nocheck = submit_job_nocheck_avx2; state->get_completed_job = get_completed_job_avx2; state->flush_job = flush_job_avx2; diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 794774b4..044c9d66 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -478,6 +478,8 @@ SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, #define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 #define GET_NEXT_JOB get_next_job_avx512 #define GET_COMPLETED_JOB get_completed_job_avx512 +#define SUBMIT_BURST submit_burst_avx512 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512 /* ====================================================================== */ @@ -1904,6 +1906,8 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) /* set handlers */ state->get_next_job = get_next_job_avx512; state->submit_job = submit_job_avx512; + state->submit_burst = submit_burst_avx512; + state->submit_burst_nocheck= submit_burst_nocheck_avx512; state->submit_job_nocheck = submit_job_nocheck_avx512; state->get_completed_job = get_completed_job_avx512; state->flush_job = flush_job_avx512; diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index f198a343..05ccc96e 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3039,4 +3039,105 @@ GET_NEXT_JOB(IMB_MGR *state) return JOBS(state, state->next_job); } +/** + * Submit multiple jobs to be processed synchronously + * + * @param [in] state pointer to multi-buffer manager + * @param [in] jobs pointer to array of jobs + * @param [in] n_jobs number of jobs to process + * + * @return number of completed jobs + */ +__forceinline +uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, const int run_check) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job->status = IMB_STATUS_BEING_PROCESSED; + + if (job->cipher_mode == IMB_CIPHER_GCM) { + if (job->cipher_direction == IMB_DIR_ENCRYPT) + SUBMIT_JOB_AES_GCM_ENC(state, job); + else + SUBMIT_JOB_AES_GCM_DEC(state, job); + completed_jobs++; + } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { + SUBMIT_JOB_CHACHA20_POLY1305(state, job); + completed_jobs++; + } else { + if (submit_new_job(state, job) != NULL) + completed_jobs++; + } + } + + /* return if all jobs complete */ + if (completed_jobs == n_jobs) + return completed_jobs; + + /* otherwise complete remaining jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + if (job->status < IMB_STATUS_COMPLETED) { + /* force job to completion */ + complete_job(state, job); + completed_jobs++; + } + } + + return completed_jobs; +} + +uint32_t +SUBMIT_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } +#endif + + return submit_burst_and_check(state, jobs, n_jobs, 1); +} + +uint32_t +SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } +#endif + + return submit_burst_and_check(state, jobs, n_jobs, 0); +} + + + #endif /* MB_MGR_CODE_H */ diff --git a/lib/include/noaesni.h b/lib/include/noaesni.h index 729348cf..71a6f773 100644 --- a/lib/include/noaesni.h +++ b/lib/include/noaesni.h @@ -38,6 +38,14 @@ IMB_DLL_EXPORT uint32_t queue_size_sse_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_no_aesni(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t +submit_burst_sse_no_aesni(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs); + IMB_DLL_EXPORT void aes_keyexp_128_sse_no_aesni(const void *key, void *enc_exp_keys, void *dec_exp_keys); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index c73e1233..beb06dda 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -762,6 +762,10 @@ typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *); typedef uint32_t (*queue_size_t)(struct IMB_MGR *); + +typedef uint32_t (*submit_burst_t)(struct IMB_MGR *, + struct IMB_JOB *, uint32_t); + typedef void (*keyexp_t)(const void *, void *, void *); typedef void (*cmac_subkey_gen_t)(const void *, void *, void *); typedef void (*hash_one_block_t)(const void *, void *); @@ -1143,6 +1147,9 @@ typedef struct IMB_MGR { chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update; chacha_poly_finalize_t chacha20_poly1305_finalize; + submit_burst_t submit_burst; + submit_burst_t submit_burst_nocheck; + /* in-order scheduler fields */ int earliest_job; /**< byte offset, -1 if none */ int next_job; /**< byte offset */ @@ -1327,6 +1334,25 @@ IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t +submit_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); + +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs); + /** * @brief Automatically initialize most performant * Multi-buffer manager based on CPU features @@ -1365,6 +1391,10 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); #define IMB_GET_COMPLETED_JOB(_mgr) ((_mgr)->get_completed_job((_mgr))) #define IMB_FLUSH_JOB(_mgr) ((_mgr)->flush_job((_mgr))) #define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr))) +#define IMB_SUBMIT_BURST(_mgr, _jobs, _n_jobs) \ + ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs))) +#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs) \ + ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs))) /* Key expansion and generation API's */ #define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \ diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index b73e1856..53e53b18 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -239,6 +239,8 @@ IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_no_aesni #define GET_NEXT_JOB get_next_job_sse_no_aesni #define GET_COMPLETED_JOB get_completed_job_sse_no_aesni +#define SUBMIT_BURST submit_burst_sse_no_aesni +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_no_aesni #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni @@ -898,6 +900,8 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) /* set SSE NO AESNI handlers */ state->get_next_job = get_next_job_sse_no_aesni; state->submit_job = submit_job_sse_no_aesni; + state->submit_burst = submit_burst_sse_no_aesni; + state->submit_burst_nocheck= submit_burst_nocheck_sse_no_aesni; state->submit_job_nocheck = submit_job_nocheck_sse_no_aesni; state->get_completed_job = get_completed_job_sse_no_aesni; state->flush_job = flush_job_sse_no_aesni; diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index ab2113a8..a7529cda 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -306,6 +306,8 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse #define GET_NEXT_JOB get_next_job_sse #define GET_COMPLETED_JOB get_completed_job_sse +#define SUBMIT_BURST submit_burst_sse +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse @@ -1220,6 +1222,8 @@ init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) /* set SSE handlers */ state->get_next_job = get_next_job_sse; state->submit_job = submit_job_sse; + state->submit_burst = submit_burst_sse; + state->submit_burst_nocheck= submit_burst_nocheck_sse; state->submit_job_nocheck = submit_job_nocheck_sse; state->get_completed_job = get_completed_job_sse; state->flush_job = flush_job_sse; -- GitLab From fc6b77f1511ec6df779c33bd0d18045057ecfe42 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 4 Mar 2022 11:15:32 +0000 Subject: [PATCH 062/369] test: Extend AES and ChachaPoly tests for burst API --- test/aes_test.c | 155 ++++++++++++++++++++++++++++++++++ test/chacha20_poly1305_test.c | 96 +++++++++++++++++++++ 2 files changed, 251 insertions(+) diff --git a/test/aes_test.c b/test/aes_test.c index ebcab393..ff8fab15 100644 --- a/test/aes_test.c +++ b/test/aes_test.c @@ -1972,6 +1972,108 @@ end_alloc: return ret; } +static int +test_aes_many_burst(struct IMB_MGR *mb_mgr, + void *enc_keys, + void *dec_keys, + const void *iv, + const uint8_t *in_text, + const uint8_t *out_text, + const unsigned text_len, + const int dir, + const int order, + const IMB_CIPHER_MODE cipher, + const int in_place, + const int key_len, + const int num_jobs) +{ + struct IMB_JOB *job, jobs[24]; + uint8_t padding[16]; + uint8_t **targets = malloc(num_jobs * sizeof(void *)); + int i, jobs_rx = 0, ret = -1; + + if (targets == NULL) + goto end_alloc; + + memset(targets, 0, num_jobs * sizeof(void *)); + memset(padding, -1, sizeof(padding)); + + for (i = 0; i < num_jobs; i++) { + targets[i] = malloc(text_len + (sizeof(padding) * 2)); + if (targets[i] == NULL) + goto end_alloc; + memset(targets[i], -1, text_len + (sizeof(padding) * 2)); + if (in_place) { + /* copy input text to the allocated buffer */ + memcpy(targets[i] + sizeof(padding), in_text, text_len); + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->cipher_direction = dir; + job->chain_order = order; + if (!in_place) { + job->dst = targets[i] + sizeof(padding); + job->src = in_text; + } else { + job->dst = targets[i] + sizeof(padding); + job->src = targets[i] + sizeof(padding); + } + job->cipher_mode = cipher; + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + job->key_len_in_bytes = key_len; + + job->iv = iv; + job->iv_len_in_bytes = 16; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = text_len; + job->user_data = targets[i]; + job->user_data2 = (void *)((uint64_t)i); + + job->hash_alg = IMB_AUTH_NULL; + } + + int completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (!aes_job_ok(job, out_text, job->user_data, padding, + sizeof(padding), text_len)) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + + end_alloc: + if (targets != NULL) { + for (i = 0; i < num_jobs; i++) + free(targets[i]); + free(targets); + } + + return ret; +} + static void test_aes_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, @@ -2027,6 +2129,19 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, + cipher, 0, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d encrypt burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + if (test_aes_many(mb_mgr, enc_keys, dec_keys, vec_tab[vect].IV, vec_tab[vect].C, vec_tab[vect].P, @@ -2040,6 +2155,19 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, + cipher, 0, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d decrypt burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + if (test_aes_many(mb_mgr, enc_keys, dec_keys, vec_tab[vect].IV, vec_tab[vect].P, vec_tab[vect].C, @@ -2053,6 +2181,19 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, + cipher, 1, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d encrypt burst in-place\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + if (test_aes_many(mb_mgr, enc_keys, dec_keys, vec_tab[vect].IV, vec_tab[vect].C, vec_tab[vect].P, @@ -2065,6 +2206,20 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, } else { test_suite_update(ctx, 1, 0); } + + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, + cipher, 1, + vec_tab[vect].Klen, num_jobs)) { + printf("error #%d decrypt burst in-place\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } printf("\n"); } diff --git a/test/chacha20_poly1305_test.c b/test/chacha20_poly1305_test.c index d7e4b52b..a584e4da 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/chacha20_poly1305_test.c @@ -437,6 +437,102 @@ test_aead(struct IMB_MGR *mb_mgr, goto end; } + /* + * ******************************************* + * BURST API TEST + * ******************************************* + */ + + /* create job array */ + IMB_JOB jobs[32] = {0}; + + jobs_rx = 0; + + /* reset buffers */ + for (i = 0; i < num_jobs; i++) { + memset(auths[i], -1, 16 + (sizeof(padding) * 2)); + memset(targets[i], -1, vec->msg_len + (sizeof(padding) * 2)); + + if (in_place) { + if (dir == IMB_DIR_ENCRYPT) + memcpy(targets[i] + sizeof(padding), + vec->plain, vec->msg_len); + else + memcpy(targets[i] + sizeof(padding), + vec->cipher, vec->msg_len); + } + } + + /** + * Set all job params before submitting burst + */ + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->cipher_direction = dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; + job->hash_alg = IMB_AUTH_CHACHA20_POLY1305; + job->enc_keys = vec->key; + job->dec_keys = vec->key; + job->key_len_in_bytes = 32; + + job->u.CHACHA20_POLY1305.aad = vec->aad; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = vec->aad_len; + + if (in_place) + job->src = targets[i] + sizeof(padding); + else + if (dir == IMB_DIR_ENCRYPT) + job->src = vec->plain; + else + job->src = vec->cipher; + job->dst = targets[i] + sizeof(padding); + + job->iv = vec->iv; + job->iv_len_in_bytes = 12; + job->msg_len_to_cipher_in_bytes = vec->msg_len; + job->cipher_start_src_offset_in_bytes = 0; + + job->msg_len_to_hash_in_bytes = vec->msg_len; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = 16; + + job->user_data = auths[i]; + } + + int completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (!aead_job_ok(mb_mgr, vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs after burst, " + "received %d\n", num_jobs, jobs_rx); + goto end; + } + + /* + * ******************************************* + * END BURST API TEST + * ******************************************* + */ ret = 0; -- GitLab From 5d0da629a5e051033974e3fc73f26e50c554c4d6 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 4 Mar 2022 11:16:08 +0000 Subject: [PATCH 063/369] perf: Add burst API support Changes: * --burst-api option added to force burst API used * --burst-size option added to set number of jobs to submit when using burst API * Output information about what API is being used --- perf/ipsec_perf.c | 130 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 104 insertions(+), 26 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 4f1a41e1..5b59f36f 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -107,6 +107,9 @@ typedef cpuset_t cpu_set_t; #define IA32_MSR_PERF_GLOBAL_CTR 0x38F #define IA32_MSR_CPU_UNHALTED_THREAD 0x30A +#define DEFAULT_BURST_SIZE 32 +#define MAX_BURST_SIZE 256 + enum arch_type_e { ARCH_SSE = 0, ARCH_AVX, @@ -909,6 +912,9 @@ static uint32_t pb_mod = 0; static int silent_progress_bar = 0; static int plot_output_option = 0; +static uint32_t burst_api = 0; /* burst API enable/disable flag */ +static uint32_t burst_size = 0; /* num jobs to pass to burst API */ + /* Return rdtsc to core cycle scale factor */ static double get_tsc_to_core_scale(const int turbo) { @@ -1879,44 +1885,84 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif time = __rdtscp(&aux); - for (i = 0; i < num_iter; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - *job = job_template; + if (burst_api) { + IMB_JOB jobs[MAX_BURST_SIZE]; + uint32_t num_jobs = num_iter; - set_job_fields(job, p_buffer, p_keys, i, index); + while (num_jobs) { + uint32_t n_jobs = + (num_jobs / burst_size) ? burst_size : num_jobs; - index = get_next_index(index); + /* set all job params */ + for (i = 0; i < n_jobs; i++) { + job = &jobs[i]; + *job = job_template; + + set_job_fields(job, p_buffer, p_keys, i, index); + + index = get_next_index(index); + } + /* submit burst */ +#ifdef DEBUG + const uint32_t completed_jobs = + IMB_SUBMIT_BURST(mb_mgr, jobs, n_jobs); + + if (completed_jobs != n_jobs) { + const int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", + err, imb_get_strerror(err)); + } + } +#else + IMB_SUBMIT_BURST_NOCHECK(mb_mgr, jobs, n_jobs); +#endif + num_jobs -= n_jobs; + } + } else { + for (i = 0; i < num_iter; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + *job = job_template; + + set_job_fields(job, p_buffer, p_keys, i, index); + + index = get_next_index(index); #ifdef DEBUG - job = IMB_SUBMIT_JOB(mb_mgr); + job = IMB_SUBMIT_JOB(mb_mgr); #else - job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); + job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); #endif - while (job) { + while (job) { #ifdef DEBUG - if (job->status != IMB_STATUS_COMPLETED) { - fprintf(stderr, "failed job, status:%d\n", - job->status); - return 1; - } + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, + "failed job, status:%d\n", + job->status); + return 1; + } #endif - job = IMB_GET_COMPLETED_JOB(mb_mgr); + job = IMB_GET_COMPLETED_JOB(mb_mgr); + } } - } - while ((job = IMB_FLUSH_JOB(mb_mgr))) { + while ((job = IMB_FLUSH_JOB(mb_mgr))) { #ifdef DEBUG - if (job->status != IMB_STATUS_COMPLETED) { - const int errc = imb_get_errno(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + const int errc = imb_get_errno(mb_mgr); - fprintf(stderr, - "failed job, status:%d, error code:%d, %s\n", - job->status, errc, imb_get_strerror(errc)); - return 1; - } + fprintf(stderr, + "failed job, status:%d, " + "error code:%d, %s\n", job->status, + errc, imb_get_strerror(errc)); + return 1; + } #else - (void)job; + (void)job; #endif - } + } + + } /* if burst api */ #ifndef _WIN32 if (use_unhalted_cycles) @@ -2618,7 +2664,9 @@ static void usage(void) " (Use when turbo enabled)\n" "--no-tsc-detect: don't check TSC to core scaling\n" "--tag-size: modify tag size\n" - "--plot: Adjust text output for direct use with plot output\n", + "--plot: Adjust text output for direct use with plot output\n" + "--burst-api: use burst API for perf tests\n" + "--burst-size: number of jobs to submit per burst\n", MAX_NUM_THREADS + 1); } @@ -3190,11 +3238,31 @@ int main(int argc, char *argv[]) } else if (strcmp(argv[i], "--tag-size") == 0) { i = get_next_num_arg((const char * const *)argv, i, argc, &tag_size, sizeof(tag_size)); + } else if (strcmp(argv[i], "--burst-api") == 0) { + burst_api = 1; + } else if (strcmp(argv[i], "--burst-size") == 0) { + i = get_next_num_arg((const char * const *)argv, i, + argc, &burst_size, + sizeof(burst_size)); + if (burst_size > (MAX_BURST_SIZE)) { + fprintf(stderr, "Burst size cannot be " + "more than %d\n", MAX_BURST_SIZE); + return EXIT_FAILURE; + } } else { usage(); return EXIT_FAILURE; } + if (burst_size != 0 && burst_api == 0) { + fprintf(stderr, "--burst-size can only be used with " + "--burst-api\n"); + return EXIT_FAILURE; + } + + if (burst_api != 0 && burst_size == 0) + burst_size = DEFAULT_BURST_SIZE; + if (aead_algo_set == 0 && cipher_algo_set == 0 && hash_algo_set == 0) { fprintf(stderr, "No cipher, hash or " @@ -3202,6 +3270,7 @@ int main(int argc, char *argv[]) usage(); return EXIT_FAILURE; } + if (aead_algo_set && (cipher_algo_set || hash_algo_set)) { fprintf(stderr, "AEAD algorithm cannot be used " "combined with another cipher/hash " @@ -3342,6 +3411,15 @@ int main(int argc, char *argv[]) "Library version: %s\n", sha_size_incr, IMB_VERSION_STR, imb_get_version_str()); + fprintf(stderr, "API type: %s", burst_api ? "burst" : "job"); + if (burst_api) + fprintf(stderr, " (burst size = %u)\n", burst_size); + else + fprintf(stderr, "\n"); + + fprintf(stderr, "GCM API type: %s\n", + use_gcm_job_api ? (burst_api ? "burst" : "job") : "direct"); + if (custom_job_params.cipher_mode == TEST_GCM) fprintf(stderr, "GCM AAD = %"PRIu64"\n", gcm_aad_size); -- GitLab From dbb47073f8c64ff4d4b09a638696c394ac37ea16 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Tue, 26 Apr 2022 14:14:18 +0000 Subject: [PATCH 064/369] [lib]: fix SSE-AVX transition --- lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm index c8aaa5bf..33ee6cbb 100644 --- a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -260,6 +260,7 @@ len_is_0: %endrep %endif + vzeroupper return: mov rbx, [rsp + _gpr_save + 8*0] -- GitLab From 2106fa2fd1fa92049deb6acdf4a05b6a2e816c2a Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 15 Apr 2022 11:11:20 +0000 Subject: [PATCH 065/369] avx512-vaes: [gcm] change of GHASH_16 macro arguments to be also able to read data and shuffle it --- lib/avx512/gcm_vaes_avx512.inc | 125 +++++++++------------------------ 1 file changed, 33 insertions(+), 92 deletions(-) diff --git a/lib/avx512/gcm_vaes_avx512.inc b/lib/avx512/gcm_vaes_avx512.inc index a0797369..a7812ba2 100644 --- a/lib/avx512/gcm_vaes_avx512.inc +++ b/lib/avx512/gcm_vaes_avx512.inc @@ -233,7 +233,7 @@ ;;; %%INPTR address is 64 byte aligned ;;; - there is an option to pass ready blocks through ZMM registers too. ;;; 4 extra parameters need to passed in such case and 21st argument can be empty -%macro GHASH_16 21-25 +%macro GHASH_16 21-22 %define %%TYPE %1 ; [in] ghash type: start (xor hash), mid, end (same as mid; no reduction), ; end_reduce (end with reduction), start_reduce %define %%GH %2 ; [in/out] ZMM ghash sum: high 128-bits @@ -256,10 +256,7 @@ %define %%ZTMP7 %19 ; [clobbered] temporary ZMM %define %%ZTMP8 %20 ; [clobbered] temporary ZMM %define %%ZTMP9 %21 ; [clobbered] temporary ZMM -%define %%DAT0 %22 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) -%define %%DAT1 %23 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) -%define %%DAT2 %24 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) -%define %%DAT3 %25 ; [in] ZMM with 4 blocks of input data (INPTR, INOFF, INDIS unused) +%define %%SHUFM %22 ; [in] ZMM with shuffle mask - provided only when input data needs shuffling %assign start_ghash 0 %assign do_reduction 0 @@ -278,14 +275,15 @@ %endif ;; ghash blocks 0-3 -%if %0 == 21 - vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS] +%if %0 == 22 + vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS] + vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else -%xdefine %%ZTMP9 %%DAT0 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS] %endif %if start_ghash != 0 - vpxorq %%ZTMP9, %%HASH + vpxorq %%ZTMP9, %%ZTMP9, %%HASH %endif vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS] vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 @@ -293,10 +291,11 @@ vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 ;; ghash blocks 4-7 -%if %0 == 21 - vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] +%if %0 == 22 + vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] + vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else -%xdefine %%ZTMP9 %%DAT1 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 64] %endif vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 64] vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 @@ -316,10 +315,11 @@ vpternlogq %%GM, %%ZTMP3, %%ZTMP7, 0x96 ; GM += T0M2 + T1M1 %endif ;; ghash blocks 8-11 -%if %0 == 21 - vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] +%if %0 == 22 + vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] + vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else -%xdefine %%ZTMP9 %%DAT2 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 128] %endif vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 128] vpclmulqdq %%ZTMP0, %%ZTMP9, %%ZTMP8, 0x11 ; T0H = a1*b1 @@ -327,10 +327,11 @@ vpclmulqdq %%ZTMP2, %%ZTMP9, %%ZTMP8, 0x01 ; T0M1 = a1*b0 vpclmulqdq %%ZTMP3, %%ZTMP9, %%ZTMP8, 0x10 ; T0M2 = a0*b1 ;; ghash blocks 12-15 -%if %0 == 21 - vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] +%if %0 == 22 + vmovdqu64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] + vpshufb %%ZTMP9, %%ZTMP9, %%SHUFM %else -%xdefine %%ZTMP9 %%DAT3 + vmovdqa64 %%ZTMP9, [%%INPTR + %%INOFF + %%INDIS + 192] %endif vmovdqu64 %%ZTMP8, [%%HKPTR + %%HKOFF + %%HKDIS + 192] vpclmulqdq %%ZTMP4, %%ZTMP9, %%ZTMP8, 0x11 ; T1H = a1*b1 @@ -753,58 +754,28 @@ cmp %%T2, (48*16) jl %%_exit_AAD_loop48x16 - vmovdqu64 %%ZT1, [%%T1 + 64*0] ; Blocks 0-3 - vmovdqu64 %%ZT2, [%%T1 + 64*1] ; Blocks 4-7 - vmovdqu64 %%ZT3, [%%T1 + 64*2] ; Blocks 8-11 - vmovdqu64 %%ZT4, [%%T1 + 64*3] ; Blocks 12-15 - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_48, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 - - vmovdqu64 %%ZT1, [%%T1 + 16*16 + 64*0] ; Blocks 16-19 - vmovdqu64 %%ZT2, [%%T1 + 16*16 + 64*1] ; Blocks 20-23 - vmovdqu64 %%ZT3, [%%T1 + 16*16 + 64*2] ; Blocks 24-27 - vmovdqu64 %%ZT4, [%%T1 + 16*16 + 64*3] ; Blocks 28-31 - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK GHASH_16 mid, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (4*64), 0, \ %%GDATA_KEY, HashKey_32, 0, NO_HASH_IN_OUT, \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 - - vmovdqu64 %%ZT1, [%%T1 + 32*16 + 64*0] ; Blocks 32-35 - vmovdqu64 %%ZT2, [%%T1 + 32*16 + 64*1] ; Blocks 36-39 - vmovdqu64 %%ZT3, [%%T1 + 32*16 + 64*2] ; Blocks 40-43 - vmovdqu64 %%ZT4, [%%T1 + 32*16 + 64*3] ; Blocks 44-47 - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (8*64), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK - sub %%T2, (48*16) + sub %%T2, (12*64) je %%_CALC_AAD_done - add %%T1, (48*16) + add %%T1, (12*64) jmp %%_get_AAD_loop48x16 %%_exit_AAD_loop48x16: @@ -813,37 +784,17 @@ jl %%_less_than_32x16 ; Get next 16 blocks - vmovdqu64 %%ZT1, [%%T1 + 64*0] - vmovdqu64 %%ZT2, [%%T1 + 64*1] - vmovdqu64 %%ZT3, [%%T1 + 64*2] - vmovdqu64 %%ZT4, [%%T1 + 64*3] - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK - GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 - - vmovdqu64 %%ZT1, [%%T1 + 16*16 + 64*0] - vmovdqu64 %%ZT2, [%%T1 + 16*16 + 64*1] - vmovdqu64 %%ZT3, [%%T1 + 16*16 + 64*2] - vmovdqu64 %%ZT4, [%%T1 + 16*16 + 64*3] - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK GHASH_16 end_reduce, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (4*64), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK sub %%T2, (32*16) je %%_CALC_AAD_done @@ -855,21 +806,11 @@ cmp %%T2, (16*16) jl %%_less_than_16x16 ; Get next 16 blocks - vmovdqu64 %%ZT1, [%%T1 + 64*0] - vmovdqu64 %%ZT2, [%%T1 + 64*1] - vmovdqu64 %%ZT3, [%%T1 + 64*2] - vmovdqu64 %%ZT4, [%%T1 + 64*3] - vpshufb %%ZT1, %%SHFMSK - vpshufb %%ZT2, %%SHFMSK - vpshufb %%ZT3, %%SHFMSK - vpshufb %%ZT4, %%SHFMSK - GHASH_16 start_reduce, %%ZT5, %%ZT6, %%ZT7, \ - NO_INPUT_PTR, NO_INPUT_PTR, NO_INPUT_PTR, \ + %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ - %%ZT14, %%ZT15, %%ZT16, NO_ZMM, \ - %%ZT1, %%ZT2, %%ZT3, %%ZT4 + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK sub %%T2, (16*16) je %%_CALC_AAD_done -- GitLab From 4a44cfeeeb328edbcc1563921c604edf3cb8076b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Apr 2022 11:28:52 +0000 Subject: [PATCH 066/369] perf: [ghash] added support for testing GHASH through direct API --- ReleaseNotes.txt | 2 +- perf/ipsec_perf.c | 69 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 6abeee3f..17c8ce6b 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -18,7 +18,7 @@ Test Applications - GHASH JOB API support added in the test application, fuzzing and xvalid tools Performance Application -- GHASH support added (through JOB API) +- GHASH support added (through JOB and direct API) Fixes - Fixed incorrect 8-buffer SNOW3G keystream generation diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 5b59f36f..ff2bd321 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -2207,6 +2207,61 @@ do_test_gcm(struct params_s *params, return time / num_iter; } +/* Performs test using GCM */ +static uint64_t +do_test_ghash(struct params_s *params, + const uint32_t num_iter, IMB_MGR *mb_mgr, + uint8_t *p_buffer, imb_uint128_t *p_keys) +{ + static DECLARE_ALIGNED(struct gcm_key_data gdata_key, 512); + uint64_t time = 0; + uint32_t aux; + uint32_t i, index = 0; + uint8_t auth_tag[16]; + + IMB_GHASH_PRE(mb_mgr, p_keys, &gdata_key); + +#ifndef _WIN32 + if (use_unhalted_cycles) + time = read_cycles(params->core); + else +#endif + time = __rdtscp(&aux); + + if (imix_list_count != 0) { + for (i = 0; i < num_iter; i++) { + uint8_t *pb = get_dst_buffer(index, p_buffer); + const uint32_t buf_size = get_next_size(i); + + IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, + auth_tag, sizeof(auth_tag)); + index = get_next_index(index); + } + } else { + for (i = 0; i < num_iter; i++) { + uint8_t *pb = get_dst_buffer(index, p_buffer); + const uint32_t buf_size = params->size_aes; + + IMB_GHASH(mb_mgr, &gdata_key, pb, buf_size, + auth_tag, sizeof(auth_tag)); + index = get_next_index(index); + } + } + +#ifndef _WIN32 + if (use_unhalted_cycles) + time = (read_cycles(params->core) - + rd_cycles_cost) - time; + else +#endif + time = __rdtscp(&aux) - time; + + if (!num_iter) + return time; + + return time / num_iter; +} + /* Computes mean of set of times after dropping bottom and top quarters */ static uint64_t mean_median(uint64_t *array, uint32_t size, @@ -2303,6 +2358,14 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, else *times = do_test_gcm(params, job_iter, mgr, p_buffer, p_keys); + } else if (params->hash_alg == TEST_AUTH_GHASH && + (!use_gcm_job_api)) { + if (job_iter == 0) + *times = do_test_ghash(params, 2 * num_iter, + mgr, p_buffer, p_keys); + else + *times = do_test_ghash(params, job_iter, mgr, + p_buffer, p_keys); } else { if (job_iter == 0) *times = do_test(mgr, params, num_iter, @@ -2630,10 +2693,10 @@ static void usage(void) "-o val: Use for the SHA size increment, default is 24\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n" - "--gcm-job-api: use JOB API for GCM perf tests" - " (raw GCM API is default)\n" + "--gcm-job-api: use JOB API for GCM/GHASH perf tests" + " (direct GCM/GHASH API is default)\n" "--gcm-sgl-api: use direct SGL API for GCM perf tests" - " (raw GCM API is default)\n" + " (direct GCM API is default)\n" "--threads num: for the number of threads to run" " Max: %d\n" "--cores mask: CPU's to run threads\n" -- GitLab From 2ff2e3ea507b04d6e28363f1083470dfa9586ca2 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Apr 2022 14:25:28 +0000 Subject: [PATCH 067/369] avx512-vaes: [ghash] perform only one reduction for message lengths below 48 blocks --- lib/avx512/gcm_vaes_avx512.inc | 149 +++++++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 16 deletions(-) diff --git a/lib/avx512/gcm_vaes_avx512.inc b/lib/avx512/gcm_vaes_avx512.inc index a7812ba2..e6241b32 100644 --- a/lib/avx512/gcm_vaes_avx512.inc +++ b/lib/avx512/gcm_vaes_avx512.inc @@ -745,14 +745,14 @@ mov %%T1, %%A_IN ; T1 = AAD mov %%T2, %%A_LEN ; T2 = aadLen - or %%T2, %%T2 - jz %%_CALC_AAD_done + cmp %%T2, (16*16) + jb %%_less_than_16x16 vmovdqa64 %%SHFMSK, [rel SHUF_MASK] %%_get_AAD_loop48x16: cmp %%T2, (48*16) - jl %%_exit_AAD_loop48x16 + jb %%_exit_AAD_loop48x16 GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ %%T1, (0*64), 0, \ @@ -780,10 +780,37 @@ %%_exit_AAD_loop48x16: ; Less than 48x16 bytes remaining + cmp %%T2, (16*16) + jb %%_less_than_16x16 + je %%_equal_16x16 cmp %%T2, (32*16) - jl %%_less_than_32x16 + jb %%_less_than_32x16 + je %%_equal_32x16 + + ;; calculate offset to hash key to start with + lea %%T3, [%%T2 + 15] + and %%T3, ~15 + neg %%T3 + add %%T3, HashKey_1 + 16 ; Get next 16 blocks + GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ + %%T1, (0*64), 0, \ + %%GDATA_KEY, %%T3, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + + GHASH_16 mid, %%ZT5, %%ZT6, %%ZT7, \ + %%T1, (4*64), 0, \ + %%GDATA_KEY, {%%T3 + 16*16}, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + + sub %%T2, (32*16) + add %%T1, (32*16) + jmp %%_less_than_16x16_remain + +%%_equal_32x16: GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_32, 0, ZWORD(%%AAD_HASH), \ @@ -795,29 +822,110 @@ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + jmp %%_CALC_AAD_done - sub %%T2, (32*16) - je %%_CALC_AAD_done +%%_less_than_32x16: + ;; calculate offset to hash key to start with + lea %%T3, [%%T2 + 15] + and %%T3, ~15 + neg %%T3 + add %%T3, HashKey_1 + 16 - add %%T1, (32*16) - jmp %%_less_than_16x16 + GHASH_16 start, %%ZT5, %%ZT6, %%ZT7, \ + %%T1, (0*64), 0, \ + %%GDATA_KEY, %%T3, 0, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + sub %%T2, (16*16) + add %%T1, (16*16) + jmp %%_less_than_16x16_remain -%%_less_than_32x16: - cmp %%T2, (16*16) - jl %%_less_than_16x16 - ; Get next 16 blocks +%%_equal_16x16: GHASH_16 start_reduce, %%ZT5, %%ZT6, %%ZT7, \ %%T1, (0*64), 0, \ %%GDATA_KEY, HashKey_16, 0, ZWORD(%%AAD_HASH), \ %%ZT0, %%ZT8, %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ %%ZT14, %%ZT15, %%ZT16, %%ZT1, %%SHFMSK + jmp %%_CALC_AAD_done - sub %%T2, (16*16) - je %%_CALC_AAD_done + ; Less than 16x16 bytes remaining +%%_less_than_16x16_remain: + ;; ZT5 (H), ZT6 (M), ZT7 (L) contain ghash sums + ;; prep mask source address + lea %%T3, [rel byte64_len_to_mask_table] + lea %%T3, [%%T3 + %%T2*8] - add %%T1, (16*16) + ;; calculate number of blocks to ghash (including partial bytes) + add DWORD(%%T2), 15 + shr DWORD(%%T2), 4 + jz %%_CALC_AAD_done ;; catch zero length + cmp DWORD(%%T2), 2 + jb %%_AAD_blocks_cont_1 + je %%_AAD_blocks_cont_2 + cmp DWORD(%%T2), 4 + jb %%_AAD_blocks_cont_3 + je %%_AAD_blocks_cont_4 + cmp DWORD(%%T2), 6 + jb %%_AAD_blocks_cont_5 + je %%_AAD_blocks_cont_6 + cmp DWORD(%%T2), 8 + jb %%_AAD_blocks_cont_7 + je %%_AAD_blocks_cont_8 + cmp DWORD(%%T2), 10 + jb %%_AAD_blocks_cont_9 + je %%_AAD_blocks_cont_10 + cmp DWORD(%%T2), 12 + jb %%_AAD_blocks_cont_11 + je %%_AAD_blocks_cont_12 + cmp DWORD(%%T2), 14 + jb %%_AAD_blocks_cont_13 + je %%_AAD_blocks_cont_14 + cmp DWORD(%%T2), 15 + je %%_AAD_blocks_cont_15 + ;; fall through for 16 blocks - ; Less than 16x16 bytes remaining + ;; The flow of each of these cases is identical: + ;; - load blocks plain text + ;; - shuffle loaded blocks + ;; - xor in current hash value into block 0 + ;; - perform up multiplications with ghash keys + ;; - jump to reduction code + +%assign I 16 + ;; generate all 16 cases using preprocessor +%rep 16 + +%%_AAD_blocks_cont_ %+ I: +%if I > 12 + sub %%T3, 12 * 16 * 8 +%elif I > 8 + sub %%T3, 8 * 16 * 8 +%elif I > 4 + sub %%T3, 4 * 16 * 8 +%endif + kmovq %%MASKREG, [%%T3] + + ZMM_LOAD_MASKED_BLOCKS_0_16 \ + I, %%T1, 0, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT17, %%MASKREG + + ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16 \ + I, vpshufb, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ + %%ZT14, %%ZT15, %%ZT16, %%ZT17, \ + %%SHFMSK, %%SHFMSK, %%SHFMSK, %%SHFMSK + + GHASH_1_TO_16 %%GDATA_KEY, ZWORD(%%AAD_HASH), \ + %%ZT0, %%ZT1, %%ZT2, %%ZT3, %%ZT4, \ + %%ZT9, %%ZT10, %%ZT11, %%ZT12, \ + ZWORD(%%AAD_HASH), %%ZT14, %%ZT15, %%ZT16, %%ZT17, I, \ + %%ZT5, %%ZT6, %%ZT7 + jmp %%_CALC_AAD_done + +%assign I (I - 1) +%endrep + + ; Less than 16x16 bytes %%_less_than_16x16: ;; prep mask source address lea %%T3, [rel byte64_len_to_mask_table] @@ -826,6 +934,7 @@ ;; calculate number of blocks to ghash (including partial bytes) add DWORD(%%T2), 15 shr DWORD(%%T2), 4 + jz %%_CALC_AAD_done ;; catch zero length cmp DWORD(%%T2), 2 jb %%_AAD_blocks_1 je %%_AAD_blocks_2 @@ -863,6 +972,14 @@ %rep 16 %%_AAD_blocks_ %+ I: +%if I >= 3 + vmovdqa64 %%SHFMSK, [rel SHUF_MASK] +%elif I == 2 + vmovdqa64 YWORD(%%SHFMSK), [rel SHUF_MASK] +%elif I == 1 + vmovdqa64 XWORD(%%SHFMSK), [rel SHUF_MASK] +%endif + %if I > 12 sub %%T3, 12 * 16 * 8 %elif I > 8 -- GitLab From 68960ce7d4beba99071032027ec9d22dba0b2d3d Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 28 Apr 2022 10:11:08 +0100 Subject: [PATCH 068/369] lib: Extend Windows .def file with new burst API --- lib/libIPSec_MB.def | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index 258cf8cf..0a93f6e1 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -538,9 +538,19 @@ EXPORTS crc7_fp_header_sse_no_aesni @512 crc7_fp_header_avx @513 crc7_fp_header_avx512 @514 - imb_get_errno @515 - imb_get_strerror @516 - init_mb_mgr_auto @517 + imb_get_errno @515 + imb_get_strerror @516 + init_mb_mgr_auto @517 imb_get_mb_mgr_size @518 imb_set_pointers_mb_mgr @519 imb_get_feature_flags @520 + submit_burst_sse @521 + submit_burst_sse_no_aesni @522 + submit_burst_avx @523 + submit_burst_avx2 @524 + submit_burst_avx512 @525 + submit_burst_nocheck_sse @526 + submit_burst_nocheck_sse_no_aesni @527 + submit_burst_nocheck_avx @528 + submit_burst_nocheck_avx2 @529 + submit_burst_nocheck_avx512 @530 -- GitLab From 831a57e5bf87a69ba53f5f8acd118ffb5a10b5f7 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 26 Apr 2022 08:18:06 +0000 Subject: [PATCH 069/369] test: [GCM] Add basic burst API tests --- test/gcm_test.c | 446 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 397 insertions(+), 49 deletions(-) diff --git a/test/gcm_test.c b/test/gcm_test.c index 2dccd934..b6887b45 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -43,6 +43,7 @@ #define IV_SZ 12 #define DIGEST_SZ 16 #define MAX_KEY_SZ 32 +#define MAX_JOBS 32 /* * 60-Byte Packet Encryption Using GCM-AES-128 @@ -1316,6 +1317,14 @@ typedef int (*gcm_enc_dec_fn_t)(IMB_MGR *, const struct gcm_key_data *, const uint8_t *, uint64_t, uint8_t *, uint64_t, IMB_KEY_SIZE_BYTES); +typedef int (*gcm_enc_dec_many_fn_t)(IMB_MGR *, const struct gcm_key_data *, + struct gcm_context_data **, + uint8_t **, const uint8_t *, + const uint64_t, const uint8_t *, + const uint64_t, const uint8_t *, + const uint64_t, uint8_t **, const uint64_t, + const IMB_KEY_SIZE_BYTES, const uint32_t); + static IMB_MGR *p_gcm_mgr = NULL; static int check_data(const uint8_t *test, const uint8_t *expected, @@ -1620,19 +1629,174 @@ sgl_aes_gcm_dec(IMB_MGR *p_mgr, return 0; } +/***************************************************************************** + * burst API + *****************************************************************************/ +static int +aes_gcm_burst(IMB_MGR *mb_mgr, + const IMB_CIPHER_DIRECTION cipher_dir, + const struct gcm_key_data *key, + const uint64_t key_len, + uint8_t **out, const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, const uint8_t *aad, + const uint64_t aad_len, uint8_t **auth_tag, + const uint64_t auth_tag_len, struct gcm_context_data **ctx, + const IMB_CIPHER_MODE cipher_mode, const IMB_SGL_STATE sgl_state, + const uint32_t num_jobs) +{ + IMB_JOB *job, jobs[MAX_JOBS]; + uint32_t i; + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + job->cipher_mode = cipher_mode; + job->chain_order = + (cipher_dir == IMB_DIR_ENCRYPT) ? + IMB_ORDER_CIPHER_HASH : + IMB_ORDER_HASH_CIPHER; + job->enc_keys = key; + job->dec_keys = key; + job->key_len_in_bytes = key_len; + job->src = in; + job->dst = out[i]; + job->msg_len_to_cipher_in_bytes = len; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = iv; + job->iv_len_in_bytes = iv_len; + job->u.GCM.aad = aad; + job->u.GCM.aad_len_in_bytes = aad_len; + job->auth_tag_output = auth_tag[i]; + job->auth_tag_output_len_in_bytes = auth_tag_len; + job->cipher_direction = cipher_dir; + if (cipher_mode == IMB_CIPHER_GCM_SGL) { + job->u.GCM.ctx = ctx[i]; + job->sgl_state = sgl_state; + job->hash_alg = IMB_AUTH_GCM_SGL; + } else + job->hash_alg = IMB_AUTH_AES_GMAC; + } + + const uint32_t completed_jobs = + IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + return -1; + } + } + + return 0; +} + +static int +burst_aes_gcm_enc(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + return aes_gcm_burst(p_mgr, IMB_DIR_ENCRYPT, key, + key_len, out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM, 0, + num_jobs); +} + +static int +burst_aes_gcm_dec(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + return aes_gcm_burst(p_mgr, IMB_DIR_DECRYPT, key, + key_len, out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM, 0, + num_jobs); +} + +static int +burst_sgl_aes_gcm(IMB_MGR *p_mgr, + IMB_CIPHER_DIRECTION cipher_dir, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + if (aes_gcm_burst(p_mgr, cipher_dir, key, + key_len, out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, + IMB_SGL_INIT, num_jobs) < 0) + return -1; + if (aes_gcm_burst(p_mgr, cipher_dir, key, + key_len, out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, + IMB_SGL_UPDATE, num_jobs) < 0) + return -1; + if (aes_gcm_burst(p_mgr, cipher_dir, key, + key_len, out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, + IMB_SGL_COMPLETE, num_jobs) < 0) + return -1; + + return 0; +} + +static int +burst_sgl_aes_gcm_enc(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + return burst_sgl_aes_gcm(p_mgr, IMB_DIR_ENCRYPT, key, ctx, out, in, + len, iv, iv_len, aad, aad_len, auth_tag, + auth_tag_len, key_len, num_jobs); +} + +static int +burst_sgl_aes_gcm_dec(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data **ctx, uint8_t **out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t **auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len, const uint32_t num_jobs) +{ + return burst_sgl_aes_gcm(p_mgr, IMB_DIR_DECRYPT, key, ctx, out, in, + len, iv, iv_len, aad, aad_len, auth_tag, + auth_tag_len, key_len, num_jobs); +} + /***************************************************************************** * job API *****************************************************************************/ static int aes_gcm_job(IMB_MGR *mb_mgr, IMB_CIPHER_DIRECTION cipher_dir, - const struct gcm_key_data *key, - uint64_t key_len, - uint8_t *out, const uint8_t *in, uint64_t len, + const struct gcm_key_data *key, const uint64_t key_len, + uint8_t *out, const uint8_t *in, const uint64_t len, const uint8_t *iv, const uint64_t iv_len, const uint8_t *aad, - uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len, - struct gcm_context_data *ctx, IMB_CIPHER_MODE cipher_mode, - IMB_SGL_STATE sgl_state) + const uint64_t aad_len, uint8_t *auth_tag, + const uint64_t auth_tag_len, struct gcm_context_data *ctx, + const IMB_CIPHER_MODE cipher_mode, const IMB_SGL_STATE sgl_state) { IMB_JOB *job; @@ -1679,12 +1843,10 @@ aes_gcm_job(IMB_MGR *mb_mgr, static int job_aes_gcm_enc(IMB_MGR *p_mgr, const struct gcm_key_data *key, - struct gcm_context_data *ctx, - uint8_t *out, const uint8_t *in, uint64_t len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, uint64_t aad_len, - uint8_t *auth_tag, uint64_t auth_tag_len, - IMB_KEY_SIZE_BYTES key_len) + struct gcm_context_data *ctx, uint8_t *out, const uint8_t *in, + const uint64_t len, const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, uint8_t *auth_tag, + const uint64_t auth_tag_len, const IMB_KEY_SIZE_BYTES key_len) { return aes_gcm_job(p_mgr, IMB_DIR_ENCRYPT, key, key_len, out, in, len, iv, iv_len, aad, aad_len, @@ -1694,12 +1856,10 @@ job_aes_gcm_enc(IMB_MGR *p_mgr, static int job_aes_gcm_dec(IMB_MGR *p_mgr, const struct gcm_key_data *key, - struct gcm_context_data *ctx, - uint8_t *out, const uint8_t *in, uint64_t len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, uint64_t aad_len, - uint8_t *auth_tag, uint64_t auth_tag_len, - IMB_KEY_SIZE_BYTES key_len) + struct gcm_context_data *ctx, uint8_t *out, const uint8_t *in, + const uint64_t len, const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, uint8_t *auth_tag, + const uint64_t auth_tag_len, const IMB_KEY_SIZE_BYTES key_len) { return aes_gcm_job(p_mgr, IMB_DIR_DECRYPT, key, key_len, out, in, len, iv, iv_len, aad, aad_len, @@ -1707,26 +1867,27 @@ job_aes_gcm_dec(IMB_MGR *p_mgr, } static int -job_sgl_aes_gcm_enc(IMB_MGR *p_mgr, - const struct gcm_key_data *key, - struct gcm_context_data *ctx, - uint8_t *out, const uint8_t *in, uint64_t len, - const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, uint64_t aad_len, - uint8_t *auth_tag, uint64_t auth_tag_len, - IMB_KEY_SIZE_BYTES key_len) +job_sgl_aes_gcm(IMB_MGR *p_mgr, + const IMB_CIPHER_DIRECTION cipher_dir, + const struct gcm_key_data *key, + struct gcm_context_data *ctx, uint8_t *out, + const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t *auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len) { - if (aes_gcm_job(p_mgr, IMB_DIR_ENCRYPT, key, + if (aes_gcm_job(p_mgr, cipher_dir, key, key_len, out, in, len, iv, iv_len, aad, aad_len, auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, IMB_SGL_INIT) < 0) return -1; - if (aes_gcm_job(p_mgr, IMB_DIR_ENCRYPT, key, + if (aes_gcm_job(p_mgr, cipher_dir, key, key_len, out, in, len, iv, iv_len, aad, aad_len, auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, IMB_SGL_UPDATE) < 0) return -1; - if (aes_gcm_job(p_mgr, IMB_DIR_ENCRYPT, key, + if (aes_gcm_job(p_mgr, cipher_dir, key, key_len, out, in, len, iv, iv_len, aad, aad_len, auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, IMB_SGL_COMPLETE) < 0) @@ -1734,32 +1895,34 @@ job_sgl_aes_gcm_enc(IMB_MGR *p_mgr, return 0; } +static int +job_sgl_aes_gcm_enc(IMB_MGR *p_mgr, + const struct gcm_key_data *key, + struct gcm_context_data *ctx, + uint8_t *out, const uint8_t *in, const uint64_t len, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t *auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len) +{ + return job_sgl_aes_gcm(p_mgr, IMB_DIR_ENCRYPT, key, ctx, + out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, key_len); +} + static int job_sgl_aes_gcm_dec(IMB_MGR *p_mgr, const struct gcm_key_data *key, struct gcm_context_data *ctx, - uint8_t *out, const uint8_t *in, uint64_t len, + uint8_t *out, const uint8_t *in, const uint64_t len, const uint8_t *iv, const uint64_t iv_len, - const uint8_t *aad, uint64_t aad_len, - uint8_t *auth_tag, uint64_t auth_tag_len, - IMB_KEY_SIZE_BYTES key_len) + const uint8_t *aad, const uint64_t aad_len, + uint8_t *auth_tag, const uint64_t auth_tag_len, + const IMB_KEY_SIZE_BYTES key_len) { - if (aes_gcm_job(p_mgr, IMB_DIR_DECRYPT, key, - key_len, out, in, len, iv, iv_len, aad, aad_len, - auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, - IMB_SGL_INIT) < 0) - return -1; - if (aes_gcm_job(p_mgr, IMB_DIR_DECRYPT, key, - key_len, out, in, len, iv, iv_len, aad, aad_len, - auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, - IMB_SGL_UPDATE) < 0) - return -1; - if (aes_gcm_job(p_mgr, IMB_DIR_DECRYPT, key, - key_len, out, in, len, iv, iv_len, aad, aad_len, - auth_tag, auth_tag_len, ctx, IMB_CIPHER_GCM_SGL, - IMB_SGL_COMPLETE) < 0) - return -1; - return 0; + return job_sgl_aes_gcm(p_mgr, IMB_DIR_DECRYPT, key, ctx, + out, in, len, iv, iv_len, aad, aad_len, + auth_tag, auth_tag_len, key_len); } /*****************************************************************************/ @@ -1916,6 +2079,165 @@ test_gcm_vectors(struct gcm_ctr_vector const *vector, free(T2_test); } +static void +test_gcm_vectors_burst(struct gcm_ctr_vector const *vector, + gcm_enc_dec_many_fn_t encfn, + gcm_enc_dec_many_fn_t decfn, + struct test_suite_context *ts) +{ + struct gcm_key_data gdata_key; + int is_error = 0; + /* Temporary array for the calculated vectors */ + struct gcm_context_data **gdata_ctx = NULL; + uint8_t **ct_test = NULL; + uint8_t **pt_test = NULL; + uint8_t **T_test = NULL; + const uint8_t *iv = vector->IV; + uint64_t iv_len = vector->IVlen; + uint32_t i, j, num_jobs = MAX_JOBS; + + /* Allocate space for the calculated ciphertext */ + ct_test = malloc(num_jobs * sizeof(void *)); + if (ct_test == NULL) { + fprintf(stderr, "Can't allocate ciphertext memory\n"); + goto test_gcm_vectors_burst_exit; + } + memset(ct_test, 0, num_jobs * sizeof(void *)); + + /* Allocate space for the calculated plaintext */ + pt_test = malloc(num_jobs * sizeof(void *)); + if (pt_test == NULL) { + fprintf(stderr, "Can't allocate plaintext memory\n"); + goto test_gcm_vectors_burst_exit; + } + memset(pt_test, 0, num_jobs * sizeof(void *)); + + /* Allocate space for the GCM context data */ + gdata_ctx = malloc(num_jobs * sizeof(void *)); + if (gdata_ctx == NULL) { + fprintf(stderr, "Can't allocate GCM ctx memory\n"); + goto test_gcm_vectors_burst_exit; + } + memset(gdata_ctx, 0, num_jobs * sizeof(void *)); + + /* Allocate space for the calculated tag */ + T_test = malloc(num_jobs * sizeof(void *)); + if (T_test == NULL) { + fprintf(stderr, "Can't allocate tag memory\n"); + goto test_gcm_vectors_burst_exit; + } + memset(T_test, 0, num_jobs * sizeof(void *)); + + /* Zero buffers */ + for (i = 0; i < num_jobs; i++) { + if (vector->Plen != 0) { + ct_test[i] = malloc(vector->Plen); + if (ct_test[i] == NULL) + goto test_gcm_vectors_burst_exit; + memset(ct_test[i], 0, vector->Plen); + + pt_test[i] = malloc(vector->Plen); + if (pt_test[i] == NULL) + goto test_gcm_vectors_burst_exit; + memset(pt_test[i], 0, vector->Plen); + } + + gdata_ctx[i] = malloc(sizeof(struct gcm_context_data)); + if (gdata_ctx[i] == NULL) + goto test_gcm_vectors_burst_exit; + memset(gdata_ctx[i], 0, sizeof(struct gcm_context_data)); + + T_test[i] = malloc(vector->Tlen); + if (T_test[i] == NULL) + goto test_gcm_vectors_burst_exit; + memset(T_test[i], 0, vector->Tlen); + } + + /* This is only required once for a given key */ + switch (vector->Klen) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_PRE(p_gcm_mgr, vector->K, &gdata_key); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_PRE(p_gcm_mgr, vector->K, &gdata_key); + break; + case IMB_KEY_256_BYTES: + default: + IMB_AES256_GCM_PRE(p_gcm_mgr, vector->K, &gdata_key); + break; + } + + /* Test encrypt and decrypt */ + for (i = 0; i < num_jobs; i++) { + /* + * Encrypt + */ + is_error = encfn(p_gcm_mgr, &gdata_key, gdata_ctx, + ct_test, vector->P, vector->Plen, + iv, iv_len, vector->A, + vector->Alen, T_test, vector->Tlen, + vector->Klen, i + 1); + + for (j = 0; j <= i; j++) { + is_error |= check_data(ct_test[j], vector->C, + vector->Plen, + "encrypted cipher text (burst)"); + is_error |= check_data(T_test[j], vector->T, + vector->Tlen, "tag (burst)"); + } + if (is_error) + test_suite_update(ts, 0, 1); + else + test_suite_update(ts, 1, 0); + /* + * Decrypt + */ + is_error = decfn(p_gcm_mgr, &gdata_key, gdata_ctx, pt_test, + vector->C, vector->Plen, iv, iv_len, vector->A, + vector->Alen, T_test, vector->Tlen, + vector->Klen, i + 1); + + for (j = 0; j <= i; j++) { + is_error |= check_data(pt_test[j], vector->P, + vector->Plen, + "decrypted plain text (burst)"); + /* + * GCM decryption outputs a 16 byte tag value + * that must be verified against the expected tag value + */ + is_error |= check_data(T_test[j], vector->T, + vector->Tlen, + "decrypted tag (burst)"); + } + if (is_error) + test_suite_update(ts, 0, 1); + else + test_suite_update(ts, 1, 0); + } + + test_gcm_vectors_burst_exit: + if (NULL != ct_test) { + for (i = 0; i < num_jobs; i++) + free(ct_test[i]); + free(ct_test); + } + if (NULL != pt_test) { + for (i = 0; i < num_jobs; i++) + free(pt_test[i]); + free(pt_test); + } + if (NULL != gdata_ctx) { + for (i = 0; i < num_jobs; i++) + free(gdata_ctx[i]); + free(gdata_ctx); + } + if (NULL != T_test) { + for (i = 0; i < num_jobs; i++) + free(T_test[i]); + free(T_test); + } +} + static void test_gcm_std_vectors(struct test_suite_context *ts128, struct test_suite_context *ts192, @@ -1952,6 +2274,10 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_sgl_aes_gcm_enc, job_sgl_aes_gcm_dec, ts128); + test_gcm_vectors_burst(&vectors[vect], + burst_sgl_aes_gcm_enc, + burst_sgl_aes_gcm_dec, + ts128); } else { test_gcm_vectors(&vectors[vect], aes_gcm_enc, @@ -1961,6 +2287,10 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_aes_gcm_enc, job_aes_gcm_dec, ts128); + test_gcm_vectors_burst(&vectors[vect], + burst_aes_gcm_enc, + burst_aes_gcm_dec, + ts128); } break; case IMB_KEY_192_BYTES: @@ -1973,6 +2303,10 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_sgl_aes_gcm_enc, job_sgl_aes_gcm_dec, ts192); + test_gcm_vectors_burst(&vectors[vect], + burst_sgl_aes_gcm_enc, + burst_sgl_aes_gcm_dec, + ts192); } else { test_gcm_vectors(&vectors[vect], aes_gcm_enc, @@ -1982,6 +2316,10 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_aes_gcm_enc, job_aes_gcm_dec, ts192); + test_gcm_vectors_burst(&vectors[vect], + burst_aes_gcm_enc, + burst_aes_gcm_dec, + ts192); } break; case IMB_KEY_256_BYTES: @@ -1994,6 +2332,11 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_sgl_aes_gcm_enc, job_sgl_aes_gcm_dec, ts256); + test_gcm_vectors_burst(&vectors[vect], + burst_sgl_aes_gcm_enc, + burst_sgl_aes_gcm_dec, + ts256); + } else { test_gcm_vectors(&vectors[vect], aes_gcm_enc, @@ -2003,6 +2346,11 @@ test_gcm_std_vectors(struct test_suite_context *ts128, job_aes_gcm_enc, job_aes_gcm_dec, ts256); + test_gcm_vectors_burst(&vectors[vect], + burst_aes_gcm_enc, + burst_aes_gcm_dec, + ts256); + } break; default: -- GitLab From bf918f5581e2ddf47644225adaec6ff45f365a57 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 11 Apr 2022 16:10:28 +0000 Subject: [PATCH 070/369] avx512: [ZUC-EIA3] remove unused variables --- lib/avx512/zuc_top_avx512.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/avx512/zuc_top_avx512.c b/lib/avx512/zuc_top_avx512.c index 6c8d672e..69d55972 100755 --- a/lib/avx512/zuc_top_avx512.c +++ b/lib/avx512/zuc_top_avx512.c @@ -705,9 +705,6 @@ void _zuc_eia3_16_buffer_avx512(const void * const pKey[NUM_AVX512_BUFS], /* Process each packet separately for the remaining bits */ for (i = 0; i < NUM_AVX512_BUFS; i++) { - const uint32_t N = lengthInBits[i] + (2 * ZUC_WORD_BITS); - uint32_t L = ((N + 31) / ZUC_WORD_BITS) - - numKeyStr*(keyStreamLengthInBits / 32); uint32_t remainBits = lengthInBits[i] - numKeyStr*keyStreamLengthInBits; uint32_t keyStr32[16*2]; @@ -748,7 +745,6 @@ void _zuc_eia3_16_buffer_avx512(const void * const pKey[NUM_AVX512_BUFS], while (remainBits >= keyStreamLengthInBits) { remainBits -= keyStreamLengthInBits; - L -= (keyStreamLengthInBits / 32); /* Generate the next key stream 8 bytes or 64 bytes */ if (!remainBits) -- GitLab From 366805bb7b0b8a084645c8babf783e1863f65a1e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 12 Apr 2022 17:39:17 +0000 Subject: [PATCH 071/369] avx512: [ZUC] refactor main auxiliary functions Refactored init, keygen and cipher assembly functions, defining register names and fixing leading spaces. --- lib/avx512/zuc_x16_avx512.asm | 1120 ++++++++++++++++++--------------- 1 file changed, 607 insertions(+), 513 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 36dfb0e5..05873384 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -462,21 +462,21 @@ align 64 ; The macro clobbers ZMM0-15 ; %macro REORDER_LFSR 3 -%define %%STATE %1 -%define %%NUM_ROUNDS %2 -%define %%LANE_MASK %3 +%define %%STATE %1 ; [in] Pointer to LFSR state +%define %%NUM_ROUNDS %2 ; [immediate] Number of key generation rounds +%define %%LANE_MASK %3 ; [in] Mask register with lanes to update %if %%NUM_ROUNDS != 16 %assign i 0 %rep 16 - vmovdqa32 APPEND(zmm,i){%%LANE_MASK}, [%%STATE + 64*i] + vmovdqa32 APPEND(zmm,i){%%LANE_MASK}, [%%STATE + 64*i] %assign i (i+1) %endrep %assign i 0 %assign j %%NUM_ROUNDS %rep 16 - vmovdqa32 [%%STATE + 64*i]{%%LANE_MASK}, APPEND(zmm,j) + vmovdqa32 [%%STATE + 64*i]{%%LANE_MASK}, APPEND(zmm,j) %assign i (i+1) %assign j ((j+1) % 16) %endrep @@ -653,20 +653,20 @@ align 64 vpshldd %%X3, %%LFSR_2, %%LFSR_0, 16 %endif %else ; USE_GFNI_VAES_VPCLMUL == 1 - vpxorq %%ZTMP, %%ZTMP - vpslld %%LFSR_15, 1 - vpblendmw %%ZTMP{%%BLEND_KMASK}, %%LFSR_14, %%ZTMP - vpblendmw %%X0{%%BLEND_KMASK}, %%ZTMP, %%LFSR_15 - vpslld %%LFSR_11, 16 - vpsrld %%LFSR_9, 15 - vporq %%X1, %%LFSR_11, %%LFSR_9 - vpslld %%LFSR_7, 16 - vpsrld %%LFSR_5, 15 - vporq %%X2, %%LFSR_7, %%LFSR_5 + vpxorq %%ZTMP, %%ZTMP + vpslld %%LFSR_15, 1 + vpblendmw %%ZTMP{%%BLEND_KMASK}, %%LFSR_14, %%ZTMP + vpblendmw %%X0{%%BLEND_KMASK}, %%ZTMP, %%LFSR_15 + vpslld %%LFSR_11, 16 + vpsrld %%LFSR_9, 15 + vporq %%X1, %%LFSR_11, %%LFSR_9 + vpslld %%LFSR_7, 16 + vpsrld %%LFSR_5, 15 + vporq %%X2, %%LFSR_7, %%LFSR_5 %if (%0 == 17) - vpslld %%LFSR_2, 16 - vpsrld %%LFSR_0, 15 - vporq %%X3, %%LFSR_2, %%LFSR_0 ; Store BRC_X3 in ZMM register + vpslld %%LFSR_2, 16 + vpsrld %%LFSR_0, 15 + vporq %%X3, %%LFSR_2, %%LFSR_0 ; Store BRC_X3 in ZMM register %endif ; %0 == 17 %endif ; USE_GFNI_VAES_VPCLMUL == 1 %endmacro @@ -694,63 +694,63 @@ align 64 %define %%W2 %%ZTMP6 %if (%0 == 14) - vpxorq %%W, %%X0, %%R1 - vpaddd %%W, %%R2 ; W = (BRC_X0 ^ F_R1) + F_R2 + vpxorq %%W, %%X0, %%R1 + vpaddd %%W, %%R2 ; W = (BRC_X0 ^ F_R1) + F_R2 %endif - vpaddd %%W1, %%R1, %%X1 ; W1 = F_R1 + BRC_X1 - vpxorq %%W2, %%R2, %%X2 ; W2 = F_R2 ^ BRC_X2 + vpaddd %%W1, %%R1, %%X1 ; W1 = F_R1 + BRC_X1 + vpxorq %%W2, %%R2, %%X2 ; W2 = F_R2 ^ BRC_X2 %if USE_GFNI_VAES_VPCLMUL == 1 - vpshldd %%ZTMP1, %%W1, %%W2, 16 - vpshldd %%ZTMP2, %%W2, %%W1, 16 + vpshldd %%ZTMP1, %%W1, %%W2, 16 + vpshldd %%ZTMP2, %%W2, %%W1, 16 %else - vpslld %%ZTMP3, %%W1, 16 - vpsrld %%ZTMP4, %%W1, 16 - vpslld %%ZTMP5, %%W2, 16 - vpsrld %%ZTMP6, %%W2, 16 - vporq %%ZTMP1, %%ZTMP3, %%ZTMP6 - vporq %%ZTMP2, %%ZTMP4, %%ZTMP5 + vpslld %%ZTMP3, %%W1, 16 + vpsrld %%ZTMP4, %%W1, 16 + vpslld %%ZTMP5, %%W2, 16 + vpsrld %%ZTMP6, %%W2, 16 + vporq %%ZTMP1, %%ZTMP3, %%ZTMP6 + vporq %%ZTMP2, %%ZTMP4, %%ZTMP5 %endif - vprold %%ZTMP3, %%ZTMP1, 10 - vprold %%ZTMP4, %%ZTMP1, 18 - vprold %%ZTMP5, %%ZTMP1, 24 - vprold %%ZTMP6, %%ZTMP1, 2 - ; ZMM1 = U = L1(P) - vpternlogq %%ZTMP1, %%ZTMP3, %%ZTMP4, 0x96 ; (A ^ B) ^ C - vpternlogq %%ZTMP1, %%ZTMP5, %%ZTMP6, 0x96 ; (A ^ B) ^ C - - vprold %%ZTMP3, %%ZTMP2, 8 - vprold %%ZTMP4, %%ZTMP2, 14 - vprold %%ZTMP5, %%ZTMP2, 22 - vprold %%ZTMP6, %%ZTMP2, 30 - ; ZMM2 = V = L2(Q) - vpternlogq %%ZTMP2, %%ZTMP3, %%ZTMP4, 0x96 ; (A ^ B) ^ C - vpternlogq %%ZTMP2, %%ZTMP5, %%ZTMP6, 0x96 ; (A ^ B) ^ C - - ; Shuffle U and V to have all S0 lookups in XMM1 and all S1 lookups in XMM2 - - ; Compress all S0 and S1 input values in each register - ; S0: Bytes 0-7,16-23,32-39,48-55 S1: Bytes 8-15,24-31,40-47,56-63 - vpshufb %%ZTMP1, [rel S0_S1_shuf] - ; S1: Bytes 0-7,16-23,32-39,48-55 S0: Bytes 8-15,24-31,40-47,56-63 - vpshufb %%ZTMP2, [rel S1_S0_shuf] - - vshufpd %%ZTMP3, %%ZTMP1, %%ZTMP2, 0xAA ; All S0 input values - vshufpd %%ZTMP4, %%ZTMP2, %%ZTMP1, 0xAA ; All S1 input values - - ; Compute S0 and S1 values - S0_comput_AVX512 %%ZTMP3, %%ZTMP1, %%ZTMP2, USE_GFNI_VAES_VPCLMUL - S1_comput_AVX512 %%ZTMP4, %%ZTMP1, %%ZTMP2, %%ZTMP5, %%ZTMP6, USE_GFNI_VAES_VPCLMUL - - ; Need to shuffle back %%ZTMP1 & %%ZTMP2 before storing output - ; (revert what was done before S0 and S1 computations) - vshufpd %%ZTMP1, %%ZTMP3, %%ZTMP4, 0xAA - vshufpd %%ZTMP2, %%ZTMP4, %%ZTMP3, 0xAA - - vpshufb %%R1, %%ZTMP1, [rel rev_S0_S1_shuf] - vpshufb %%R2, %%ZTMP2, [rel rev_S1_S0_shuf] + vprold %%ZTMP3, %%ZTMP1, 10 + vprold %%ZTMP4, %%ZTMP1, 18 + vprold %%ZTMP5, %%ZTMP1, 24 + vprold %%ZTMP6, %%ZTMP1, 2 + ; ZMM1 = U = L1(P) + vpternlogq %%ZTMP1, %%ZTMP3, %%ZTMP4, 0x96 ; (A ^ B) ^ C + vpternlogq %%ZTMP1, %%ZTMP5, %%ZTMP6, 0x96 ; (A ^ B) ^ C + + vprold %%ZTMP3, %%ZTMP2, 8 + vprold %%ZTMP4, %%ZTMP2, 14 + vprold %%ZTMP5, %%ZTMP2, 22 + vprold %%ZTMP6, %%ZTMP2, 30 + ; ZMM2 = V = L2(Q) + vpternlogq %%ZTMP2, %%ZTMP3, %%ZTMP4, 0x96 ; (A ^ B) ^ C + vpternlogq %%ZTMP2, %%ZTMP5, %%ZTMP6, 0x96 ; (A ^ B) ^ C + + ; Shuffle U and V to have all S0 lookups in XMM1 and all S1 lookups in XMM2 + + ; Compress all S0 and S1 input values in each register + ; S0: Bytes 0-7,16-23,32-39,48-55 S1: Bytes 8-15,24-31,40-47,56-63 + vpshufb %%ZTMP1, [rel S0_S1_shuf] + ; S1: Bytes 0-7,16-23,32-39,48-55 S0: Bytes 8-15,24-31,40-47,56-63 + vpshufb %%ZTMP2, [rel S1_S0_shuf] + + vshufpd %%ZTMP3, %%ZTMP1, %%ZTMP2, 0xAA ; All S0 input values + vshufpd %%ZTMP4, %%ZTMP2, %%ZTMP1, 0xAA ; All S1 input values + + ; Compute S0 and S1 values + S0_comput_AVX512 %%ZTMP3, %%ZTMP1, %%ZTMP2, USE_GFNI_VAES_VPCLMUL + S1_comput_AVX512 %%ZTMP4, %%ZTMP1, %%ZTMP2, %%ZTMP5, %%ZTMP6, USE_GFNI_VAES_VPCLMUL + + ; Need to shuffle back %%ZTMP1 & %%ZTMP2 before storing output + ; (revert what was done before S0 and S1 computations) + vshufpd %%ZTMP1, %%ZTMP3, %%ZTMP4, 0xAA + vshufpd %%ZTMP2, %%ZTMP4, %%ZTMP3, 0xAA + + vpshufb %%R1, %%ZTMP1, [rel rev_S0_S1_shuf] + vpshufb %%R2, %%ZTMP2, [rel rev_S1_S0_shuf] %endmacro ; @@ -788,74 +788,74 @@ align 64 %define %%KMASK2 %24 ; [clobbered] Temporary K mask %if (%0 == 18) - vmovdqu64 [%%KS + %%KEY_OFF*4], %%DATA64B_L0 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 - - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512], %%DATA64B_L4 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 2*64], %%DATA64B_L6 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 3*64], %%DATA64B_L7 - - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2], %%DATA64B_L8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 - - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3], %%DATA64B_L12 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 + vmovdqu64 [%%KS + %%KEY_OFF*4], %%DATA64B_L0 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 + + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512], %%DATA64B_L4 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 2*64], %%DATA64B_L6 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 3*64], %%DATA64B_L7 + + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2], %%DATA64B_L8 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 + + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3], %%DATA64B_L12 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 %else - pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_0_4_8_12] - kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - ; Shifting left 8 bytes of KS for lanes which first 8 bytes are skipped - vpalignr %%DATA64B_L3{%%KMASK1}, %%DATA64B_L3, %%DATA64B_L2, 8 - vpalignr %%DATA64B_L2{%%KMASK1}, %%DATA64B_L2, %%DATA64B_L1, 8 - vpalignr %%DATA64B_L1{%%KMASK1}, %%DATA64B_L1, %%DATA64B_L0, 8 - vpalignr %%DATA64B_L0{%%KMASK1}, %%DATA64B_L0, %%DATA64B_L3, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4]{%%KMASK2}, %%DATA64B_L0 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 - - pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_1_5_9_13] - kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L7{%%KMASK1}, %%DATA64B_L7, %%DATA64B_L6, 8 - vpalignr %%DATA64B_L6{%%KMASK1}, %%DATA64B_L6, %%DATA64B_L5, 8 - vpalignr %%DATA64B_L5{%%KMASK1}, %%DATA64B_L5, %%DATA64B_L4, 8 - vpalignr %%DATA64B_L4{%%KMASK1}, %%DATA64B_L4, %%DATA64B_L7, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512]{%%KMASK2}, %%DATA64B_L4 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*2], %%DATA64B_L6 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*3], %%DATA64B_L7 - - pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_2_6_10_14] - kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L11{%%KMASK1}, %%DATA64B_L11, %%DATA64B_L10, 8 - vpalignr %%DATA64B_L10{%%KMASK1}, %%DATA64B_L10, %%DATA64B_L9, 8 - vpalignr %%DATA64B_L9{%%KMASK1}, %%DATA64B_L9, %%DATA64B_L8, 8 - vpalignr %%DATA64B_L8{%%KMASK1}, %%DATA64B_L8, %%DATA64B_L11, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2]{%%KMASK2}, %%DATA64B_L8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 - - pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_3_7_11_15] - kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L15{%%KMASK1}, %%DATA64B_L15, %%DATA64B_L14, 8 - vpalignr %%DATA64B_L14{%%KMASK1}, %%DATA64B_L14, %%DATA64B_L13, 8 - vpalignr %%DATA64B_L13{%%KMASK1}, %%DATA64B_L13, %%DATA64B_L12, 8 - vpalignr %%DATA64B_L12{%%KMASK1}, %%DATA64B_L12, %%DATA64B_L15, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3]{%%KMASK2}, %%DATA64B_L12 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 + pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_0_4_8_12] + kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] + kmovb %%KMASK2, [%%MOV_MASK + %%TMP] + ; Shifting left 8 bytes of KS for lanes which first 8 bytes are skipped + vpalignr %%DATA64B_L3{%%KMASK1}, %%DATA64B_L3, %%DATA64B_L2, 8 + vpalignr %%DATA64B_L2{%%KMASK1}, %%DATA64B_L2, %%DATA64B_L1, 8 + vpalignr %%DATA64B_L1{%%KMASK1}, %%DATA64B_L1, %%DATA64B_L0, 8 + vpalignr %%DATA64B_L0{%%KMASK1}, %%DATA64B_L0, %%DATA64B_L3, 8 + vmovdqu64 [%%KS + %%KEY_OFF*4]{%%KMASK2}, %%DATA64B_L0 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 + + pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_1_5_9_13] + kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] + kmovb %%KMASK2, [%%MOV_MASK + %%TMP] + vpalignr %%DATA64B_L7{%%KMASK1}, %%DATA64B_L7, %%DATA64B_L6, 8 + vpalignr %%DATA64B_L6{%%KMASK1}, %%DATA64B_L6, %%DATA64B_L5, 8 + vpalignr %%DATA64B_L5{%%KMASK1}, %%DATA64B_L5, %%DATA64B_L4, 8 + vpalignr %%DATA64B_L4{%%KMASK1}, %%DATA64B_L4, %%DATA64B_L7, 8 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512]{%%KMASK2}, %%DATA64B_L4 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*2], %%DATA64B_L6 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*3], %%DATA64B_L7 + + pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_2_6_10_14] + kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] + kmovb %%KMASK2, [%%MOV_MASK + %%TMP] + vpalignr %%DATA64B_L11{%%KMASK1}, %%DATA64B_L11, %%DATA64B_L10, 8 + vpalignr %%DATA64B_L10{%%KMASK1}, %%DATA64B_L10, %%DATA64B_L9, 8 + vpalignr %%DATA64B_L9{%%KMASK1}, %%DATA64B_L9, %%DATA64B_L8, 8 + vpalignr %%DATA64B_L8{%%KMASK1}, %%DATA64B_L8, %%DATA64B_L11, 8 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2]{%%KMASK2}, %%DATA64B_L8 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 + + pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_3_7_11_15] + kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] + kmovb %%KMASK2, [%%MOV_MASK + %%TMP] + vpalignr %%DATA64B_L15{%%KMASK1}, %%DATA64B_L15, %%DATA64B_L14, 8 + vpalignr %%DATA64B_L14{%%KMASK1}, %%DATA64B_L14, %%DATA64B_L13, 8 + vpalignr %%DATA64B_L13{%%KMASK1}, %%DATA64B_L13, %%DATA64B_L12, 8 + vpalignr %%DATA64B_L12{%%KMASK1}, %%DATA64B_L12, %%DATA64B_L15, 8 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3]{%%KMASK2}, %%DATA64B_L12 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 %endif %endmacro @@ -873,10 +873,10 @@ align 64 %define %%KEY_OFF %6 ; [in] Offset to start writing Keystream %define %%LANE_GROUP %7 ; [immediate] 0, 1, 2 or 3 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP], %%DATA64B_L0 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512], %%DATA64B_L1 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512*2], %%DATA64B_L2 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512*3], %%DATA64B_L3 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP], %%DATA64B_L0 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512], %%DATA64B_L1 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512*2], %%DATA64B_L2 + vmovdqu64 [%%KS + %%KEY_OFF*4 + 64*%%LANE_GROUP + 512*3], %%DATA64B_L3 %endmacro ; @@ -888,10 +888,10 @@ align 64 %define %%ZTMP %3 ; [clobbered] Temporary ZMM register %define %%MASK31 %4 ; [in] ZMM register containing 0x7FFFFFFF's in all dwords - vpaddd %%IN_OUT, %%IN2 - vpsrld %%ZTMP, %%IN_OUT, 31 - vpandq %%IN_OUT, %%MASK31 - vpaddd %%IN_OUT, %%ZTMP + vpaddd %%IN_OUT, %%IN2 + vpsrld %%ZTMP, %%IN_OUT, 31 + vpandq %%IN_OUT, %%MASK31 + vpaddd %%IN_OUT, %%ZTMP %endmacro ; @@ -903,9 +903,9 @@ align 64 %define %%MASK31 %3 ; [in] ZMM register containing 0x7FFFFFFF's in all dwords %define %%N_BITS %4 ; [immediate] Number of bits to rotate for each dword - vpslld %%ZTMP, %%IN_OUT, %%N_BITS - vpsrld %%IN_OUT, %%IN_OUT, (31 - %%N_BITS) - vpternlogq %%IN_OUT, %%ZTMP, %%MASK31, 0xA8 ; (A | B) & C + vpslld %%ZTMP, %%IN_OUT, %%N_BITS + vpsrld %%IN_OUT, %%IN_OUT, (31 - %%N_BITS) + vpternlogq %%IN_OUT, %%ZTMP, %%MASK31, 0xA8 ; (A | B) & C %endmacro ; @@ -915,7 +915,7 @@ align 64 ; If init mode, add W to the calculation above. ; S_16 -> S_15 for next round ; -%macro LFSR_UPDT16 13 +%macro LFSR_UPDT16 12 %define %%STATE %1 ; [in] ZUC state %define %%ROUND_NUM %2 ; [in] Round number %define %%LANE_MASK %3 ; [in] Mask register with lanes to update @@ -927,37 +927,36 @@ align 64 %define %%ZTMP %9 ; [clobbered] Temporary ZMM register %define %%MASK_31 %10 ; [in] Mask_31 %define %%W %11 ; [in/clobbered] In init mode, contains W for all 16 lanes -%define %%KTMP %12 ; [clobbered] Temporary K mask -%define %%MODE %13 ; [constant] "init" / "work" mode +%define %%MODE %12 ; [constant] "init" / "work" mode - vmovdqa64 %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*64] - vmovdqa64 %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*64] - vmovdqa64 %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*64] - vmovdqa64 %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*64] - vmovdqa64 %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*64] + vmovdqa64 %%LFSR_0, [%%STATE + (( 0 + %%ROUND_NUM) % 16)*64] + vmovdqa64 %%LFSR_4, [%%STATE + (( 4 + %%ROUND_NUM) % 16)*64] + vmovdqa64 %%LFSR_10, [%%STATE + ((10 + %%ROUND_NUM) % 16)*64] + vmovdqa64 %%LFSR_13, [%%STATE + ((13 + %%ROUND_NUM) % 16)*64] + vmovdqa64 %%LFSR_15, [%%STATE + ((15 + %%ROUND_NUM) % 16)*64] - ; Calculate LFSR feedback (S_16) + ; Calculate LFSR feedback (S_16) - ; In Init mode, W is added to the S_16 calculation + ; In Init mode, W is added to the S_16 calculation %ifidn %%MODE, init - ADD_MOD31 %%W, %%LFSR_0, %%ZTMP, %%MASK_31 + ADD_MOD31 %%W, %%LFSR_0, %%ZTMP, %%MASK_31 %else - vmovdqa64 %%W, %%LFSR_0 + vmovdqa64 %%W, %%LFSR_0 %endif - ROT_MOD31 %%LFSR_0, %%ZTMP, %%MASK_31, 8 - ADD_MOD31 %%W, %%LFSR_0, %%ZTMP, %%MASK_31 - ROT_MOD31 %%LFSR_4, %%ZTMP, %%MASK_31, 20 - ADD_MOD31 %%W, %%LFSR_4, %%ZTMP, %%MASK_31 - ROT_MOD31 %%LFSR_10, %%ZTMP, %%MASK_31, 21 - ADD_MOD31 %%W, %%LFSR_10, %%ZTMP, %%MASK_31 - ROT_MOD31 %%LFSR_13, %%ZTMP, %%MASK_31, 17 - ADD_MOD31 %%W, %%LFSR_13, %%ZTMP, %%MASK_31 - ROT_MOD31 %%LFSR_15, %%ZTMP, %%MASK_31, 15 - ADD_MOD31 %%W, %%LFSR_15, %%ZTMP, %%MASK_31 - - vmovdqa32 [%%STATE + (( 0 + %%ROUND_NUM) % 16)*64]{%%LANE_MASK}, %%W - - ; LFSR_S16 = (LFSR_S15++) = eax + ROT_MOD31 %%LFSR_0, %%ZTMP, %%MASK_31, 8 + ADD_MOD31 %%W, %%LFSR_0, %%ZTMP, %%MASK_31 + ROT_MOD31 %%LFSR_4, %%ZTMP, %%MASK_31, 20 + ADD_MOD31 %%W, %%LFSR_4, %%ZTMP, %%MASK_31 + ROT_MOD31 %%LFSR_10, %%ZTMP, %%MASK_31, 21 + ADD_MOD31 %%W, %%LFSR_10, %%ZTMP, %%MASK_31 + ROT_MOD31 %%LFSR_13, %%ZTMP, %%MASK_31, 17 + ADD_MOD31 %%W, %%LFSR_13, %%ZTMP, %%MASK_31 + ROT_MOD31 %%LFSR_15, %%ZTMP, %%MASK_31, 15 + ADD_MOD31 %%W, %%LFSR_15, %%ZTMP, %%MASK_31 + + vmovdqa32 [%%STATE + (( 0 + %%ROUND_NUM) % 16)*64]{%%LANE_MASK}, %%W + + ; LFSR_S16 = (LFSR_S15++) = eax %endmacro ; @@ -975,12 +974,12 @@ align 64 %define %%LFSR %3 ;; [out] ZMM register to contain initialized LFSR regs %define %%ZTMP %4 ;; [clobbered] ZMM temporary register - vbroadcasti64x2 %%LFSR, [%%KEY] - vbroadcasti64x2 %%ZTMP, [%%IV] - vpshufb %%LFSR, [rel shuf_mask_key] - vpsrld %%LFSR, 1 - vpshufb %%ZTMP, [rel shuf_mask_iv] - vpternlogq %%LFSR, %%ZTMP, [rel EK_d64], 0xFE ; A OR B OR C + vbroadcasti64x2 %%LFSR, [%%KEY] + vbroadcasti64x2 %%ZTMP, [%%IV] + vpshufb %%LFSR, [rel shuf_mask_key] + vpsrld %%LFSR, 1 + vpshufb %%ZTMP, [rel shuf_mask_iv] + vpternlogq %%LFSR, %%ZTMP, [rel EK_d64], 0xFE ; A OR B OR C %endmacro @@ -1030,134 +1029,149 @@ align 64 %macro INIT_16_AVX512 1 %define %%KEY_SIZE %1 ; [in] Key size (128 or 256) -%ifdef LINUX - %define pKe rdi - %define pIv rsi - %define pState rdx - %define lane_mask ecx -%else - %define pKe rcx - %define pIv rdx - %define pState r8 - %define lane_mask r9d -%endif -%define tag_sz r10d ; Only used in ZUC-256 (caller written in assembly, so using a hardcoded register) -%define tag_sz_q r10 - -%define %%X0 zmm10 -%define %%X1 zmm11 -%define %%X2 zmm12 -%define %%W zmm13 -%define %%R1 zmm14 -%define %%R2 zmm15 - - FUNC_SAVE - - mov rax, pState - - kmovw k2, lane_mask +%define pKe arg1 +%define pIv arg2 +%define pState arg3 +%define lane_mask arg4 + +%define tag_sz r10d ; Only used in ZUC-256 (caller written in assembly, so using a hardcoded register) +%define tag_sz_q r10 + +%define %%TMP r14 +%define %%TMP2 r15 + +%define %%ZTMP1 zmm0 +%define %%ZTMP2 zmm1 +%define %%ZTMP3 zmm2 +%define %%ZTMP4 zmm3 +%define %%ZTMP5 zmm4 +%define %%ZTMP6 zmm5 +%define %%ZTMP7 zmm6 +%define %%ZTMP8 zmm7 +%define %%ZTMP9 zmm8 +%define %%ZTMP10 zmm9 +%define %%ZTMP11 zmm10 +%define %%ZTMP12 zmm11 +%define %%ZTMP13 zmm12 +%define %%ZTMP14 zmm13 +%define %%ZTMP15 zmm14 +%define %%ZTMP16 zmm15 + +%define %%LFSR1 zmm16 +%define %%LFSR2 zmm17 +%define %%LFSR3 zmm18 +%define %%LFSR4 zmm19 +%define %%LFSR5 zmm20 +%define %%LFSR6 zmm21 +%define %%LFSR7 zmm22 +%define %%LFSR8 zmm23 +%define %%LFSR9 zmm24 +%define %%LFSR10 zmm25 +%define %%LFSR11 zmm26 +%define %%LFSR12 zmm27 +%define %%LFSR13 zmm28 +%define %%LFSR14 zmm29 +%define %%LFSR15 zmm30 +%define %%LFSR16 zmm31 + +%define %%X0 %%ZTMP10 +%define %%X1 %%ZTMP11 +%define %%X2 %%ZTMP12 +%define %%W %%ZTMP13 +%define %%R1 %%ZTMP14 +%define %%R2 %%ZTMP15 +%define %%MASK31 %%ZTMP16 + +%define %%BLEND_KMASK k1 ; Mask to blend LFSRs 14&15 +%define %%INIT_LANE_KMASK k2 ; Mask containing lanes to initialize +%define %%SHIFT_KMASK k3 ; Mask to shift 4 bytes only in the 15th dword +%define %%IV_KMASK k4 ; Mask to read 10 bytes of IV + + kmovw %%INIT_LANE_KMASK, DWORD(lane_mask) %if %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea r13, [rel EK256_d64] - bsf tag_sz, tag_sz - dec tag_sz - shl tag_sz, 6 - add r13, tag_sz_q - mov r11, 0x4000 ; Mask to shift 4 bits only in the 15th dword - kmovq k1, r11 - mov r11, 0x3ff ; Mask to read 10 bytes of IV - kmovq k3, r11 + ; Get pointer to constants (depending on tag size, this will point at + ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) + lea %%TMP, [rel EK256_d64] + bsf tag_sz, tag_sz + dec tag_sz + shl tag_sz, 6 + add tag_sz_q, %%TMP + mov %%TMP2, 0x4000 + kmovq %%SHIFT_KMASK, %%TMP2 + mov %%TMP2, 0x3ff + kmovq %%IV_KMASK, %%TMP2 %endif - ; Set LFSR registers for Packet 1 - mov r9, [pKe] ; Load Key 1 pointer - lea r10, [pIv] ; Load IV 1 pointer - -%if %%KEY_SIZE == 128 - INIT_LFSR_128 r9, r10, zmm0, zmm1 -%else - INIT_LFSR_256 r9, r10, zmm0, zmm3, zmm5, zmm7, zmm9, zmm11, r13, k1, k3 -%endif - ; Set LFSR registers for Packets 2-15 -%assign idx 1 -%assign reg_lfsr 2 -%assign reg_tmp 3 -%rep 14 - mov r9, [pKe + 8*idx] ; Load Key N pointer - lea r10, [pIv + 32*idx] ; Load IV N pointer + ; Set LFSR registers for Packets 1-16 +%assign %%IDX 0 +%assign %%LFSR_IDX 1 +%rep 16 + mov %%TMP, [pKe + 8*%%IDX] ; Load Key N pointer + lea %%TMP2, [pIv + 32*%%IDX] ; Load IV N pointer %if %%KEY_SIZE == 128 - INIT_LFSR_128 r9, r10, APPEND(zmm, reg_lfsr), APPEND(zmm, reg_tmp) + INIT_LFSR_128 %%TMP, %%TMP2, APPEND(%%LFSR, %%LFSR_IDX), %%ZTMP1 %else - INIT_LFSR_256 r9, r10, APPEND(zmm, reg_lfsr), zmm3, zmm5, zmm7, zmm9, zmm11, r13, k1, k3 + INIT_LFSR_256 %%TMP, %%TMP2, APPEND(%%LFSR, %%LFSR_IDX), %%ZTMP1, \ + %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, tag_sz_q, \ + %%SHIFT_KMASK, %%IV_KMASK %endif -%assign idx (idx + 1) -%assign reg_lfsr (reg_lfsr + 2) -%assign reg_tmp (reg_tmp + 2) +%assign %%IDX (%%IDX + 1) +%assign %%LFSR_IDX (%%LFSR_IDX + 1) %endrep - ; Set LFSR registers for Packet 16 - mov r9, [pKe + 8*15] ; Load Key 16 pointer - lea r10, [pIv + 32*15] ; Load IV 16 pointer -%if %%KEY_SIZE == 128 - INIT_LFSR_128 r9, r10, zmm30, zmm31 -%else - INIT_LFSR_256 r9, r10, zmm30, zmm3, zmm5, zmm7, zmm9, zmm11, r13, k1, k3 -%endif - ; Store LFSR registers in memory (reordering first, so all S0 regs - ; are together, then all S1 regs... until S15) - TRANSPOSE16_U32 zmm0, zmm2, zmm4, zmm6, zmm8, zmm10, zmm12, zmm14, \ - zmm16, zmm18, zmm20, zmm22, zmm24, zmm26, zmm28, zmm30, \ - zmm1, zmm3, zmm5, zmm7, zmm9, zmm11, zmm13, zmm15, \ - zmm17, zmm19, zmm21, zmm23, zmm25, zmm27 + ; Store LFSR registers in memory (reordering first, so all S0 regs + ; are together, then all S1 regs... until S15) + TRANSPOSE16_U32 %%LFSR1, %%LFSR2, %%LFSR3, %%LFSR4, %%LFSR5, %%LFSR6, %%LFSR7, %%LFSR8, \ + %%LFSR9, %%LFSR10, %%LFSR11, %%LFSR12, %%LFSR13, %%LFSR14, %%LFSR15, %%LFSR16, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12, %%ZTMP13, %%ZTMP14 -%assign i 0 -%assign j 0 +%assign %%IDX 0 +%assign %%LFSR_IDX 1 %rep 16 - vmovdqa32 [pState + 64*i]{k2}, APPEND(zmm, j) -%assign i (i+1) -%assign j (j+2) + vmovdqa32 [pState + 64*%%IDX]{%%INIT_LANE_KMASK}, APPEND(%%LFSR, %%LFSR_IDX) +%assign %%IDX (%%IDX+1) +%assign %%LFSR_IDX (%%LFSR_IDX+1) %endrep - ; Load read-only registers - vmovdqa64 zmm0, [rel mask31] - mov edx, 0xAAAAAAAA - kmovd k1, edx + ; Load read-only registers + vmovdqa64 %%MASK31, [rel mask31] + mov DWORD(%%TMP), 0xAAAAAAAA + kmovd %%BLEND_KMASK, DWORD(%%TMP) - ; Zero out R1, R2 - vpxorq %%R1, %%R1 - vpxorq %%R2, %%R2 + ; Zero out R1, R2 + vpxorq %%R1, %%R1 + vpxorq %%R2, %%R2 ; Shift LFSR 32-times, update state variables -%assign N 0 +%assign %%N 0 %rep 32 - BITS_REORG16 rax, N, k2, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, \ - zmm7, zmm8, zmm9, k1, %%X0, %%X1, %%X2 - NONLIN_FUN16 rax, k2, %%X0, %%X1, %%X2, %%R1, %%R2, \ - zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, %%W - vpsrld %%W,1 ; Shift out LSB of W - - LFSR_UPDT16 rax, N, k2, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, zmm0, %%W, k7, init ; W used in LFSR update -%assign N N+1 + BITS_REORG16 pState, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2 + NONLIN_FUN16 pState, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W + vpsrld %%W, 1 ; Shift out LSB of W + + LFSR_UPDT16 pState, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%W, init ; W used in LFSR update +%assign %%N (%%N + 1) %endrep - ; And once more, initial round from keygen phase = 33 times - BITS_REORG16 rax, 0, k2, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, \ - zmm8, zmm9, k1, %%X0, %%X1, %%X2 - NONLIN_FUN16 rax, k2, %%X0, %%X1, %%X2, %%R1, %%R2, \ - zmm1, zmm2, zmm3, zmm4, zmm5, zmm6 + ; And once more, initial round from keygen phase = 33 times + BITS_REORG16 pState, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2 + NONLIN_FUN16 pState, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 - LFSR_UPDT16 rax, 0, k2, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, zmm0, %%W, k7, work + LFSR_UPDT16 pState, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%W, work - ; Update R1, R2 - vmovdqa32 [rax + OFS_R1]{k2}, %%R1 - vmovdqa32 [rax + OFS_R2]{k2}, %%R2 - FUNC_RESTORE + ; Update R1, R2 + vmovdqa32 [pState + OFS_R1]{%%INIT_LANE_KMASK}, %%R1 + vmovdqa32 [pState + OFS_R2]{%%INIT_LANE_KMASK}, %%R2 -%endmacro +%endmacro ; INIT_16_AVX512 ;; ;; void asm_ZucInitialization_16_avx512(ZucKey16_t *pKeys, ZucIv16_t *pIvs, @@ -1165,10 +1179,15 @@ align 64 ;; MKGLOBAL(ZUC128_INIT,function,internal) ZUC128_INIT: - endbranch64 - INIT_16_AVX512 128 + endbranch64 - ret + FUNC_SAVE + + INIT_16_AVX512 128 + + FUNC_RESTORE + + ret ;; ;; void asm_Zuc256Initialization_16_avx512(ZucKey16_t *pKeys, ZucIv16_t *pIvs, @@ -1176,10 +1195,15 @@ ZUC128_INIT: ;; MKGLOBAL(ZUC256_INIT,function,internal) ZUC256_INIT: - endbranch64 - INIT_16_AVX512 256 + endbranch64 - ret + FUNC_SAVE + + INIT_16_AVX512 256 + + FUNC_RESTORE + + ret ; ; Generate N*4 bytes of keystream @@ -1191,124 +1215,166 @@ ZUC256_INIT: %define %%KEY_OFF %3 ; [in] Offset to start writing Keystream %define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate all keystream words - %define pState arg1 - %define pKS arg2 - -%define %%X0 zmm10 -%define %%X1 zmm11 -%define %%X2 zmm12 -%define %%W zmm13 -%define %%R1 zmm14 -%define %%R2 zmm15 +%define pState arg1 +%define pKS arg2 + +%define %%TMP1 r10 +%define %%TMP2 r12 +%define %%TMP3 r13 + +%define %%ZTMP1 zmm0 +%define %%ZTMP2 zmm1 +%define %%ZTMP3 zmm2 +%define %%ZTMP4 zmm3 +%define %%ZTMP5 zmm4 +%define %%ZTMP6 zmm5 +%define %%ZTMP7 zmm6 +%define %%ZTMP8 zmm7 +%define %%ZTMP9 zmm8 +%define %%ZTMP10 zmm9 +%define %%ZTMP11 zmm10 +%define %%ZTMP12 zmm11 +%define %%ZTMP13 zmm12 +%define %%ZTMP14 zmm13 +%define %%ZTMP15 zmm14 +%define %%ZTMP16 zmm15 + +%define %%KSTR1 zmm16 +%define %%KSTR2 zmm17 +%define %%KSTR3 zmm18 +%define %%KSTR4 zmm19 +%define %%KSTR5 zmm20 +%define %%KSTR6 zmm21 +%define %%KSTR7 zmm22 +%define %%KSTR8 zmm23 +%define %%KSTR9 zmm24 +%define %%KSTR10 zmm25 +%define %%KSTR11 zmm26 +%define %%KSTR12 zmm27 +%define %%KSTR13 zmm28 +%define %%KSTR14 zmm29 +%define %%KSTR15 zmm30 +%define %%KSTR16 zmm31 + +%define %%X0 %%ZTMP10 +%define %%X1 %%ZTMP11 +%define %%X2 %%ZTMP12 +%define %%W %%ZTMP13 +%define %%R1 %%ZTMP14 +%define %%R2 %%ZTMP15 +%define %%MASK31 %%ZTMP16 + +%define %%BLEND_KMASK k1 ; Mask to blend LFSRs 14&15 +%define %%FULL_LANE_KMASK k2 ; Mask with lanes to generate all keystream words +%define %%ALL_KMASK k3 ; Mask with all 1's +%define %%SKIP_LANE_KMASK k4 ; Mask with lanes to skip some keystream words +%define %%TMP_KMASK1 k5 +%define %%TMP_KMASK2 k6 - FUNC_SAVE - - ; Load read-only registers - vmovdqa64 zmm0, [rel mask31] - mov r10d, 0xAAAAAAAA - kmovd k1, r10d + ; Load read-only registers + vmovdqa64 %%MASK31, [rel mask31] + mov DWORD(%%TMP1), 0xAAAAAAAA + kmovd %%BLEND_KMASK, DWORD(%%TMP1) %if (%0 == 4) - kmovd k2, DWORD(%%LANE_MASK) - knotd k4, k2 - mov r10d, 0x0000FFFF - kmovd k3, r10d + kmovd %%FULL_LANE_KMASK, DWORD(%%LANE_MASK) + knotd %%SKIP_LANE_KMASK, %%FULL_LANE_KMASK + mov DWORD(%%TMP1), 0x0000FFFF + kmovd %%ALL_KMASK, DWORD(%%TMP1) %else - mov r10d, 0x0000FFFF - kmovd k2, r10d - kmovd k3, k2 + mov DWORD(%%TMP1), 0x0000FFFF + kmovd %%FULL_LANE_KMASK, DWORD(%%TMP1) + kmovd %%ALL_KMASK, %%FULL_LANE_KMASK %endif - ; Read R1/R2 - vmovdqa32 %%R1, [pState + OFS_R1] - vmovdqa32 %%R2, [pState + OFS_R2] -; Store all 4 bytes of keystream in a single 64-byte buffer + ; Read R1/R2 + vmovdqa32 %%R1, [pState + OFS_R1] + vmovdqa32 %%R2, [pState + OFS_R2] + + ; Store all 4 bytes of keystream in a single 64-byte buffer %if (%%NUM_ROUNDS == 1) - BITS_REORG16 pState, 1, k2, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, \ - zmm7, zmm8, zmm9, k1, %%X0, %%X1, %%X2, zmm16 - NONLIN_FUN16 pState, k2, %%X0, %%X1, %%X2, %%R1, %%R2, \ - zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7 - ; OFS_X3 XOR W (zmm7) - vpxorq zmm16, zmm7 - LFSR_UPDT16 pState, 1, k2, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, zmm0, zmm7, k7, work - vmovdqa32 [pState + OFS_R1]{k2}, %%R1 - vmovdqa32 [pState + OFS_R2]{k2}, %%R2 + BITS_REORG16 pState, 1, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, %%KSTR1 + NONLIN_FUN16 pState, %%FULL_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W + ; OFS_X3 XOR W + vpxorq %%KSTR1, %%W + LFSR_UPDT16 pState, 1, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%ZTMP7, work + vmovdqa32 [pState + OFS_R1]{%%FULL_LANE_KMASK}, %%R1 + vmovdqa32 [pState + OFS_R2]{%%FULL_LANE_KMASK}, %%R2 %else ;; %%NUM_ROUNDS != 1 - ; Generate N*4B of keystream in N rounds - ; Generate first bytes of KS for all lanes -%assign N 1 -%assign idx 16 + ; Generate N*4B of keystream in N rounds + ; Generate first bytes of KS for all lanes +%assign %%N 1 +%assign %%IDX 1 %rep (%%NUM_ROUNDS-2) - BITS_REORG16 pState, N, k3, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, \ - zmm7, zmm8, zmm9, k1, %%X0, %%X1, %%X2, APPEND(zmm, idx) - NONLIN_FUN16 pState, k3, %%X0, %%X1, %%X2, %%R1, %%R2, \ - zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7 - ; OFS_X3 XOR W (zmm7) - vpxorq APPEND(zmm, idx), zmm7 - LFSR_UPDT16 pState, N, k3, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, zmm0, zmm7, k7, work -%assign N N+1 -%assign idx (idx + 1) + BITS_REORG16 pState, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%IDX) + NONLIN_FUN16 pState, %%ALL_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W + ; OFS_X3 XOR W + vpxorq APPEND(%%KSTR, %%IDX), %%W + LFSR_UPDT16 pState, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%ZTMP7, work +%assign %%N %%N+1 +%assign %%IDX (%%IDX + 1) %endrep %if (%%NUM_ROUNDS > 2) - vmovdqa32 [pState + OFS_R1]{k3}, %%R1 - vmovdqa32 [pState + OFS_R2]{k3}, %%R2 + vmovdqa32 [pState + OFS_R1]{%%ALL_KMASK}, %%R1 + vmovdqa32 [pState + OFS_R2]{%%ALL_KMASK}, %%R2 %endif - ; Generate rest of the KS bytes (last 8 bytes) for selected lanes + ; Generate rest of the KS bytes (last 8 bytes) for selected lanes %rep 2 - BITS_REORG16 pState, N, k2, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, \ - zmm7, zmm8, zmm9, k1, %%X0, %%X1, %%X2, APPEND(zmm, idx) - NONLIN_FUN16 pState, k2, %%X0, %%X1, %%X2, %%R1, %%R2, \ - zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7 - ; OFS_X3 XOR W (zmm7) - vpxorq APPEND(zmm, idx), zmm7 - LFSR_UPDT16 pState, N, k2, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, zmm0, zmm7, k7, work -%assign N N+1 -%assign idx (idx + 1) + BITS_REORG16 pState, %%N, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%IDX) + NONLIN_FUN16 pState, %%FULL_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W + ; OFS_X3 XOR W + vpxorq APPEND(%%KSTR, %%IDX), %%W + LFSR_UPDT16 pState, %%N, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%ZTMP7, work +%assign %%N %%N+1 +%assign %%IDX (%%IDX + 1) %endrep - vmovdqa32 [pState + OFS_R1]{k2}, %%R1 - vmovdqa32 [pState + OFS_R2]{k2}, %%R2 + vmovdqa32 [pState + OFS_R1]{%%FULL_LANE_KMASK}, %%R1 + vmovdqa32 [pState + OFS_R2]{%%FULL_LANE_KMASK}, %%R2 %endif ;; (%%NUM_ROUNDS == 1) %if (%%STORE_SINGLE == 1) - vmovdqa32 [pKS]{k2}, zmm16 + vmovdqa32 [pKS]{%%FULL_LANE_KMASK}, zmm16 %else - ; ZMM16-31 contain the keystreams for each round - ; Perform a 32-bit 16x16 transpose to have up to 64 bytes - ; (NUM_ROUNDS * 4B) of each lane in a different register - TRANSPOSE16_U32_INTERLEAVED zmm16, zmm17, zmm18, zmm19, zmm20, zmm21, zmm22, zmm23, \ - zmm24, zmm25, zmm26, zmm27, zmm28, zmm29, zmm30, zmm31, \ - zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7, \ - zmm8, zmm9 + ; Perform a 32-bit 16x16 transpose to have up to 64 bytes + ; (NUM_ROUNDS * 4B) of each lane in a different register + TRANSPOSE16_U32_INTERLEAVED %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%KSTR5, %%KSTR6, %%KSTR7, %%KSTR8, \ + %%KSTR9, %%KSTR10, %%KSTR11, %%KSTR12, %%KSTR13, %%KSTR14, %%KSTR15, %%KSTR16, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%ZTMP9, %%ZTMP10 %if (%0 == 4) - lea r12, [rel alignr_mask] - lea r13, [rel mov_mask] - STORE_KSTR16 pKS, zmm6, zmm4, zmm28, zmm16, zmm7, zmm5, zmm29, zmm17, \ - zmm2, zmm0, zmm30, zmm18, zmm3, zmm1, zmm31, zmm19, %%KEY_OFF, \ - %%LANE_MASK, r12, r13, r10, k3, k5 + lea %%TMP1, [rel alignr_mask] + lea %%TMP2, [rel mov_mask] + STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ + %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF, \ + %%LANE_MASK, %%TMP1, %%TMP2, %%TMP3, %%TMP_KMASK1,%%TMP_KMASK2 %else - STORE_KSTR16 pKS, zmm6, zmm4, zmm28, zmm16, zmm7, zmm5, zmm29, zmm17, \ - zmm2, zmm0, zmm30, zmm18, zmm3, zmm1, zmm31, zmm19, %%KEY_OFF + STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ + %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF %endif %endif ;; %%STORE_SINGLE == 1 - ; Reorder LFSR registers + ; Reorder LFSR registers %if (%0 == 4) - REORDER_LFSR pState, %%NUM_ROUNDS, k2 + REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK %if (%%NUM_ROUNDS >= 2) - REORDER_LFSR pState, (%%NUM_ROUNDS - 2), k4 ; 2 less rounds for "old" buffers + REORDER_LFSR pState, (%%NUM_ROUNDS - 2), %%SKIP_LANE_KMASK ; 2 less rounds for "old" buffers %endif %else - REORDER_LFSR pState, %%NUM_ROUNDS, k2 + REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK %endif - FUNC_RESTORE - -%endmacro +%endmacro ; KEYGEN_16_AVX512 ;; ;; Reverse bits of each byte of a XMM register @@ -1394,6 +1460,7 @@ ZUC256_INIT: %define %%LEN %5 ; [in] Pointer to array of remaining length to digest %define %%NROUNDS %6 ; [in/clobbered] Number of rounds of 64 bytes of data to digest +%define %%TMP r12 %define %%DATA_ADDR0 rbx %define %%DATA_ADDR1 r12 %define %%DATA_ADDR2 r13 @@ -1451,21 +1518,28 @@ ZUC256_INIT: %define %%KS_2 zmm26 %define %%KS_3 zmm27 +%define %%BLEND_KMASK k1 ; Mask to blend LFSRs 14&15 +%define %%ALL_KMASK k2 ; Mask with all 1's +%define %%SHUF_DATA_KMASK k3 ; Mask to shuffle data +%define %%PERM_DIGEST_KMASK1 k4 ; Permutation mask for digests +%define %%PERM_DIGEST_KMASK2 k5 ; Permulation mask for digests +%define %%TMP_KMASK k6 + xor %%OFFSET, %%OFFSET - mov r12d, 0xAAAAAAAA - kmovd k1, r12d + mov DWORD(%%TMP), 0xAAAAAAAA + kmovd %%BLEND_KMASK, DWORD(%%TMP) - mov r12d, 0x0000FFFF - kmovd k2, r12d + mov DWORD(%%TMP), 0x0000FFFF + kmovd %%ALL_KMASK, DWORD(%%TMP) - mov r12d, 0x55555555 - kmovd k3, r12d + mov DWORD(%%TMP), 0x55555555 + kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP) - mov r12d, 0x3333 - kmovd k4, r12d - mov r12d, 0xCCCC - kmovd k5, r12d + mov DWORD(%%TMP), 0x3333 + kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) + mov DWORD(%%TMP), 0xCCCC + kmovd %%PERM_DIGEST_KMASK2, DWORD(%%TMP) vpxorq %%DIGEST_0, %%DIGEST_0 vpxorq %%DIGEST_1, %%DIGEST_1 @@ -1476,9 +1550,9 @@ ZUC256_INIT: vmovdqa64 %%MASK31, [rel mask31] %if USE_GFNI_VAES_VPCLMUL == 0 - vmovdqa64 %%REV_TABLE_L, [bit_reverse_table_l] - vmovdqa64 %%REV_TABLE_H, [bit_reverse_table_h] - vmovdqa64 %%REV_AND_TABLE, [bit_reverse_and_table] + vmovdqa64 %%REV_TABLE_L, [rel bit_reverse_table_l] + vmovdqa64 %%REV_TABLE_H, [rel bit_reverse_table_h] + vmovdqa64 %%REV_AND_TABLE, [rel bit_reverse_and_table] %endif ; Read R1/R2 @@ -1506,14 +1580,14 @@ ZUC256_INIT: %assign %%idx 0 %rep 4 - BITS_REORG16 %%STATE, %%N, k2, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ - %%ZTMP7, %%ZTMP8, %%ZTMP9, k1, %%X0, %%X1, %%X2, APPEND(%%KS_, %%idx) - NONLIN_FUN16 %%STATE, k2, %%X0, %%X1, %%X2, %%R1, %%R2, \ + BITS_REORG16 %%STATE, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KS_, %%idx) + NONLIN_FUN16 %%STATE, %%ALL_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7 ; OFS_X3 XOR W (%%ZTMP7) vpxorq APPEND(%%KS_, %%idx), %%ZTMP7 - LFSR_UPDT16 %%STATE, %%N, k2, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ - %%ZTMP6, %%MASK31, %%ZTMP7, k7, work + LFSR_UPDT16 %%STATE, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%ZTMP7, work ;; Transpose and store KS every 16 bytes %if %%idx == 3 @@ -1542,7 +1616,7 @@ ZUC256_INIT: vgf2p8affineqb %%ZTMP7, %%ZTMP1, [rel bit_reverse_table], 0x00 ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA %%ZTMP7, %%ZKS_L, %%ZKS_H, APPEND(%%DIGEST_, %%LANE_GROUP), k3, \ + DIGEST_DATA %%ZTMP7, %%ZKS_L, %%ZKS_H, APPEND(%%DIGEST_, %%LANE_GROUP), %%SHUF_DATA_KMASK, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 %else ; USE_GFNI_VAES_VPCLMUL == 1 @@ -1566,7 +1640,7 @@ ZUC256_INIT: %%REV_AND_TABLE, %%XTMP2, %%XTMP3 ; Digest 16 bytes of data with 24 bytes of KS, for one buffer - DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, APPEND(%%XDIGEST_, %%J), k3, \ + DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, APPEND(%%XDIGEST_, %%J), %%SHUF_DATA_KMASK, \ %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 ; Once all 64 bytes of data have been digested, insert them in temporary ZMM register @@ -1614,8 +1688,8 @@ ZUC256_INIT: ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. - vpermt2d %%DIGEST_0{k4}{z}, %%ZTMP2, %%DIGEST_1 - vpermt2d %%DIGEST_2{k5}{z}, %%ZTMP3, %%DIGEST_3 + vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 + vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C vmovdqu64 [%%T], %%ZTMP1 @@ -1634,10 +1708,10 @@ ZUC256_INIT: ; Update array of lengths (if lane is valid, so length < UINT16_MAX) vmovdqa64 YWORD(%%ZTMP2), [%%LEN] - vpcmpw k1, YWORD(%%ZTMP2), [rel all_ffs], 4 ; k1 -> valid lanes + vpcmpw %%TMP_KMASK, YWORD(%%ZTMP2), [rel all_ffs], 4 ; valid lanes shl %%OFFSET, 3 ; Convert to bits vpbroadcastw YWORD(%%ZTMP1), DWORD(%%OFFSET) - vpsubw YWORD(%%ZTMP2){k1}, YWORD(%%ZTMP1) + vpsubw YWORD(%%ZTMP2){%%TMP_KMASK}, YWORD(%%ZTMP1) vmovdqa64 [%%LEN], YWORD(%%ZTMP2) %endmacro @@ -1648,10 +1722,15 @@ ZUC256_INIT: ;; MKGLOBAL(ZUC_KEYGEN64B_16,function,internal) ZUC_KEYGEN64B_16: - endbranch64 - KEYGEN_16_AVX512 16, 0, arg3 + endbranch64 + + FUNC_SAVE + + KEYGEN_16_AVX512 16, 0, arg3 - ret + FUNC_RESTORE + + ret ;; ;; void asm_Eia3_Nx64B_AVX512_16(ZucState16_t *pState, ;; uint32_t *pKeyStr, @@ -1672,20 +1751,20 @@ ZUC_EIA3_N64B: %define LEN r10 %define NROUNDS r11 %endif - endbranch64 + endbranch64 %ifndef LINUX - mov LEN, arg5 - mov NROUNDS, arg6 + mov LEN, arg5 + mov NROUNDS, arg6 %endif - FUNC_SAVE + FUNC_SAVE - ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS + ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS - FUNC_RESTORE + FUNC_RESTORE - ret + ret ; ;; void asm_ZucGenKeystream64B_16_skip8_avx512(state16_t *pSta, u32* pKeyStr[16], @@ -1694,10 +1773,15 @@ ZUC_EIA3_N64B: ;; MKGLOBAL(ZUC_KEYGEN64B_SKIP8_16,function,internal) ZUC_KEYGEN64B_SKIP8_16: - endbranch64 - KEYGEN_16_AVX512 16, 0, arg3, arg4 + endbranch64 + + FUNC_SAVE - ret + KEYGEN_16_AVX512 16, 0, arg3, arg4 + + FUNC_RESTORE + + ret ;; ;; void asm_ZucGenKeystream8B_16_avx512(state16_t *pSta, u32* pKeyStr[16], @@ -1705,10 +1789,15 @@ ZUC_KEYGEN64B_SKIP8_16: ;; MKGLOBAL(ZUC_KEYGEN8B_16,function,internal) ZUC_KEYGEN8B_16: - endbranch64 - KEYGEN_16_AVX512 2, 0, arg3 + endbranch64 + + FUNC_SAVE + + KEYGEN_16_AVX512 2, 0, arg3 + + FUNC_RESTORE - ret + ret ;; ;; void asm_ZucGenKeystream4B_16_avx512(state16_t *pSta, u32 pKeyStr[16], @@ -1716,67 +1805,72 @@ ZUC_KEYGEN8B_16: ;; MKGLOBAL(ZUC_KEYGEN4B_16,function,internal) ZUC_KEYGEN4B_16: - endbranch64 - KEYGEN_16_AVX512 1, 1, 0, arg3 + endbranch64 + + FUNC_SAVE - ret + KEYGEN_16_AVX512 1, 1, 0, arg3 + + FUNC_RESTORE + + ret %macro KEYGEN_VAR_16_AVX512 2-3 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds (GP dowrd register) %define %%KEY_OFF %2 ; [in] Offset to start writing Keystream %define %%LANE_MASK %3 ; [in] Lane mask with lanes to generate full keystream (rest 2 words less) - cmp %%NUM_ROUNDS, 16 - je %%_num_rounds_is_16 - cmp %%NUM_ROUNDS, 8 - je %%_num_rounds_is_8 - jb %%_rounds_is_1_7 + cmp %%NUM_ROUNDS, 16 + je %%_num_rounds_is_16 + cmp %%NUM_ROUNDS, 8 + je %%_num_rounds_is_8 + jb %%_rounds_is_1_7 - ; Final blocks 9-16 - cmp %%NUM_ROUNDS, 12 - je %%_num_rounds_is_12 - jb %%_rounds_is_9_11 + ; Final blocks 9-16 + cmp %%NUM_ROUNDS, 12 + je %%_num_rounds_is_12 + jb %%_rounds_is_9_11 - ; Final blocks 13-15 - cmp %%NUM_ROUNDS, 14 - je %%_num_rounds_is_14 - ja %%_num_rounds_is_15 - jb %%_num_rounds_is_13 + ; Final blocks 13-15 + cmp %%NUM_ROUNDS, 14 + je %%_num_rounds_is_14 + ja %%_num_rounds_is_15 + jb %%_num_rounds_is_13 %%_rounds_is_9_11: - cmp %%NUM_ROUNDS, 10 - je %%_num_rounds_is_10 - ja %%_num_rounds_is_11 - jb %%_num_rounds_is_9 + cmp %%NUM_ROUNDS, 10 + je %%_num_rounds_is_10 + ja %%_num_rounds_is_11 + jb %%_num_rounds_is_9 %%_rounds_is_1_7: - cmp %%NUM_ROUNDS, 4 - je %%_num_rounds_is_4 - jb %%_rounds_is_1_3 + cmp %%NUM_ROUNDS, 4 + je %%_num_rounds_is_4 + jb %%_rounds_is_1_3 - ; Final blocks 5-7 - cmp %%NUM_ROUNDS, 6 - je %%_num_rounds_is_6 - ja %%_num_rounds_is_7 - jb %%_num_rounds_is_5 + ; Final blocks 5-7 + cmp %%NUM_ROUNDS, 6 + je %%_num_rounds_is_6 + ja %%_num_rounds_is_7 + jb %%_num_rounds_is_5 %%_rounds_is_1_3: - cmp %%NUM_ROUNDS, 2 - je %%_num_rounds_is_2 - ja %%_num_rounds_is_3 + cmp %%NUM_ROUNDS, 2 + je %%_num_rounds_is_2 + ja %%_num_rounds_is_3 - ; Rounds = 1 if fall-through -%assign I 1 + ; Rounds = 1 if fall-through +%assign %%I 1 %rep 16 -APPEND(%%_num_rounds_is_,I): +APPEND(%%_num_rounds_is_,%%I): %if (%0 == 3) - KEYGEN_16_AVX512 I, 0, %%KEY_OFF, %%LANE_MASK + KEYGEN_16_AVX512 %%I, 0, %%KEY_OFF, %%LANE_MASK %else - KEYGEN_16_AVX512 I, 0, %%KEY_OFF + KEYGEN_16_AVX512 %%I, 0, %%KEY_OFF %endif - jmp %%_done + jmp %%_done -%assign I (I + 1) +%assign %%I (%%I + 1) %endrep %%_done: @@ -1789,11 +1883,15 @@ APPEND(%%_num_rounds_is_,I): ;; MKGLOBAL(ZUC_KEYGEN_16,function,internal) ZUC_KEYGEN_16: - endbranch64 + endbranch64 - KEYGEN_VAR_16_AVX512 arg4, arg3 + FUNC_SAVE + + KEYGEN_VAR_16_AVX512 arg4, arg3 - ret + FUNC_RESTORE + + ret ;; ;; void asm_ZucGenKeystream_16_skip8_avx512(state16_t *pSta, u32* pKeyStr[16], @@ -1803,17 +1901,17 @@ ZUC_KEYGEN_16: ;; MKGLOBAL(ZUC_KEYGEN_SKIP8_16,function,internal) ZUC_KEYGEN_SKIP8_16: -%ifdef LINUX - %define arg5 r8d -%else - %define arg5 [rsp + 40] -%endif - endbranch64 + endbranch64 + + mov r10, arg5 - mov r10d, arg5 - KEYGEN_VAR_16_AVX512 r10d, arg3, arg4 + FUNC_SAVE - ret + KEYGEN_VAR_16_AVX512 r10d, arg3, arg4 + + FUNC_RESTORE + + ret ;; ;; Encrypts up to 64 bytes of data @@ -1856,7 +1954,7 @@ ZUC_KEYGEN_SKIP8_16: ; Shuffle bytes within KS words to XOR with plaintext later vpshufb APPEND(zmm, idx), [rel swap_mask] LFSR_UPDT16 rax, N, %%LANE_MASK, zmm1, zmm2, zmm3, zmm4, zmm5, \ - zmm6, %%MASK_31, zmm7, k7, work + zmm6, %%MASK_31, zmm7, work %assign N (N + 1) %assign idx (idx + 1) %endrep @@ -1929,19 +2027,10 @@ ZUC_KEYGEN_SKIP8_16: MKGLOBAL(CIPHER_16,function,internal) CIPHER_16: -%ifdef LINUX - %define pState rdi - %define pIn rsi - %define pOut rdx - %define lengths rcx - %define arg5 r8 -%else - %define pState rcx - %define pIn rdx - %define pOut r8 - %define lengths r9 - %define arg5 [rsp + 40] -%endif +%define pState arg1 +%define pIn arg2 +%define pOut arg3 +%define lengths arg4 %define min_length r10 %define buf_idx r11 @@ -2112,18 +2201,10 @@ _no_final_rounds: align 64 MKGLOBAL(ZUC_ROUND64B_16,function,internal) ZUC_ROUND64B_16: - endbranch64 -%ifdef LINUX - %define T rdi - %define KS rsi - %define DATA rdx - %define LEN rcx -%else - %define T rcx - %define KS rdx - %define DATA r8 - %define LEN r9 -%endif +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define LEN arg4 %if USE_GFNI_VAES_VPCLMUL == 1 %define DATA_ADDR0 rbx @@ -2167,13 +2248,15 @@ ZUC_ROUND64B_16: %define YTMP1 YWORD(ZTMP1) + endbranch64 + FUNC_SAVE mov r12d, 0x55555555 kmovd k1, r12d ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, ;; since the keystream is laid out this way, which chunks of - ;; 16 bytes interleved. First the 128 bytes for + ;; 16 bytes interleaved. First the 128 bytes for ;; buffers 0,4,8,12 (total of 512 bytes), then the 128 bytes ;; for buffers 1,5,9,13, and so on %assign IDX 0 @@ -2280,9 +2363,9 @@ ZUC_ROUND64B_16: FUNC_SAVE - vmovdqa xmm5, [bit_reverse_table_l] - vmovdqa xmm6, [bit_reverse_table_h] - vmovdqa xmm7, [bit_reverse_and_table] + vmovdqa xmm5, [rel bit_reverse_table_l] + vmovdqa xmm6, [rel bit_reverse_table_h] + vmovdqa xmm7, [rel bit_reverse_and_table] mov r12d, 0x55555555 kmovd k1, r12d @@ -2432,8 +2515,6 @@ ZUC_ROUND64B_16: mov MIN_LEN, arg5 - FUNC_SAVE - vpbroadcastw ymm0, MIN_LEN ; Get mask of non-NULL lanes (lengths not set to UINT16_MAX, indicating that lane is not valid) vmovdqa ymm1, [LEN] @@ -2460,9 +2541,9 @@ ZUC_ROUND64B_16: xor OFFSET, OFFSET %if USE_GFNI_VAES_VPCLMUL != 1 - vmovdqa xmm5, [bit_reverse_table_l] - vmovdqa xmm6, [bit_reverse_table_h] - vmovdqa xmm7, [bit_reverse_and_table] + vmovdqa xmm5, [rel bit_reverse_table_l] + vmovdqa xmm6, [rel bit_reverse_table_h] + vmovdqa xmm7, [rel bit_reverse_and_table] %endif mov r12d, 0x55555555 @@ -2761,8 +2842,6 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): %endrep %%_ks_copied: vzeroupper - FUNC_RESTORE - ret %endmacro ;; @@ -2777,8 +2856,14 @@ align 64 MKGLOBAL(ZUC128_REMAINDER_16,function,internal) ZUC128_REMAINDER_16: endbranch64 + + FUNC_SAVE + REMAINDER_16 128 + FUNC_RESTORE + + ret ;; ;; extern void asm_Eia3_256_RemainderAVX512_16(uint32_t *T, const void **ks, const void **data, uint64_t n_bits) ;; @@ -2791,8 +2876,15 @@ align 64 MKGLOBAL(ZUC256_REMAINDER_16,function,internal) ZUC256_REMAINDER_16: endbranch64 + + FUNC_SAVE + REMAINDER_16 256 + FUNC_RESTORE + + ret + ; Following functions only need AVX512 instructions (no VAES, GFNI, etc.) %if USE_GFNI_VAES_VPCLMUL == 0 ;; @@ -2809,7 +2901,6 @@ ZUC256_REMAINDER_16: align 64 MKGLOBAL(asm_Eia3RemainderAVX512,function,internal) asm_Eia3RemainderAVX512: - endbranch64 %ifdef LINUX %define T rdi %define KS rsi @@ -2825,11 +2916,13 @@ asm_Eia3RemainderAVX512: %define N_BYTES rbx %define OFFSET r15 + endbranch64 + FUNC_SAVE - vmovdqa xmm5, [bit_reverse_table_l] - vmovdqa xmm6, [bit_reverse_table_h] - vmovdqa xmm7, [bit_reverse_and_table] + vmovdqa xmm5, [rel bit_reverse_table_l] + vmovdqa xmm6, [rel bit_reverse_table_h] + vmovdqa xmm7, [rel bit_reverse_and_table] vpxor xmm9, xmm9 mov r12d, 0x55555555 kmovd k2, r12d @@ -3002,7 +3095,6 @@ Eia3RoundsAVX_end: align 64 MKGLOBAL(asm_Eia3Round64BAVX512,function,internal) asm_Eia3Round64BAVX512: - endbranch64 %ifdef LINUX %define T rdi %define KS rsi @@ -3013,11 +3105,13 @@ asm_Eia3Round64BAVX512: %define DATA r8 %endif + endbranch64 + FUNC_SAVE - vmovdqa xmm5, [bit_reverse_table_l] - vmovdqa xmm6, [bit_reverse_table_h] - vmovdqa xmm7, [bit_reverse_and_table] + vmovdqa xmm5, [rel bit_reverse_table_l] + vmovdqa xmm6, [rel bit_reverse_table_h] + vmovdqa xmm7, [rel bit_reverse_and_table] vpxor xmm9, xmm9 mov r12d, 0x55555555 -- GitLab From e758a0620e92e35f06a36d79d66dac6ed0296582 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 20 Apr 2022 08:42:12 +0000 Subject: [PATCH 072/369] avx512: [ZUC-EIA3-256] Initialize tags at LFSR init --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 20 +-- lib/avx512/zuc_x16_avx512.asm | 161 +++++++++++++----- 2 files changed, 123 insertions(+), 58 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index f7484ec6..364eee4e 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -265,7 +265,7 @@ mksection .text lea arg2, [r11 + _zuc_args_IV] lea arg3, [r11 + _zuc_state] movzx DWORD(arg4), word [r11 + _zuc_init_not_done] - mov r10, 2 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + xor r10, r10 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register call ZUC256_INIT_16 @@ -458,7 +458,7 @@ mksection .text lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] movzx DWORD(arg4), word [r12 + _zuc_init_not_done] - mov r10, 2 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + xor r10, r10 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register call ZUC256_INIT_16 @@ -891,16 +891,10 @@ FLUSH_JOB_ZUC256_EEA3: lea arg3, [r12 + _zuc_state] movzx DWORD(arg4), word [r12 + _zuc_init_not_done] mov r10, 4 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + lea r11, [r12 + _zuc_args_digest] ; Argument 6 hardcoded to r11 call ZUC256_INIT_16 - lea arg1, [r12 + _zuc_state] - lea arg2, [r12 + _zuc_args_digest] - movzx DWORD(arg3), word [r12 + _zuc_init_not_done] - - ; Generate first 4 bytes of keystream, used as the initial value of digests - call ZUC_KEYGEN4B_16 - %endif ;; %%KEY_SIZE == 128 ZUC_EIA3_16_BUFFER r12, %%KEY_SIZE, DWORD(tmp), DWORD(tmp2), tmp3 @@ -1072,16 +1066,10 @@ FLUSH_JOB_ZUC256_EEA3: lea arg3, [r12 + _zuc_state] movzx DWORD(arg4), word [r12 + _zuc_init_not_done] mov r10, 4 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + lea r11, [r12 + _zuc_args_digest] ; Argument 6 hardcoded to r11 call ZUC256_INIT_16 - lea arg1, [r12 + _zuc_state] - lea arg2, [r12 + _zuc_args_digest] - movzx DWORD(arg3), word [r12 + _zuc_init_not_done] - - ; Generate first 4 bytes of keystream, used as the initial value of digests - call ZUC_KEYGEN4B_16 - %endif ;; %%KEY_SIZE == 128 %%skip_init_flush_eia3: diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 05873384..a2b536ed 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -61,6 +61,12 @@ EK_d64: dd 0x0044D700, 0x0026BC00, 0x00626B00, 0x00135E00, 0x00578900, 0x0035E200, 0x00713500, 0x0009AF00 dd 0x004D7800, 0x002F1300, 0x006BC400, 0x001AF100, 0x005E2600, 0x003C4D00, 0x00789A00, 0x0047AC00 +; Constants to be used to initialize the LFSR registers +; The tables contain four different sets of constants: +; 0-63 bytes: Encryption +; 64-127 bytes: Authentication with tag size = 4 +; 128-191 bytes: Authentication with tag size = 8 +; 192-255 bytes: Authentication with tag size = 16 align 64 EK256_d64: dd 0x00220000, 0x002F0000, 0x00240000, 0x002A0000, 0x006D0000, 0x00400000, 0x00400000, 0x00400000 @@ -995,10 +1001,19 @@ align 64 %define %%ZTMP3 %6 ;; [clobbered] ZMM temporary register %define %%ZTMP4 %7 ;; [clobbered] ZMM temporary register %define %%ZTMP5 %8 ;; [clobbered] ZMM temporary register -%define %%CONSTANTS %9 ;; [in] Address to constants -%define %%SHIFT_MASK %10 ;; [in] Mask register to shift K_31 -%define %%IV_MASK %11 ;; [in] Mask register to read IV (last 10 bytes) - +%define %%SHIFT_MASK %9 ;; [in] Mask register to shift K_31 +%define %%IV_MASK %10 ;; [in] Mask register to read IV (last 10 bytes) +%define %%TAG_SIZE %11 ;; [in] Tag size (0, 4, 8 or 16 bytes) + +%if %%TAG_SIZE == 0 +%define %%CONSTANTS rel EK256_d64 +%elif %%TAG_SIZE == 4 +%define %%CONSTANTS rel EK256_EIA3_4 +%elif %%TAG_SIZE == 8 +%define %%CONSTANTS rel EK256_EIA3_8 +%elif %%TAG_SIZE == 16 +%define %%CONSTANTS rel EK256_EIA3_16 +%endif vmovdqu8 XWORD(%%ZTMP4){%%IV_MASK}, [%%IV + 16] ; Zero out first 2 bits of IV bytes 17-24 vpandq XWORD(%%ZTMP4), [rel iv_mask_low_6] @@ -1026,16 +1041,16 @@ align 64 vporq %%LFSR, [%%CONSTANTS] %endmacro -%macro INIT_16_AVX512 1 -%define %%KEY_SIZE %1 ; [in] Key size (128 or 256) - -%define pKe arg1 -%define pIv arg2 -%define pState arg3 -%define lane_mask arg4 - -%define tag_sz r10d ; Only used in ZUC-256 (caller written in assembly, so using a hardcoded register) -%define tag_sz_q r10 +%macro INIT_16_AVX512 8-9 +%define %%KEY %1 ; [in] Array of 16 key pointers +%define %%IV %2 ; [in] Array of 16 IV pointers +%define %%STATE %3 ; [in] State +%define %%LANE_MASK %4 ; [in] Mask register with lanes to update +%define %%TMP %5 ; [clobbered] Temporary GP register +%define %%TMP2 %6 ; [clobbered] Temporary GP register +%define %%KEY_SIZE %7 ; [in] Key size (128 or 256) +%define %%TAG_SIZE %8 ; [in] Tag size (0, 4, 8 or 16 bytes) +%define %%TAGS %9 ; [in] Array of temporary tags %define %%TMP r14 %define %%TMP2 r15 @@ -1082,25 +1097,23 @@ align 64 %define %%R2 %%ZTMP15 %define %%MASK31 %%ZTMP16 +%define %%KSTR1 zmm16 +%define %%KSTR2 zmm17 +%define %%KSTR3 zmm18 +%define %%KSTR4 zmm19 + %define %%BLEND_KMASK k1 ; Mask to blend LFSRs 14&15 %define %%INIT_LANE_KMASK k2 ; Mask containing lanes to initialize %define %%SHIFT_KMASK k3 ; Mask to shift 4 bytes only in the 15th dword %define %%IV_KMASK k4 ; Mask to read 10 bytes of IV - kmovw %%INIT_LANE_KMASK, DWORD(lane_mask) + kmovw %%INIT_LANE_KMASK, DWORD(%%LANE_MASK) %if %%KEY_SIZE == 256 - ; Get pointer to constants (depending on tag size, this will point at - ; constants for encryption, authentication with 4-byte, 8-byte or 16-byte tags) - lea %%TMP, [rel EK256_d64] - bsf tag_sz, tag_sz - dec tag_sz - shl tag_sz, 6 - add tag_sz_q, %%TMP - mov %%TMP2, 0x4000 - kmovq %%SHIFT_KMASK, %%TMP2 - mov %%TMP2, 0x3ff - kmovq %%IV_KMASK, %%TMP2 + mov %%TMP, 0x4000 ; Mask to shift 4 bits only in the 15th dword + kmovq %%SHIFT_KMASK, %%TMP + mov %%TMP, 0x3ff ; Mask to read 10 bytes of IV + kmovq %%IV_KMASK, %%TMP %endif ; Set LFSR registers for Packets 1-16 @@ -1113,8 +1126,8 @@ align 64 INIT_LFSR_128 %%TMP, %%TMP2, APPEND(%%LFSR, %%LFSR_IDX), %%ZTMP1 %else INIT_LFSR_256 %%TMP, %%TMP2, APPEND(%%LFSR, %%LFSR_IDX), %%ZTMP1, \ - %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, tag_sz_q, \ - %%SHIFT_KMASK, %%IV_KMASK + %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%SHIFT_KMASK, %%IV_KMASK, %%TAG_SIZE %endif %assign %%IDX (%%IDX + 1) %assign %%LFSR_IDX (%%LFSR_IDX + 1) @@ -1147,43 +1160,80 @@ align 64 ; Shift LFSR 32-times, update state variables %assign %%N 0 %rep 32 - BITS_REORG16 pState, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + BITS_REORG16 %%STATE, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2 - NONLIN_FUN16 pState, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + NONLIN_FUN16 %%STATE, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W vpsrld %%W, 1 ; Shift out LSB of W - LFSR_UPDT16 pState, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + LFSR_UPDT16 %%STATE, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ %%ZTMP6, %%MASK31, %%W, init ; W used in LFSR update %assign %%N (%%N + 1) %endrep ; And once more, initial round from keygen phase = 33 times - BITS_REORG16 pState, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ + BITS_REORG16 %%STATE, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, \ %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2 - NONLIN_FUN16 pState, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + NONLIN_FUN16 %%STATE, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 - LFSR_UPDT16 pState, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + LFSR_UPDT16 %%STATE, 0, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ %%ZTMP6, %%MASK31, %%W, work + ; Generate extra 4, 8 or 16 bytes of KS for initial tags +%if %%TAG_SIZE == 4 +%define %%NUM_ROUNDS 1 +%elif %%TAG_SIZE == 8 +%define %%NUM_ROUNDS 2 +%elif %%TAG_SIZE == 16 +%define %%NUM_ROUNDS 4 +%else +%define %%NUM_ROUNDS 0 +%endif + +%assign %%N 1 +%rep %%NUM_ROUNDS + BITS_REORG16 %%STATE, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%N) + NONLIN_FUN16 %%STATE, %%INIT_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%W + ; OFS_X3 XOR W + vpxorq APPEND(%%KSTR, %%N), %%W + LFSR_UPDT16 %%STATE, %%N, %%INIT_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%MASK31, %%ZTMP7, work +%assign %%N %%N+1 +%endrep + ; Update R1, R2 - vmovdqa32 [pState + OFS_R1]{%%INIT_LANE_KMASK}, %%R1 - vmovdqa32 [pState + OFS_R2]{%%INIT_LANE_KMASK}, %%R2 + vmovdqa32 [%%STATE + OFS_R1]{%%INIT_LANE_KMASK}, %%R1 + vmovdqa32 [%%STATE + OFS_R2]{%%INIT_LANE_KMASK}, %%R2 + +%if %%TAG_SIZE == 4 + vmovdqa32 [%%TAGS]{%%INIT_LANE_KMASK}, %%KSTR1 + REORDER_LFSR %%STATE, 1, %%INIT_LANE_KMASK +%elif %%TAG_SIZE == 8 ;; TODO +%elif %%TAG_SIZE == 16 ;; TODO +%endif %endmacro ; INIT_16_AVX512 ;; ;; void asm_ZucInitialization_16_avx512(ZucKey16_t *pKeys, ZucIv16_t *pIvs, -;; ZucState16_t *pState) +;; ZucState16_t *pState, +;; const uint64_t lane_mask) ;; MKGLOBAL(ZUC128_INIT,function,internal) ZUC128_INIT: +%define pKe arg1 +%define pIv arg2 +%define pState arg3 +%define lane_mask arg4 + endbranch64 FUNC_SAVE - INIT_16_AVX512 128 + INIT_16_AVX512 pKe, pIv, pState, lane_mask, r12, r13, 128, 0 FUNC_RESTORE @@ -1191,15 +1241,42 @@ ZUC128_INIT: ;; ;; void asm_Zuc256Initialization_16_avx512(ZucKey16_t *pKeys, ZucIv16_t *pIvs, -;; ZucState16_t *pState, uint32_t tag_sz) +;; ZucState16_t *pState, +;; const uint64_t lane_mask, +;; const uint32_t tag_sz, +;; void *tags) ;; MKGLOBAL(ZUC256_INIT,function,internal) ZUC256_INIT: +%define pKe arg1 +%define pIv arg2 +%define pState arg3 +%define lane_mask arg4 +%define tag_sz r10 +%define tags r11 + endbranch64 + or tag_sz, tag_sz + jz init_for_cipher + + ;; TODO: Check for 8B and 16B tags + cmp tag_sz, 4 + je init_for_auth_tag_4B + +init_for_cipher: + FUNC_SAVE + + INIT_16_AVX512 pKe, pIv, pState, lane_mask, r12, r13, 256, 0, tags + + FUNC_RESTORE + + ret + +init_for_auth_tag_4B: FUNC_SAVE - INIT_16_AVX512 256 + INIT_16_AVX512 pKe, pIv, pState, lane_mask, r12, r13, 256, 4, tags FUNC_RESTORE @@ -1325,7 +1402,7 @@ ZUC256_INIT: vmovdqa32 [pState + OFS_R2]{%%ALL_KMASK}, %%R2 %endif - ; Generate rest of the KS bytes (last 8 bytes) for selected lanes + ; Generate rest of the KS bytes (last 8 bytes) for selected lanes %rep 2 BITS_REORG16 pState, %%N, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%IDX) @@ -1357,7 +1434,7 @@ ZUC256_INIT: lea %%TMP2, [rel mov_mask] STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF, \ - %%LANE_MASK, %%TMP1, %%TMP2, %%TMP3, %%TMP_KMASK1,%%TMP_KMASK2 + %%LANE_MASK, %%TMP1, %%TMP2, %%TMP3, %%TMP_KMASK1, %%TMP_KMASK2 %else STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF -- GitLab From 71b045d2b77b60ddda96777443d978691b097992 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 26 Apr 2022 16:18:17 +0000 Subject: [PATCH 073/369] avx512: [ZUC] remove obsolete function --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 2 - .../mb_mgr_zuc_submit_flush_gfni_avx512.asm | 1 - lib/avx512/zuc_x16_avx512.asm | 44 +++++-------------- lib/avx512/zuc_x16_vaes_avx512.asm | 1 - lib/include/zuc_internal.h | 30 ------------- 5 files changed, 11 insertions(+), 67 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index 364eee4e..dbf82258 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -69,8 +69,6 @@ extern asm_ZucCipher_16_avx512 extern asm_ZucCipher_16_gfni_avx512 extern asm_Zuc256Initialization_16_avx512 extern asm_Zuc256Initialization_16_gfni_avx512 -extern asm_ZucGenKeystream4B_16_avx512 -extern asm_ZucGenKeystream4B_16_gfni_avx512 extern asm_Eia3RemainderAVX512_16 extern asm_Eia3RemainderAVX512_16_VPCLMUL extern asm_Eia3_256_RemainderAVX512_16 diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm index 788219cb..17c7a522 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm @@ -35,7 +35,6 @@ %define ZUC128_INIT_16 asm_ZucInitialization_16_gfni_avx512 %define ZUC_CIPHER asm_ZucCipher_16_gfni_avx512 %define ZUC256_INIT_16 asm_Zuc256Initialization_16_gfni_avx512 -%define ZUC_KEYGEN4B_16 asm_ZucGenKeystream4B_16_gfni_avx512 %define ZUC_REMAINDER_16 asm_Eia3RemainderAVX512_16_VPCLMUL %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16_VPCLMUL %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_gfni_avx512 diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index a2b536ed..ec0b76bb 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -44,7 +44,6 @@ %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16 %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_avx512 %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_avx512 -%define ZUC_KEYGEN4B_16 asm_ZucGenKeystream4B_16_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_avx512 %define ZUC_KEYGEN8B_SKIP8_16 asm_ZucGenKeystream8B_16_skip8_avx512 @@ -1286,11 +1285,10 @@ init_for_auth_tag_4B: ; Generate N*4 bytes of keystream ; for 16 buffers (where N is number of rounds) ; -%macro KEYGEN_16_AVX512 3-4 +%macro KEYGEN_16_AVX512 2-3 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds -%define %%STORE_SINGLE %2 ; [in] If 1, KS will be stored continuously in a single buffer -%define %%KEY_OFF %3 ; [in] Offset to start writing Keystream -%define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate all keystream words +%define %%KEY_OFF %2 ; [in] Offset to start writing Keystream +%define %%LANE_MASK %3 ; [in] Lane mask with lanes to generate all keystream words %define pState arg1 %define pKS arg2 @@ -1353,7 +1351,7 @@ init_for_auth_tag_4B: mov DWORD(%%TMP1), 0xAAAAAAAA kmovd %%BLEND_KMASK, DWORD(%%TMP1) -%if (%0 == 4) +%if (%0 == 3) kmovd %%FULL_LANE_KMASK, DWORD(%%LANE_MASK) knotd %%SKIP_LANE_KMASK, %%FULL_LANE_KMASK mov DWORD(%%TMP1), 0x0000FFFF @@ -1419,9 +1417,6 @@ init_for_auth_tag_4B: vmovdqa32 [pState + OFS_R2]{%%FULL_LANE_KMASK}, %%R2 %endif ;; (%%NUM_ROUNDS == 1) -%if (%%STORE_SINGLE == 1) - vmovdqa32 [pKS]{%%FULL_LANE_KMASK}, zmm16 -%else ; Perform a 32-bit 16x16 transpose to have up to 64 bytes ; (NUM_ROUNDS * 4B) of each lane in a different register TRANSPOSE16_U32_INTERLEAVED %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%KSTR5, %%KSTR6, %%KSTR7, %%KSTR8, \ @@ -1429,7 +1424,7 @@ init_for_auth_tag_4B: %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ %%ZTMP9, %%ZTMP10 -%if (%0 == 4) +%if (%0 == 3) lea %%TMP1, [rel alignr_mask] lea %%TMP2, [rel mov_mask] STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ @@ -1439,10 +1434,9 @@ init_for_auth_tag_4B: STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF %endif -%endif ;; %%STORE_SINGLE == 1 ; Reorder LFSR registers -%if (%0 == 4) +%if (%0 == 3) REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK %if (%%NUM_ROUNDS >= 2) REORDER_LFSR pState, (%%NUM_ROUNDS - 2), %%SKIP_LANE_KMASK ; 2 less rounds for "old" buffers @@ -1803,7 +1797,7 @@ ZUC_KEYGEN64B_16: FUNC_SAVE - KEYGEN_16_AVX512 16, 0, arg3 + KEYGEN_16_AVX512 16, arg3 FUNC_RESTORE @@ -1854,7 +1848,7 @@ ZUC_KEYGEN64B_SKIP8_16: FUNC_SAVE - KEYGEN_16_AVX512 16, 0, arg3, arg4 + KEYGEN_16_AVX512 16, arg3, arg4 FUNC_RESTORE @@ -1870,23 +1864,7 @@ ZUC_KEYGEN8B_16: FUNC_SAVE - KEYGEN_16_AVX512 2, 0, arg3 - - FUNC_RESTORE - - ret - -;; -;; void asm_ZucGenKeystream4B_16_avx512(state16_t *pSta, u32 pKeyStr[16], -;; const u32 lane_mask) -;; -MKGLOBAL(ZUC_KEYGEN4B_16,function,internal) -ZUC_KEYGEN4B_16: - endbranch64 - - FUNC_SAVE - - KEYGEN_16_AVX512 1, 1, 0, arg3 + KEYGEN_16_AVX512 2, arg3 FUNC_RESTORE @@ -1941,9 +1919,9 @@ ZUC_KEYGEN4B_16: %rep 16 APPEND(%%_num_rounds_is_,%%I): %if (%0 == 3) - KEYGEN_16_AVX512 %%I, 0, %%KEY_OFF, %%LANE_MASK + KEYGEN_16_AVX512 %%I, %%KEY_OFF, %%LANE_MASK %else - KEYGEN_16_AVX512 %%I, 0, %%KEY_OFF + KEYGEN_16_AVX512 %%I, %%KEY_OFF %endif jmp %%_done diff --git a/lib/avx512/zuc_x16_vaes_avx512.asm b/lib/avx512/zuc_x16_vaes_avx512.asm index a7efd55e..138d0584 100644 --- a/lib/avx512/zuc_x16_vaes_avx512.asm +++ b/lib/avx512/zuc_x16_vaes_avx512.asm @@ -35,7 +35,6 @@ %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16_VPCLMUL %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_gfni_avx512 %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_gfni_avx512 -%define ZUC_KEYGEN4B_16 asm_ZucGenKeystream4B_16_gfni_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_gfni_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_gfni_avx512 %define ZUC_KEYGEN8B_SKIP8_16 asm_ZucGenKeystream8B_16_skip8_gfni_avx512 diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 548948e8..204cfe67 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -863,36 +863,6 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream8B_16_gfni_avx512(ZucState16_t *pState, uint32_t *pKeyStr, const unsigned key_off); -/** - ****************************************************************************** - * - * @description - * Definition of the external function that implements the working - * stage of the ZUC algorithm. The function will generate 4 bytes of - * keystream for sixteen packets in parallel. - * - * @param[in] pState Pointer to a ZUC state structure of type - * @ref ZucState16_t - * - * @param[in,out] pKeyStr Pointer to buffer to write consecutively 4 - * bytes of keystream for the 16 input buffers - * - * @param[in] lane_mask Mask containing lanes which will have 4 - * bytes of KS generated (no bytes generated - * for the rest) - * @pre - * A successful call to @ref asm_ZucInitialization_16 to initialize the ZUC - * state. - * - *****************************************************************************/ -IMB_DLL_LOCAL void asm_ZucGenKeystream4B_16_avx512(ZucState16_t *pState, - uint32_t pKeyStr[16], - const uint32_t lane_mask); - -IMB_DLL_LOCAL void asm_ZucGenKeystream4B_16_gfni_avx512(ZucState16_t *pState, - uint32_t pKeyStr[16], - const uint32_t lane_mask); - /** ****************************************************************************** * -- GitLab From 0a7a94bf2faf6aa4a489b4f31e1dc5f9d56e69c8 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 25 Apr 2022 13:29:01 +0000 Subject: [PATCH 074/369] avx512: [ZUC-EIA3] replace some code with DIGEST_DATA macro --- lib/avx512/zuc_x16_avx512.asm | 107 +++++++--------------------------- 1 file changed, 22 insertions(+), 85 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index ec0b76bb..d25ea913 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2310,10 +2310,10 @@ ZUC_ROUND64B_16: mov r12d, 0x55555555 kmovd k1, r12d ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, - ;; since the keystream is laid out this way, which chunks of + ;; since the keystream is laid out this way, with chunks of ;; 16 bytes interleaved. First the 128 bytes for ;; buffers 0,4,8,12 (total of 512 bytes), then the 128 bytes - ;; for buffers 1,5,9,13, and so on + ;; for buffers 1,5,9,13, and so on. %assign IDX 0 %rep 4 vpxorq APPEND(DIGEST_, IDX), APPEND(DIGEST_, IDX) @@ -2338,26 +2338,9 @@ ZUC_ROUND64B_16: ;; Reverse bits of next 16 bytes from all 4 buffers vgf2p8affineqb ZTMP1, APPEND(DATA_TRANS,I), [rel bit_reverse_table], 0x00 - ;; ZUC authentication part - ;; - 4x32 data bits - ;; - set up KS - vpalignr ZTMP2, APPEND(KS_TRANS, J), APPEND(KS_TRANS, I), 8 - vpshufd ZTMP3, APPEND(KS_TRANS, I), 0x61 - vpshufd ZTMP4, ZTMP2, 0x61 - - ;; - set up DATA - vpshufd APPEND(DATA_TRANS, I){k1}{z}, ZTMP1, 0x10 - vpshufd ZTMP2{k1}{z}, ZTMP1, 0x32 - - ;; - clmul - ;; - xor the results from 4 32-bit words together - vpclmulqdq ZTMP5, APPEND(DATA_TRANS, I), ZTMP3, 0x00 - vpclmulqdq ZTMP6, APPEND(DATA_TRANS, I), ZTMP3, 0x11 - vpclmulqdq ZTMP7, ZTMP2, ZTMP4, 0x00 - vpclmulqdq ZTMP8, ZTMP2, ZTMP4, 0x11 - - vpternlogq ZTMP5, ZTMP6, ZTMP8, 0x96 - vpternlogq APPEND(DIGEST_, IDX), ZTMP5, ZTMP7, 0x96 + ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers + DIGEST_DATA ZTMP1, APPEND(KS_TRANS, I), APPEND(KS_TRANS, J), APPEND(DIGEST_, IDX), \ + k1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7 %assign J (J + 1) %assign I (I + 1) @@ -2450,12 +2433,9 @@ ZUC_ROUND64B_16: vpshufb xmm8, xmm6, xmm1 ; bit reverse low nibbles (use high table) vpshufb xmm4, xmm5, xmm2 ; bit reverse high nibbles (use low table) - vpor xmm8, xmm4 - ; xmm8 - bit reversed data bytes + vpor xmm8, xmm4 ; xmm8 - bit reversed data bytes - ;; ZUC authentication part - ;; - 4x32 data bits - ;; - set up KS + ; Read the next 2 blocks of 16 bytes of KS %if K != 0 vmovdqa xmm11, xmm12 vmovdqu xmm12, [KS + (16*J + I*512) + (K + 1)*(16*4)] @@ -2463,23 +2443,9 @@ ZUC_ROUND64B_16: vmovdqu xmm11, [KS + (16*J + I*512)] vmovdqu xmm12, [KS + (16*J + I*512) + (16*4)] %endif - vpalignr xmm13, xmm12, xmm11, 8 - vpshufd xmm2, xmm11, 0x61 - vpshufd xmm3, xmm13, 0x61 - - ;; - set up DATA - vpshufd xmm0{k1}{z}, xmm8, 0x10 - vpshufd xmm1{k1}{z}, xmm8, 0x32 - - ;; - clmul - ;; - xor the results from 4 32-bit words together - vpclmulqdq xmm13, xmm0, xmm2, 0x00 - vpclmulqdq xmm14, xmm0, xmm2, 0x11 - vpclmulqdq xmm15, xmm1, xmm3, 0x00 - vpclmulqdq xmm8, xmm1, xmm3, 0x11 - - vpternlogq xmm13, xmm14, xmm8, 0x96 - vpternlogq xmm9, xmm13, xmm15, 0x96 + ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers + DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k1, \ + xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 %assign K (K + 1) %endrep @@ -2644,9 +2610,7 @@ ZUC_ROUND64B_16: %endif ; xmm8 - bit reversed data bytes - ;; ZUC authentication part - ;; - 4x32 data bits - ;; - set up KS + ; Read the next 2 blocks of 16 bytes of KS %if K != 0 vmovdqa xmm11, xmm12 vmovdqu xmm12, [KS + (16*I + J*512) + OFFSET*4 + (16*4)] @@ -2654,23 +2618,9 @@ ZUC_ROUND64B_16: vmovdqu xmm11, [KS + (16*I + J*512) + (0*4)] vmovdqu xmm12, [KS + (16*I + J*512) + (16*4)] %endif - vpalignr xmm13, xmm12, xmm11, 8 - vpshufd xmm2, xmm11, 0x61 - vpshufd xmm3, xmm13, 0x61 - - ;; - set up DATA - vpshufd xmm0{k2}{z}, xmm8, 0x10 - vpshufd xmm1{k2}{z}, xmm8, 0x32 - - ;; - clmul - ;; - xor the results from 4 32-bit words together - vpclmulqdq xmm13, xmm0, xmm2, 0x00 - vpclmulqdq xmm14, xmm0, xmm2, 0x11 - vpclmulqdq xmm15, xmm1, xmm3, 0x00 - vpclmulqdq xmm8, xmm1, xmm3, 0x11 - - vpternlogq xmm13, xmm14, xmm8, 0x96 - vpternlogq xmm9, xmm13, xmm15, 0x96 + ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers + DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k2, \ + xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 add OFFSET, 16 sub TMP, 128 %assign K (K + 1) @@ -2688,15 +2638,6 @@ APPEND3(%%Eia3RoundsAVX512_dq_end,I,J): lea r11, [rel byte64_len_to_mask_table] kmovq k1, [r11 + N_BYTES*8] - ;; Set up KS - shl OFFSET, 2 - vmovdqu xmm1, [KS + (16*I + J*512) + OFFSET] - vmovdqu xmm2, [KS + (16*I + J*512) + OFFSET + 16*4] - shr OFFSET, 2 - vpalignr xmm13, xmm2, xmm1, 8 - vpshufd xmm11, xmm1, 0x61 - vpshufd xmm12, xmm13, 0x61 - ;; read up to 16 bytes of data, zero bits not needed if partial byte and bit-reverse vmovdqu8 xmm0{k1}{z}, [DATA_ADDR + OFFSET] ; check if there is a partial byte (less than 8 bits in last byte) @@ -2733,19 +2674,15 @@ APPEND3(%%Eia3RoundsAVX512_dq_end,I,J): vpor xmm8, xmm3 %endif - ;; - set up DATA - vpshufd xmm0{k2}{z}, xmm8, 0x10 ; D 0-3 || Os || D 4-7 || 0s - vpshufd xmm1{k2}{z}, xmm8, 0x32 ; D 8-11 || 0s || D 12-15 || 0s - - ;; - clmul - ;; - xor the results from 4 32-bit words together - vpclmulqdq xmm13, xmm0, xmm11, 0x00 - vpclmulqdq xmm14, xmm0, xmm11, 0x11 - vpclmulqdq xmm15, xmm1, xmm12, 0x00 - vpclmulqdq xmm8, xmm1, xmm12, 0x11 - vpternlogq xmm9, xmm14, xmm13, 0x96 - vpternlogq xmm9, xmm15, xmm8, 0x96 + ; Read the next 2 blocks of 16 bytes of KS + shl OFFSET, 2 + vmovdqu xmm11, [KS + (16*I + J*512) + OFFSET] + vmovdqu xmm12, [KS + (16*I + J*512) + OFFSET + 16*4] + shr OFFSET, 2 + ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers + DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k2, \ + xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 APPEND3(%%Eia3RoundsAVX_end,I,J): vinserti32x4 APPEND(DIGEST_, I), xmm9, J %assign J (J + 1) -- GitLab From 02f9d9e2bbd1369218b5f2d14f700ed16235db65 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 26 Apr 2022 19:15:05 +0000 Subject: [PATCH 075/369] avx512: [ZUC-EIA3] Add tag size parameter to Round64B_16 function --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 1 + lib/avx512/zuc_top_avx512.c | 8 +- lib/avx512/zuc_x16_avx512.asm | 455 ++++++++++-------- lib/include/zuc_internal.h | 20 +- 4 files changed, 284 insertions(+), 200 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index dbf82258..e206c161 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -669,6 +669,7 @@ FLUSH_JOB_ZUC256_EEA3: lea arg2, [%%OOO + _zuc_args_KS] lea arg3, [%%OOO + _zuc_args_in] lea arg4, [%%OOO + _zuc_lens] + mov arg5, 4 call ZUC_ROUND64B diff --git a/lib/avx512/zuc_top_avx512.c b/lib/avx512/zuc_top_avx512.c index 69d55972..eb0e1f36 100755 --- a/lib/avx512/zuc_top_avx512.c +++ b/lib/avx512/zuc_top_avx512.c @@ -145,12 +145,12 @@ cipher_16(ZucState16_t *pState, const uint64_t *pIn[16], uint64_t *pOut[16], static inline void round64B_16(uint32_t *T, const uint32_t *ks, const void **data, - uint16_t *lens, const unsigned use_gfni) + uint16_t *lens, const unsigned use_gfni, const uint64_t tag_sz) { if (use_gfni) - asm_Eia3Round64B_16_VPCLMUL(T, ks, data, lens); + asm_Eia3Round64B_16_VPCLMUL(T, ks, data, lens, tag_sz); else - asm_Eia3Round64BAVX512_16(T, ks, data, lens); + asm_Eia3Round64BAVX512_16(T, ks, data, lens, tag_sz); } static inline @@ -700,7 +700,7 @@ void _zuc_eia3_16_buffer_avx512(const void * const pKey[NUM_AVX512_BUFS], else keystr_64B_gen_16(&state, keyStr, 64, use_gfni); round64B_16(T, keyStr, - (const void **)pIn8, lens, use_gfni); + (const void **)pIn8, lens, use_gfni, 4); } /* Process each packet separately for the remaining bits */ diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index d25ea913..87fbc9db 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2233,270 +2233,347 @@ _no_final_rounds: ret + ;; -;;extern void asm_Eia3Round64B_16(uint32_t *T, const void *KS, -;; const void **DATA, uint16_t *LEN); +;; Updates authentication tag T of 16 buffers based on keystream KS and DATA +;; (GFNI/VAES/VPCLMULQDQ version) ;; -;; Updates authentication tag T of 16 buffers based on keystream KS and DATA. -;; - it processes 64 bytes of DATA of buffers -;; - reads data in 16 byte chunks from different buffers -;; (first buffers 0,4,8,12; then 1,5,9,13; etc) and bit reverses them -;; - reads KS (when utilizing VPCLMUL instructions, it reads 64 bytes directly, -;; containing 16 bytes of KS for 4 different buffers) -;; - employs clmul for the XOR & ROL part -;; - copies top 64 bytes of KS to bottom (for the next round) -;; - Updates Data pointers for next rounds -;; - Updates array of lengths -;; -;; @param [in] T: Array of digests for all 16 buffers -;; @param [in] KS: Pointer to 128 bytes of keystream for all 16 buffers (2048 bytes in total) -;; @param [in] DATA: Array of pointers to data for all 16 buffers -;; @param [in] LEN: Array of lengths for all 16 buffers -;; -align 64 -MKGLOBAL(ZUC_ROUND64B_16,function,internal) -ZUC_ROUND64B_16: -%define T arg1 -%define KS arg2 -%define DATA arg3 -%define LEN arg4 +%macro ROUND64B_16_GFNI 11 +%define %%T %1 ; [in] Pointer to digests +%define %%KS %2 ; [in] Pointer to keystream (128x16 bytes) +%define %%DATA %3 ; [in] Pointer to array of pointers to data buffers +%define %%LEN %4 ; [in] Pointer to array of remaining length to digest +%define %%TMP1 %5 ; [clobbered] Temporary GP register +%define %%TMP2 %6 ; [clobbered] Temporary GP register +%define %%TMP3 %7 ; [clobbered] Temporary GP register +%define %%TMP4 %8 ; [clobbered] Temporary GP register +%define %%TMP5 %9 ; [clobbered] Temporary GP register +%define %%TMP6 %10 ; [clobbered] Temporary GP register +%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) + +%define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data +%define %%PERM_DIGEST_KMASK1 k2 ; Permutation mask for digests +%define %%PERM_DIGEST_KMASK2 k3 ; Permulation mask for digests +%define %%TMP_KMASK k4 + +%define %%DATA_ADDR0 %%TMP3 +%define %%DATA_ADDR1 %%TMP4 +%define %%DATA_ADDR2 %%TMP5 +%define %%DATA_ADDR3 %%TMP6 + +%define %%DATA_TRANS0 zmm19 +%define %%DATA_TRANS1 zmm20 +%define %%DATA_TRANS2 zmm21 +%define %%DATA_TRANS3 zmm22 +%define %%DATA_TRANS0x xmm19 +%define %%DATA_TRANS1x xmm20 +%define %%DATA_TRANS2x xmm21 +%define %%DATA_TRANS3x xmm22 + +%define %%KS_TRANS0 zmm23 +%define %%KS_TRANS1 zmm24 +%define %%KS_TRANS2 zmm25 +%define %%KS_TRANS3 zmm26 +%define %%KS_TRANS4 zmm27 +%define %%KS_TRANS0x xmm23 +%define %%KS_TRANS1x xmm24 +%define %%KS_TRANS2x xmm25 +%define %%KS_TRANS3x xmm26 +%define %%KS_TRANS4x xmm27 -%if USE_GFNI_VAES_VPCLMUL == 1 -%define DATA_ADDR0 rbx -%define DATA_ADDR1 r10 -%define DATA_ADDR2 r11 -%define DATA_ADDR3 r12 - -%define DATA_TRANS0 zmm19 -%define DATA_TRANS1 zmm20 -%define DATA_TRANS2 zmm21 -%define DATA_TRANS3 zmm22 -%define DATA_TRANS0x xmm19 -%define DATA_TRANS1x xmm20 -%define DATA_TRANS2x xmm21 -%define DATA_TRANS3x xmm22 - -%define KS_TRANS0 zmm23 -%define KS_TRANS1 zmm24 -%define KS_TRANS2 zmm25 -%define KS_TRANS3 zmm26 -%define KS_TRANS4 zmm27 -%define KS_TRANS0x xmm23 -%define KS_TRANS1x xmm24 -%define KS_TRANS2x xmm25 -%define KS_TRANS3x xmm26 -%define KS_TRANS4x xmm27 - -%define DIGEST_0 zmm28 -%define DIGEST_1 zmm29 -%define DIGEST_2 zmm30 -%define DIGEST_3 zmm31 - -%define ZTMP1 zmm0 -%define ZTMP2 zmm1 -%define ZTMP3 zmm2 -%define ZTMP4 zmm3 -%define ZTMP5 zmm4 -%define ZTMP6 zmm5 -%define ZTMP7 zmm6 -%define ZTMP8 zmm7 - -%define YTMP1 YWORD(ZTMP1) +%define %%DIGEST_0 zmm28 +%define %%DIGEST_1 zmm29 +%define %%DIGEST_2 zmm30 +%define %%DIGEST_3 zmm31 - endbranch64 +%define %%ZTMP1 zmm0 +%define %%ZTMP2 zmm1 +%define %%ZTMP3 zmm2 +%define %%ZTMP4 zmm3 +%define %%ZTMP5 zmm4 +%define %%ZTMP6 zmm5 +%define %%ZTMP7 zmm6 +%define %%ZTMP8 zmm7 - FUNC_SAVE +%define %%YTMP1 YWORD(%%ZTMP1) - mov r12d, 0x55555555 - kmovd k1, r12d + mov DWORD(%%TMP1), 0x55555555 + kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP1) ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, ;; since the keystream is laid out this way, with chunks of ;; 16 bytes interleaved. First the 128 bytes for ;; buffers 0,4,8,12 (total of 512 bytes), then the 128 bytes ;; for buffers 1,5,9,13, and so on. -%assign IDX 0 +%assign %%IDX 0 %rep 4 - vpxorq APPEND(DIGEST_, IDX), APPEND(DIGEST_, IDX) + vpxorq APPEND(%%DIGEST_, %%IDX), APPEND(%%DIGEST_, %%IDX) - mov DATA_ADDR0, [DATA + IDX*8 + 0*32] - mov DATA_ADDR1, [DATA + IDX*8 + 1*32] - mov DATA_ADDR2, [DATA + IDX*8 + 2*32] - mov DATA_ADDR3, [DATA + IDX*8 + 3*32] + mov %%DATA_ADDR0, [%%DATA + %%IDX*8 + 0*32] + mov %%DATA_ADDR1, [%%DATA + %%IDX*8 + 1*32] + mov %%DATA_ADDR2, [%%DATA + %%IDX*8 + 2*32] + mov %%DATA_ADDR3, [%%DATA + %%IDX*8 + 3*32] - vmovdqu64 KS_TRANS0, [KS + IDX*64*2*4] + vmovdqu64 %%KS_TRANS0, [%%KS + %%IDX*64*2*4] -%assign I 0 -%assign J 1 +%assign %%I 0 +%assign %%J 1 %rep 4 - vmovdqu64 XWORD(APPEND(DATA_TRANS, I)), [DATA_ADDR0 + 16*I] - vinserti32x4 APPEND(DATA_TRANS, I), [DATA_ADDR1 + 16*I], 1 - vinserti32x4 APPEND(DATA_TRANS, I), [DATA_ADDR2 + 16*I], 2 - vinserti32x4 APPEND(DATA_TRANS, I), [DATA_ADDR3 + 16*I], 3 + vmovdqu64 XWORD(APPEND(%%DATA_TRANS, %%I)), [%%DATA_ADDR0 + 16*%%I] + vinserti32x4 APPEND(%%DATA_TRANS, %%I), [%%DATA_ADDR1 + 16*%%I], 1 + vinserti32x4 APPEND(%%DATA_TRANS, %%I), [%%DATA_ADDR2 + 16*%%I], 2 + vinserti32x4 APPEND(%%DATA_TRANS, %%I), [%%DATA_ADDR3 + 16*%%I], 3 - vmovdqu64 APPEND(KS_TRANS, J), [KS + IDX*64*2*4 + 64*J] + vmovdqu64 APPEND(%%KS_TRANS, %%J), [%%KS + %%IDX*64*2*4 + 64*%%J] ;; Reverse bits of next 16 bytes from all 4 buffers - vgf2p8affineqb ZTMP1, APPEND(DATA_TRANS,I), [rel bit_reverse_table], 0x00 + vgf2p8affineqb %%ZTMP1, APPEND(%%DATA_TRANS,%%I), [rel bit_reverse_table], 0x00 ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA ZTMP1, APPEND(KS_TRANS, I), APPEND(KS_TRANS, J), APPEND(DIGEST_, IDX), \ - k1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7 + DIGEST_DATA %%ZTMP1, APPEND(%%KS_TRANS, %%I), APPEND(%%KS_TRANS, %%J), APPEND(%%DIGEST_, %%IDX), \ + %%SHUF_DATA_KMASK, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7 -%assign J (J + 1) -%assign I (I + 1) +%assign %%J (%%J + 1) +%assign %%I (%%I + 1) %endrep ; Memcpy KS 64-127 bytes to 0-63 bytes - vmovdqa64 ZTMP4, [KS + IDX*4*64*2 + 64*4] - vmovdqa64 ZTMP1, [KS + IDX*4*64*2 + 64*5] - vmovdqa64 ZTMP2, [KS + IDX*4*64*2 + 64*6] - vmovdqa64 ZTMP3, [KS + IDX*4*64*2 + 64*7] - vmovdqa64 [KS + IDX*4*64*2], ZTMP4 - vmovdqa64 [KS + IDX*4*64*2 + 64], ZTMP1 - vmovdqa64 [KS + IDX*4*64*2 + 64*2], ZTMP2 - vmovdqa64 [KS + IDX*4*64*2 + 64*3], ZTMP3 - -%assign IDX (IDX + 1) + vmovdqa64 %%ZTMP4, [%%KS + %%IDX*4*64*2 + 64*4] + vmovdqa64 %%ZTMP1, [%%KS + %%IDX*4*64*2 + 64*5] + vmovdqa64 %%ZTMP2, [%%KS + %%IDX*4*64*2 + 64*6] + vmovdqa64 %%ZTMP3, [%%KS + %%IDX*4*64*2 + 64*7] + vmovdqa64 [%%KS + %%IDX*4*64*2], %%ZTMP4 + vmovdqa64 [%%KS + %%IDX*4*64*2 + 64], %%ZTMP1 + vmovdqa64 [%%KS + %%IDX*4*64*2 + 64*2], %%ZTMP2 + vmovdqa64 [%%KS + %%IDX*4*64*2 + 64*3], %%ZTMP3 + +%assign %%IDX (%%IDX + 1) %endrep ;; - update tags - mov r12, 0x3333 - mov r13, 0xCCCC - kmovq k1, r12 - kmovq k2, r13 - - vmovdqu64 ZTMP1, [T] ; Input tags - vmovdqa64 ZTMP2, [rel shuf_mask_tags_0_4_8_12] - vmovdqa64 ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] + mov %%TMP1, 0x3333 + mov %%TMP2, 0xCCCC + kmovq %%PERM_DIGEST_KMASK1, %%TMP1 + kmovq %%PERM_DIGEST_KMASK2, %%TMP2 + + vmovdqu64 %%ZTMP1, [%%T] ; Input tags + vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_4_8_12] + vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. - vpermt2d DIGEST_0{k1}{z}, ZTMP2, DIGEST_1 - vpermt2d DIGEST_2{k2}{z}, ZTMP3, DIGEST_3 - vpternlogq ZTMP1, DIGEST_0, DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [T], ZTMP1 + vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 + vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 + vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C + vmovdqu64 [%%T], %%ZTMP1 ; Update data pointers - vmovdqu64 ZTMP1, [DATA] - vmovdqu64 ZTMP2, [DATA + 64] - vpaddq ZTMP1, [rel add_64] - vpaddq ZTMP2, [rel add_64] - vmovdqu64 [DATA], ZTMP1 - vmovdqu64 [DATA + 64], ZTMP2 + vmovdqu64 %%ZTMP1, [%%DATA] + vmovdqu64 %%ZTMP2, [%%DATA + 64] + vpaddq %%ZTMP1, [rel add_64] + vpaddq %%ZTMP2, [rel add_64] + vmovdqu64 [%%DATA], %%ZTMP1 + vmovdqu64 [%%DATA + 64], %%ZTMP2 ; Update array of lengths (subtract 512 bits from all lengths if valid lane) - vmovdqa YTMP1, [LEN] - vpcmpw k1, YTMP1, [rel all_ffs], 4 - vpsubw YTMP1{k1}, [rel all_512w] - vmovdqa [LEN], YTMP1 + vmovdqa %%YTMP1, [LEN] + vpcmpw %%TMP_KMASK, %%YTMP1, [rel all_ffs], 4 + vpsubw %%YTMP1{%%TMP_KMASK}, [rel all_512w] + vmovdqa [%%LEN], %%YTMP1 -%else ; USE_GFNI_VAES_VPCLMUL == 1 - -%define DIGEST_0 zmm28 -%define DIGEST_1 zmm29 -%define DIGEST_2 zmm30 -%define DIGEST_3 zmm31 - -%define DATA_ADDR r10 - - FUNC_SAVE +%endmacro - vmovdqa xmm5, [rel bit_reverse_table_l] - vmovdqa xmm6, [rel bit_reverse_table_h] - vmovdqa xmm7, [rel bit_reverse_and_table] - mov r12d, 0x55555555 - kmovd k1, r12d +;; +;; Updates authentication tag T of 16 buffers based on keystream KS and DATA. +;; +%macro ROUND64B_16_NO_GFNI 11 +%define %%T %1 ; [in] Pointer to digests +%define %%KS %2 ; [in] Pointer to keystream (128x16 bytes) +%define %%DATA %3 ; [in] Pointer to array of pointers to data buffers +%define %%LEN %4 ; [in] Pointer to array of remaining length to digest +%define %%TMP1 %5 ; [clobbered] Temporary GP register +%define %%TMP2 %6 ; [clobbered] Temporary GP register +%define %%TMP3 %7 ; [clobbered] Temporary GP register +%define %%TMP4 %8 ; [clobbered] Temporary GP register +%define %%TMP5 %9 ; [clobbered] Temporary GP register +%define %%TMP6 %10 ; [clobbered] Temporary GP register +%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) + +%define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data +%define %%PERM_DIGEST_KMASK1 k2 ; Permutation mask for digests +%define %%PERM_DIGEST_KMASK2 k3 ; Permulation mask for digests +%define %%TMP_KMASK k4 + +%define %%REV_TABLE_L xmm0 +%define %%REV_TABLE_H xmm1 +%define %%REV_AND_TABLE xmm2 +%define %%TEMP_DIGEST xmm3 +%define %%KS_L xmm4 +%define %%KS_H xmm5 +%define %%XDATA xmm6 +%define %%XTMP1 xmm7 +%define %%XTMP2 xmm8 +%define %%XTMP3 xmm9 +%define %%XTMP4 xmm10 +%define %%XTMP5 xmm11 +%define %%XTMP6 xmm12 + +%define %%ZTMP1 zmm24 +%define %%ZTMP2 zmm25 +%define %%ZTMP3 zmm26 +%define %%ZTMP4 zmm27 +%define %%DIGEST_0 zmm28 +%define %%DIGEST_1 zmm29 +%define %%DIGEST_2 zmm30 +%define %%DIGEST_3 zmm31 + +%define %%YTMP1 ymm24 + +%define %%DATA_ADDR %%TMP3 + + vmovdqa %%REV_TABLE_L, [rel bit_reverse_table_l] + vmovdqa %%REV_TABLE_H, [rel bit_reverse_table_h] + vmovdqa %%REV_AND_TABLE, [rel bit_reverse_and_table] + + mov DWORD(%%TMP1), 0x55555555 + kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP1) ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, ;; since the keystream is laid out this way, which chunks of ;; 16 bytes interleved. First the 128 bytes for ;; buffers 0,4,8,12 (total of 512 bytes), then the 128 bytes ;; for buffers 1,5,9,13, and so on -%assign I 0 +%assign %%I 0 %rep 4 -%assign J 0 +%assign %%J 0 %rep 4 - vpxor xmm9, xmm9 - mov DATA_ADDR, [DATA + 8*(J*4 + I)] + vpxor %%TEMP_DIGEST, %%TEMP_DIGEST + mov %%DATA_ADDR, [%%DATA + 8*(%%J*4 + %%I)] -%assign K 0 +%assign %%K 0 %rep 4 ;; read 16 bytes and reverse bits - vmovdqu xmm0, [DATA_ADDR + 16*K] - vpand xmm1, xmm0, xmm7 + vmovdqu %%XTMP1, [%%DATA_ADDR + 16*%%K] + vpand %%XTMP2, %%XTMP1, %%REV_AND_TABLE - vpandn xmm2, xmm7, xmm0 - vpsrld xmm2, 4 + vpandn %%XTMP3, %%REV_AND_TABLE, %%XTMP1 + vpsrld %%XTMP3, 4 - vpshufb xmm8, xmm6, xmm1 ; bit reverse low nibbles (use high table) - vpshufb xmm4, xmm5, xmm2 ; bit reverse high nibbles (use low table) + vpshufb %%XDATA, %%REV_TABLE_H, %%XTMP2 ; bit reverse low nibbles (use high table) + vpshufb %%XTMP4, %%REV_TABLE_L, %%XTMP3 ; bit reverse high nibbles (use low table) - vpor xmm8, xmm4 ; xmm8 - bit reversed data bytes + vpor %%XDATA, %%XDATA, %%XTMP4 ; %%DATA - bit reversed data bytes ; Read the next 2 blocks of 16 bytes of KS -%if K != 0 - vmovdqa xmm11, xmm12 - vmovdqu xmm12, [KS + (16*J + I*512) + (K + 1)*(16*4)] +%if %%K != 0 + vmovdqa %%KS_L, %%KS_H + vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (%%K + 1)*(16*4)] %else - vmovdqu xmm11, [KS + (16*J + I*512)] - vmovdqu xmm12, [KS + (16*J + I*512) + (16*4)] + vmovdqu %%KS_L, [%%KS + (16*%%J + %%I*512)] + vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (16*4)] %endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k1, \ - xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 -%assign K (K + 1) +%assign %%K (%%K + 1) %endrep - vinserti32x4 APPEND(DIGEST_, I), xmm9, J -%assign J (J + 1) + vinserti32x4 APPEND(%%DIGEST_, %%I), %%TEMP_DIGEST, %%J +%assign %%J (%%J + 1) %endrep ; Memcpy KS 64-127 bytes to 0-63 bytes - vmovdqa64 zmm23, [KS + I*4*64*2 + 64*4] - vmovdqa64 zmm24, [KS + I*4*64*2 + 64*5] - vmovdqa64 zmm25, [KS + I*4*64*2 + 64*6] - vmovdqa64 zmm26, [KS + I*4*64*2 + 64*7] - vmovdqa64 [KS + I*4*64*2], zmm23 - vmovdqa64 [KS + I*4*64*2 + 64], zmm24 - vmovdqa64 [KS + I*4*64*2 + 64*2], zmm25 - vmovdqa64 [KS + I*4*64*2 + 64*3], zmm26 -%assign I (I + 1) + vmovdqa64 %%ZTMP1, [%%KS + %%I*4*64*2 + 64*4] + vmovdqa64 %%ZTMP2, [%%KS + %%I*4*64*2 + 64*5] + vmovdqa64 %%ZTMP3, [%%KS + %%I*4*64*2 + 64*6] + vmovdqa64 %%ZTMP4, [%%KS + %%I*4*64*2 + 64*7] + vmovdqa64 [%%KS + %%I*4*64*2], %%ZTMP1 + vmovdqa64 [%%KS + %%I*4*64*2 + 64], %%ZTMP2 + vmovdqa64 [%%KS + %%I*4*64*2 + 64*2], %%ZTMP3 + vmovdqa64 [%%KS + %%I*4*64*2 + 64*3], %%ZTMP4 +%assign %%I (%%I + 1) %endrep ;; - update tags - mov r12, 0x3333 - mov r13, 0xCCCC - kmovq k1, r12 - kmovq k2, r13 + mov %%TMP1, 0x3333 + mov %%TMP2, 0xCCCC + kmovq %%PERM_DIGEST_KMASK1, %%TMP1 + kmovq %%PERM_DIGEST_KMASK2, %%TMP2 - vmovdqu64 zmm4, [T] ; Input tags - vmovdqa64 zmm0, [rel shuf_mask_tags_0_4_8_12] - vmovdqa64 zmm1, [rel shuf_mask_tags_0_4_8_12 + 64] + vmovdqu64 %%ZTMP1, [%%T] ; Input tags + vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_4_8_12] + vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. - vpermt2d DIGEST_0{k1}{z}, zmm0, DIGEST_1 - vpermt2d DIGEST_2{k2}{z}, zmm1, DIGEST_3 - vpternlogq zmm4, DIGEST_0, DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [T], zmm4 + vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 + vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 + vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C + vmovdqu64 [%%T], %%ZTMP1 ; Update data pointers - vmovdqu64 zmm0, [DATA] - vmovdqu64 zmm1, [DATA + 64] - vpaddq zmm0, [rel add_64] - vpaddq zmm1, [rel add_64] - vmovdqu64 [DATA], zmm0 - vmovdqu64 [DATA + 64], zmm1 + vmovdqu64 %%ZTMP2, [%%DATA] + vmovdqu64 %%ZTMP3, [%%DATA + 64] + vpaddq %%ZTMP2, [rel add_64] + vpaddq %%ZTMP3, [rel add_64] + vmovdqu64 [%%DATA], %%ZTMP2 + vmovdqu64 [%%DATA + 64], %%ZTMP3 ; Update array of lengths (if lane is valid, so length < UINT16_MAX) - vmovdqa ymm2, [LEN] - vpcmpw k1, ymm2, [rel all_ffs], 4 ; k1 -> valid lanes - vpsubw ymm2{k1}, [rel all_512w] - vmovdqa [LEN], ymm2 + vmovdqa64 %%YTMP1, [%%LEN] + vpcmpw %%TMP_KMASK, %%YTMP1, [rel all_ffs], 4 ; valid lanes + vpsubw %%YTMP1{%%TMP_KMASK}, [rel all_512w] + vmovdqa64 [%%LEN], %%YTMP1 + +%endmacro + +;; +;;extern void asm_Eia3Round64B_16(void *T, const void *KS, +;; const void **DATA, uint16_t *LEN); +;; +;; Updates authentication tag T of 16 buffers based on keystream KS and DATA. +;; - it processes 64 bytes of DATA of buffers +;; - reads data in 16 byte chunks from different buffers +;; (first buffers 0,4,8,12; then 1,5,9,13; etc) and bit reverses them +;; - reads KS (when utilizing VPCLMUL instructions, it reads 64 bytes directly, +;; containing 16 bytes of KS for 4 different buffers) +;; - employs clmul for the XOR & ROL part +;; - copies top 64 bytes of KS to bottom (for the next round) +;; - Updates Data pointers for next rounds +;; - Updates array of lengths +;; +;; @param [in] T: Array of digests for all 16 buffers +;; @param [in] KS: Pointer to 128 bytes of keystream for all 16 buffers (2048 bytes in total) +;; @param [in] DATA: Array of pointers to data for all 16 buffers +;; @param [in] LEN: Array of lengths for all 16 buffers +;; @param [in] TAG_SZ: Tag size (4, 8 or 16 bytes) +;; +align 64 +MKGLOBAL(ZUC_ROUND64B_16,function,internal) +ZUC_ROUND64B_16: +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define LEN arg4 +%define TAG_SZ arg5 + + endbranch64 + + ; TODO: 8-byte and 16-byte digests + cmp TAG_SZ, 4 + je round_4B + +round_4B: + + FUNC_SAVE + +%if USE_GFNI_VAES_VPCLMUL == 1 + ROUND64B_16_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 4 +%else + ROUND64B_16_NO_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 4 +%endif -%endif ;; USE_GFNI_VAES_VPCLMUL == 0 FUNC_RESTORE ret diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 204cfe67..a18efede 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1140,26 +1140,32 @@ IMB_DLL_LOCAL void asm_Eia3Remainder_avx(void *T, const void *ks, * Definition of the external function to update the authentication tag * based on keystream and data (AVX variant) * - * @param[in] T Authentication tag + * @param[in] T Array of authentication tags for 16 buffers * - * @param[in] ks Pointer to key stream + * @param[in] ks Array of keystreams for 16 buffers * - * @param[in] data Pointer to the data + * @param[in] data Array of pointers to the data for 16 buffers + * + * @param[in] len Array of remaining lengths for 16 buffers + * + * @param[in] tag_sz Tag size (4, 8 or 16 bytes) * * @pre * None * *****************************************************************************/ -IMB_DLL_LOCAL void asm_Eia3Round64BAVX512_16(uint32_t *T, +IMB_DLL_LOCAL void asm_Eia3Round64BAVX512_16(void *T, const uint32_t *ks, const void **data, - uint16_t *len); + uint16_t *len, + const uint64_t tag_sz); -IMB_DLL_LOCAL void asm_Eia3Round64B_16_VPCLMUL(uint32_t *T, +IMB_DLL_LOCAL void asm_Eia3Round64B_16_VPCLMUL(void *T, const uint32_t *ks, const void **data, - uint16_t *len); + uint16_t *len, + const uint64_t tag_sz); IMB_DLL_LOCAL void asm_Eia3Round64BAVX512(uint32_t *T, const void *ks, const void *data); -- GitLab From 9b1640d137e88fa0b2c16e141e7d8d6f6d5afbc4 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 26 Apr 2022 21:20:49 +0000 Subject: [PATCH 076/369] avx512: [ZUC-EIA3] Add tag size parameter to Remainder_16 function --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 1 + lib/avx512/zuc_x16_avx512.asm | 460 ++++++++++-------- lib/include/zuc_internal.h | 6 +- 3 files changed, 266 insertions(+), 201 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index e206c161..b5c9671b 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -737,6 +737,7 @@ FLUSH_JOB_ZUC256_EEA3: %if %%KEY_SIZE == 128 call ZUC_REMAINDER_16 %else + mov arg6, 4 ; Hardcoded to 4-byte digest for now call ZUC256_REMAINDER_16 %endif diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 87fbc9db..a00d0364 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -389,8 +389,8 @@ align 64 %define arg2 rdx %define arg3 r8 %define arg4 r9 -%define arg5 [rsp + 40] -%define arg6 [rsp + 48] +%define arg5 qword [rsp + 40] +%define arg6 qword [rsp + 48] %endif %define OFS_R1 (16*(4*16)) @@ -2578,74 +2578,104 @@ round_4B: ret -%macro REMAINDER_16 1 -%define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) - -%ifdef LINUX - %define T rdi - %define KS rsi - %define DATA rdx - %define LEN rcx - %define arg5 r8d -%else - %define T rcx - %define KS rdx - %define DATA r8 - %define LEN r9 - %define arg5 [rsp + 40] -%endif - -%define DIGEST_0 zmm28 -%define DIGEST_1 zmm29 -%define DIGEST_2 zmm30 -%define DIGEST_3 zmm31 - -%define DATA_ADDR r12 -%define KS_ADDR r13 - -%define N_BYTES r14 -%define OFFSET r15 - -%define MIN_LEN r10d -%define MIN_LEN_Q r10 -%define IDX rax -%define TMP rbx - - mov MIN_LEN, arg5 +%macro REMAINDER_16 14 +%define %%T %1 ; [in] Pointer to digests +%define %%KS %2 ; [in] Pointer to keystream (128x16 bytes) +%define %%DATA %3 ; [in] Pointer to array of pointers to data buffers +%define %%LEN %4 ; [in] Pointer to array of remaining length to digest +%define %%MIN_LEN %5 ; [in] Minimum common length +%define %%TMP1 %6 ; [clobbered] Temporary GP register +%define %%TMP2 %7 ; [clobbered] Temporary GP register +%define %%TMP3 %8 ; [clobbered] Temporary GP register +%define %%TMP4 %9 ; [clobbered] Temporary GP register +%define %%TMP5 %10 ; [clobbered] Temporary GP register +%define %%TMP6 %11 ; [clobbered] Temporary GP register +%define %%TMP7 %12 ; [clobbered] Temporary GP register +%define %%KEY_SIZE %13 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %14 ; [constant] Tag size (4, 8 or 16 bytes) + +%define %%DIGEST_0 zmm28 +%define %%DIGEST_1 zmm29 +%define %%DIGEST_2 zmm30 +%define %%DIGEST_3 zmm31 - vpbroadcastw ymm0, MIN_LEN +;; +;; There are two main parts in this code: +;; - 1st part: digest data +;; - 2nd part: reading final KS words and XOR'ing with digest +;; +%define %%DATA_ADDR %%TMP2 ; %%DATA_ADDR only used in 1st part / %%TMP2 only used in 2nd part +%define %%OFFSET %%TMP3 ; %%OFFSET only used in 1st part / %%TMP3 only used in 2nd part +%define %%KS_ADDR %%TMP7 ; %%KS_ADDR used in all code +%define %%N_BYTES %%TMP6 ; %%N_BYTES only used in 1st part + +%define %%LEN_BUF %%TMP4 ; %%LEN_BUF only used in 2nd part +%define %%IDX %%TMP5 ; %%IDX Only used in 2nd part +%define %%DIGEST %%TMP6 ; %%DIGEST only used in 2nd part + +%define %%YTMP1 ymm7 +%define %%YTMP2 ymm8 +%define %%YTMP3 ymm9 +%define %%YTMP4 ymm10 + +%define %%REV_TABLE_L xmm0 +%define %%REV_TABLE_H xmm1 +%define %%REV_AND_TABLE xmm2 +%define %%TEMP_DIGEST xmm3 +%define %%KS_L xmm4 +%define %%KS_H xmm5 +%define %%XDATA xmm6 +%define %%XTMP1 xmm7 +%define %%XTMP2 xmm8 +%define %%XTMP3 xmm9 +%define %%XTMP4 xmm10 +%define %%XTMP5 xmm11 +%define %%XTMP6 xmm12 + +%define %%ZTMP1 zmm7 +%define %%ZTMP2 zmm8 +%define %%ZTMP3 zmm9 +%define %%ZTMP4 zmm10 + +%define %%VALID_KMASK k1 ; Mask with valid lanes +%define %%SHUF_DATA_KMASK k2 ; Mask to shuffle data +%define %%PERM_DIGEST_KMASK1 k3 ; Permutation mask for digests +%define %%PERM_DIGEST_KMASK2 k4 ; Permulation mask for digests +%define %%TEMP_KMASK k5 + + vpbroadcastw %%YTMP1, DWORD(%%MIN_LEN) ; Get mask of non-NULL lanes (lengths not set to UINT16_MAX, indicating that lane is not valid) - vmovdqa ymm1, [LEN] - vpcmpw k1, ymm1, [rel all_ffs], 4 ; NEQ + vmovdqa %%YTMP2, [%%LEN] + vpcmpw %%VALID_KMASK, %%YTMP2, [rel all_ffs], 4 ; NEQ ; Round up to nearest multiple of 32 bits - vpaddw ymm0{k1}, [rel all_31w] - vpandq ymm0, [rel all_ffe0w] + vpaddw %%YTMP1{%%VALID_KMASK}, [rel all_31w] + vpandq %%YTMP1, [rel all_ffe0w] ; Calculate remaining bits to authenticate after function call - vpcmpuw k2, ymm1, ymm0, 1 ; Get mask of lengths that will be < 0 after subtracting - vpsubw ymm2{k1}, ymm1, ymm0 - vpxorq ymm3, ymm3 + vpcmpuw %%TEMP_KMASK, %%YTMP2, %%YTMP1, 1 ; Get mask of lengths that will be < 0 after subtracting + vpsubw %%YTMP3{%%VALID_KMASK}, %%YTMP2, %%YTMP1 + vpxorq %%YTMP4, %%YTMP4 ; Set to zero the lengths of the lanes which are going to be completed - vmovdqu16 ymm2{k2}, ymm3 ; YMM2 contain final lengths - vmovdqu16 [LEN]{k1}, ymm2 ; Update in memory the final updated lengths + vmovdqu16 %%YTMP3{%%TEMP_KMASK}, %%YTMP4 ; YMM2 contain final lengths + vmovdqu16 [%%LEN]{%%VALID_KMASK}, %%YTMP3 ; Update in memory the final updated lengths ; Calculate number of bits to authenticate (up to 511 bits), ; for each lane, and store it in stack to be used later - vpsubw ymm1{k1}{z}, ymm2 ; Bits to authenticate in all lanes (zero out length of NULL lanes) + vpsubw %%YTMP2{%%VALID_KMASK}{z}, %%YTMP3 ; Bits to authenticate in all lanes (zero out length of NULL lanes) sub rsp, 32 - vmovdqu [rsp], ymm1 + vmovdqu [rsp], %%YTMP2 - xor OFFSET, OFFSET + xor %%OFFSET, %%OFFSET %if USE_GFNI_VAES_VPCLMUL != 1 - vmovdqa xmm5, [rel bit_reverse_table_l] - vmovdqa xmm6, [rel bit_reverse_table_h] - vmovdqa xmm7, [rel bit_reverse_and_table] + vmovdqa %%REV_TABLE_L, [rel bit_reverse_table_l] + vmovdqa %%REV_TABLE_H, [rel bit_reverse_table_h] + vmovdqa %%REV_AND_TABLE, [rel bit_reverse_and_table] %endif mov r12d, 0x55555555 - kmovd k2, r12d + kmovd %%SHUF_DATA_KMASK, r12d ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, ;; since the keystream is laid out this way, which chunks of @@ -2658,297 +2688,329 @@ round_4B: %rep 4 ; Read length to authenticate for each buffer - movzx TMP, word [rsp + 2*(I*4 + J)] + movzx %%LEN_BUF, word [rsp + 2*(I*4 + J)] - vpxor xmm9, xmm9 + vpxor %%TEMP_DIGEST, %%TEMP_DIGEST - xor OFFSET, OFFSET - mov DATA_ADDR, [DATA + 8*(I*4 + J)] + xor %%OFFSET, %%OFFSET + mov %%DATA_ADDR, [%%DATA + 8*(I*4 + J)] %assign K 0 %rep 4 - cmp TMP, 128 + cmp %%LEN_BUF, 128 jb APPEND3(%%Eia3RoundsAVX512_dq_end,I,J) ;; read 16 bytes and reverse bits - vmovdqu xmm0, [DATA_ADDR + OFFSET] + vmovdqu %%XTMP1, [%%DATA_ADDR + %%OFFSET] %if USE_GFNI_VAES_VPCLMUL == 1 - vgf2p8affineqb xmm8, xmm0, [rel bit_reverse_table], 0x00 + vgf2p8affineqb %%XDATA, %%XTMP1, [rel bit_reverse_table], 0x00 %else - vpand xmm1, xmm0, xmm7 + vpand %%XTMP2, %%XTMP1, %%REV_AND_TABLE - vpandn xmm2, xmm7, xmm0 - vpsrld xmm2, 4 + vpandn %%XTMP3, %%REV_AND_TABLE, %%XTMP1 + vpsrld %%XTMP3, 4 - vpshufb xmm8, xmm6, xmm1 ; bit reverse low nibbles (use high table) - vpshufb xmm4, xmm5, xmm2 ; bit reverse high nibbles (use low table) + vpshufb %%XDATA, %%REV_TABLE_H, %%XTMP2 ; bit reverse low nibbles (use high table) + vpshufb %%XTMP4, %%REV_TABLE_L, %%XTMP3 ; bit reverse high nibbles (use low table) - vpor xmm8, xmm4 + vpor %%XDATA, %%XTMP4 %endif - ; xmm8 - bit reversed data bytes + ; %%XDATA - bit reversed data bytes - ; Read the next 2 blocks of 16 bytes of KS + ; Read the next 2 blocks of 16 bytes of %%KS %if K != 0 - vmovdqa xmm11, xmm12 - vmovdqu xmm12, [KS + (16*I + J*512) + OFFSET*4 + (16*4)] + vmovdqa %%KS_L, %%KS_H + vmovdqu %%KS_H, [%%KS + (16*I + J*512) + %%OFFSET*4 + (16*4)] %else - vmovdqu xmm11, [KS + (16*I + J*512) + (0*4)] - vmovdqu xmm12, [KS + (16*I + J*512) + (16*4)] + vmovdqu %%KS_L, [%%KS + (16*I + J*512) + (0*4)] + vmovdqu %%KS_H, [%%KS + (16*I + J*512) + (16*4)] %endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k2, \ - xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 - add OFFSET, 16 - sub TMP, 128 + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + add %%OFFSET, 16 + sub %%LEN_BUF, 128 %assign K (K + 1) %endrep APPEND3(%%Eia3RoundsAVX512_dq_end,I,J): - or TMP, TMP + or %%LEN_BUF, %%LEN_BUF jz APPEND3(%%Eia3RoundsAVX_end,I,J) ; Get number of bytes - mov N_BYTES, TMP - add N_BYTES, 7 - shr N_BYTES, 3 + mov %%N_BYTES, %%LEN_BUF + add %%N_BYTES, 7 + shr %%N_BYTES, 3 - lea r11, [rel byte64_len_to_mask_table] - kmovq k1, [r11 + N_BYTES*8] + lea %%TMP1, [rel byte64_len_to_mask_table] + kmovq %%TEMP_KMASK, [%%TMP1 + %%N_BYTES*8] ;; read up to 16 bytes of data, zero bits not needed if partial byte and bit-reverse - vmovdqu8 xmm0{k1}{z}, [DATA_ADDR + OFFSET] + vmovdqu8 %%XTMP1{%%TEMP_KMASK}{z}, [%%DATA_ADDR + %%OFFSET] ; check if there is a partial byte (less than 8 bits in last byte) - mov rax, TMP - and rax, 0x7 - shl rax, 4 - lea r11, [rel bit_mask_table] - add r11, rax + mov %%TMP2, %%LEN_BUF + and %%TMP2, 0x7 + shl %%TMP2, 4 + lea %%TMP1, [rel bit_mask_table] + add %%TMP1, %%TMP2 ; Get mask to clear last bits - vmovdqa xmm3, [r11] + vmovdqa %%XTMP4, [%%TMP1] ; Shift left 16-N bytes to have the last byte always at the end of the XMM register ; to apply mask, then restore by shifting right same amount of bytes - mov r11, 16 - sub r11, N_BYTES - ; r13 = DATA_ADDR can be used at this stage - XVPSLLB xmm0, r11, xmm4, r13 - vpandq xmm0, xmm3 - XVPSRLB xmm0, r11, xmm4, r13 + mov %%TMP1, 16 + sub %%TMP1, %%N_BYTES + XVPSLLB %%XTMP1, %%TMP1, %%XTMP5, %%TMP2 + vpandq %%XTMP1, %%XTMP4 + XVPSRLB %%XTMP1, %%TMP1, %%XTMP5, %%TMP2 %if USE_GFNI_VAES_VPCLMUL == 1 - vgf2p8affineqb xmm8, xmm0, [rel bit_reverse_table], 0x00 + vgf2p8affineqb %%XDATA, %%XTMP1, [rel bit_reverse_table], 0x00 %else ; Bit reverse input data - vpand xmm1, xmm0, xmm7 + vpand %%XTMP2, %%XTMP1, %%REV_AND_TABLE - vpandn xmm2, xmm7, xmm0 - vpsrld xmm2, 4 + vpandn %%XTMP3, %%REV_AND_TABLE, %%XTMP1 + vpsrld %%XTMP3, 4 - vpshufb xmm8, xmm6, xmm1 ; bit reverse low nibbles (use high table) - vpshufb xmm3, xmm5, xmm2 ; bit reverse high nibbles (use low table) + vpshufb %%XDATA, %%REV_TABLE_H, %%XTMP2 ; bit reverse low nibbles (use high table) + vpshufb %%XTMP4, %%REV_TABLE_L, %%XTMP3 ; bit reverse high nibbles (use low table) - vpor xmm8, xmm3 + vpor %%XDATA, %%XTMP4 %endif ; Read the next 2 blocks of 16 bytes of KS - shl OFFSET, 2 - vmovdqu xmm11, [KS + (16*I + J*512) + OFFSET] - vmovdqu xmm12, [KS + (16*I + J*512) + OFFSET + 16*4] - shr OFFSET, 2 + shl %%OFFSET, 2 + vmovdqu %%KS_L, [%%KS + (16*I + J*512) + %%OFFSET] + vmovdqu %%KS_H, [%%KS + (16*I + J*512) + %%OFFSET + 16*4] + shr %%OFFSET, 2 - ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA xmm8, xmm11, xmm12, xmm9, k2, \ - xmm0, xmm1, xmm2, xmm3, xmm13, xmm14 + ; Digest 16 bytes of data with 24 bytes of %%KS, for 4 buffers + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, k2, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 APPEND3(%%Eia3RoundsAVX_end,I,J): - vinserti32x4 APPEND(DIGEST_, I), xmm9, J + vinserti32x4 APPEND(%%DIGEST_, I), %%TEMP_DIGEST, J %assign J (J + 1) %endrep %assign I (I + 1) %endrep ;; - update tags - mov TMP, 0x00FF - kmovq k1, TMP - mov TMP, 0xFF00 - kmovq k2, TMP - - vmovdqu64 zmm4, [T] ; Input tags - vmovdqa64 zmm0, [rel shuf_mask_tags_0_1_2_3] - vmovdqa64 zmm1, [rel shuf_mask_tags_0_1_2_3 + 64] + mov %%TMP1, 0x00FF + kmovq %%PERM_DIGEST_KMASK1, %%TMP1 + mov %%TMP1, 0xFF00 + kmovq %%PERM_DIGEST_KMASK2, %%TMP1 + + vmovdqu64 %%ZTMP1, [%%T] ; Input tags + vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_1_2_3] + vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_1_2_3 + 64] ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. - vpermt2d DIGEST_0{k1}{z}, zmm0, DIGEST_1 - vpermt2d DIGEST_2{k2}{z}, zmm1, DIGEST_3 - vpternlogq zmm4, DIGEST_0, DIGEST_2, 0x96 ; A XOR B XOR C + vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 + vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 + vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqa64 [T], zmm4 ; Store temporary digests + vmovdqa64 [%%T], %%ZTMP1 ; Store temporary digests ; These last steps should be done only for the buffers that ; have no more data to authenticate - xor IDX, IDX + xor %%IDX, %%IDX %%start_loop: ; Update data pointer - movzx r11d, word [rsp + IDX*2] - shr r11d, 3 ; length authenticated in bytes - add [DATA + IDX*8], r11 + movzx DWORD(%%TMP1), word [rsp + %%IDX*2] + shr DWORD(%%TMP1), 3 ; length authenticated in bytes + add [%%DATA + %%IDX*8], %%TMP1 - cmp word [LEN + 2*IDX], 0 + cmp word [%%LEN + 2*%%IDX], 0 jnz %%skip_comput - mov r11, IDX - and r11, 0x3 - shl r11, 9 ; * 512 + mov %%TMP1, %%IDX + and %%TMP1, 0x3 + shl %%TMP1, 9 ; * 512 - mov r12, IDX - shr r12, 2 - shl r12, 4 ; * 16 - add r11, r12 - lea KS_ADDR, [KS + r11] + mov %%TMP2, %%IDX + shr %%TMP2, 2 + shl %%TMP2, 4 ; * 16 + add %%TMP1, %%TMP2 + lea %%KS_ADDR, [%%KS + %%TMP1] ; Read digest - mov r12d, [T + 4*IDX] + mov DWORD(%%DIGEST), [%%T + 4*%%IDX] ; Read keyStr[MIN_LEN / 32] - movzx TMP, word [rsp + 2*IDX] - mov r15, TMP - shr r15, 5 - mov r11, r15 - shr r15, 2 - shl r15, (4+2) - and r11, 0x3 - shl r11, 2 - add r15, r11 - mov r11, r15 - and r11, 0xf - cmp r11, 12 + movzx %%LEN_BUF, word [rsp + 2*%%IDX] + mov %%TMP2, %%LEN_BUF + shr %%TMP2, 5 + mov %%TMP3, %%TMP2 + shr %%TMP2, 2 + shl %%TMP2, (4+2) + and %%TMP3, 0x3 + shl %%TMP3, 2 + add %%TMP2, %%TMP3 + mov %%TMP3, %%TMP2 + and %%TMP3, 0xf + cmp %%TMP3, 12 je %%_read_2dwords - mov r11, [KS_ADDR + r15] + mov %%TMP1, [%%KS_ADDR + %%TMP2] jmp %%_ks_qword_read - ;; The 8 bytes of KS are separated + ;; The 8 bytes of %%KS are separated %%_read_2dwords: - mov r11d, [KS_ADDR + r15] - mov r15d, [KS_ADDR + r15 + (4+48)] - shl r15, 32 - or r11, r15 + mov DWORD(%%TMP1), [%%KS_ADDR + %%TMP2] + mov DWORD(%%TMP2), [%%KS_ADDR + %%TMP2 + (4+48)] + shl %%TMP2, 32 + or %%TMP1, %%TMP2 %%_ks_qword_read: ; Rotate left by MIN_LEN % 32 - mov r15, rcx - mov rcx, TMP + mov %%TMP2, rcx + mov rcx, %%LEN_BUF and rcx, 0x1F - rol r11, cl - mov rcx, r15 + rol %%TMP1, cl + mov rcx, %%TMP2 ; XOR with current digest - xor r12d, r11d + xor DWORD(%%DIGEST), DWORD(%%TMP1) %if %%KEY_SIZE == 128 ; Read keystr[L - 1] (last dword of keyStr) - add TMP, (31 + 64) - shr TMP, 5 ; L - dec TMP - mov r11, TMP - shr r11, 2 - shl r11, (4+2) - and TMP, 0x3 - shl TMP, 2 - add TMP, r11 - mov r11d, [KS_ADDR + TMP] + add %%LEN_BUF, (31 + 64) + shr %%LEN_BUF, 5 ; L + dec %%LEN_BUF + mov %%TMP2, %%LEN_BUF + shr %%TMP2, 2 + shl %%TMP2, (4+2) + and %%LEN_BUF, 0x3 + shl %%LEN_BUF, 2 + add %%LEN_BUF, %%TMP2 + mov DWORD(%%TMP2), [%%KS_ADDR + %%LEN_BUF] ; XOR with current digest - xor r12d, r11d + xor DWORD(%%DIGEST), DWORD(%%TMP2) %endif ; byte swap and write digest out - bswap r12d - mov [T + 4*IDX], r12d + bswap DWORD(%%DIGEST) + mov [%%T + 4*%%IDX], DWORD(%%DIGEST) %%skip_comput: - inc IDX - cmp IDX, 16 + inc %%IDX + cmp %%IDX, 16 jne %%start_loop add rsp, 32 - ; Memcpy last 8 bytes of KS into start - add MIN_LEN, 31 - shr MIN_LEN, 5 - shl MIN_LEN, 2 ; Offset where to copy the last 8 bytes from - - mov r12d, MIN_LEN - shr MIN_LEN, 4 - shl MIN_LEN, (4+2) - and r12d, 0xf - add MIN_LEN, r12d - cmp r12d, 12 + ; Memcpy last 8 bytes of %%KS into start + add DWORD(%%MIN_LEN), 31 + shr DWORD(%%MIN_LEN), 5 + shl DWORD(%%MIN_LEN), 2 ; Offset where to copy the last 8 bytes from + + mov DWORD(%%TMP1), DWORD(%%MIN_LEN) + shr DWORD(%%MIN_LEN), 4 + shl DWORD(%%MIN_LEN), (4+2) + and DWORD(%%TMP1), 0xf + add DWORD(%%MIN_LEN), DWORD(%%TMP1) + cmp DWORD(%%TMP1), 12 je %%_copy_2dwords %assign %%i 0 %rep 4 %assign %%j 0 %rep 4 - mov TMP, [KS + 512*%%i + 16*%%j + MIN_LEN_Q] - mov [KS + 512*%%i + 16*%%j], TMP + mov %%TMP1, [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], %%TMP1 %assign %%j (%%j + 1) %endrep %assign %%i (%%i + 1) %endrep jmp %%_ks_copied - ;; The 8 bytes of KS are separated + ;; The 8 bytes of %%KS are separated %%_copy_2dwords: %assign %%i 0 %rep 4 %assign %%j 0 %rep 4 - mov DWORD(TMP), [KS + 512*%%i + 16*%%j + MIN_LEN_Q] - mov [KS + 512*%%i + 16*%%j], DWORD(TMP) - mov DWORD(TMP), [KS + 512*%%i + 16*%%j + (48+4) + MIN_LEN_Q] - mov [KS + 512*%%i + 16*%%j + 4], DWORD(TMP) + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], DWORD(%%TMP1) + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + (48+4) + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j + 4], DWORD(%%TMP1) %assign %%j (%%j + 1) %endrep %assign %%i (%%i + 1) %endrep %%_ks_copied: vzeroupper -%endmacro +%endmacro ; REMAINDER_16 ;; -;; extern void asm_Eia3RemainderAVX512_16(uint32_t *T, const void **ks, const void **data, uint64_t n_bits) +;; extern void asm_Eia3RemainderAVX512_16(uint32_t *T, const void **ks, +;; const void **data, uint16_t *len, +;; const uint64_t n_bits) ;; ;; @param [in] T: Array of digests for all 16 buffers ;; @param [in] KS : Array of pointers to key stream for all 16 buffers ;; @param [in] DATA : Array of pointers to data for all 16 buffers -;; @param [in] N_BITS (number data bits to process) +;; @param [in] N_BITS : Number of common data bits to process ;; align 64 MKGLOBAL(ZUC128_REMAINDER_16,function,internal) ZUC128_REMAINDER_16: + +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define LEN arg4 + +%define N_BITS r10 + endbranch64 + mov N_BITS, arg5 + FUNC_SAVE - REMAINDER_16 128 + REMAINDER_16 T, KS, DATA, LEN, N_BITS, rax, rbx, r11, r12, r13, r14, r15, 128, 4 FUNC_RESTORE ret ;; -;; extern void asm_Eia3_256_RemainderAVX512_16(uint32_t *T, const void **ks, const void **data, uint64_t n_bits) +;; extern void asm_Eia3_256_RemainderAVX512_16(void *T, const void **ks, +;; const void **data, uint16_t *len, +;; const uint64_t n_bits, +;; const uint64_t tag_size) ;; ;; @param [in] T: Array of digests for all 16 buffers ;; @param [in] KS : Array of pointers to key stream for all 16 buffers ;; @param [in] DATA : Array of pointers to data for all 16 buffers -;; @param [in] N_BITS (number data bits to process) +;; @param [in] N_BITS : Number data bits to process +;; @param [in] TAG_SIZE : Tag size (4, 8 or 16 bytes) ;; align 64 MKGLOBAL(ZUC256_REMAINDER_16,function,internal) ZUC256_REMAINDER_16: + +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define LEN arg4 + +%define N_BITS r10 + +%define TAG_SZ arg6 + endbranch64 + mov N_BITS, arg5 + + ; TODO: 8-byte and 16-byte digests + cmp TAG_SZ, 4 + je remainder_4B + +remainder_4B: FUNC_SAVE - REMAINDER_16 256 + REMAINDER_16 T, KS, DATA, LEN, N_BITS, rax, rbx, r11, r12, r13, r14, r15, 256, 4 FUNC_RESTORE diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index a18efede..4bd82b9b 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1202,7 +1202,8 @@ IMB_DLL_LOCAL uint32_t asm_Eia3_256_RemainderAVX512_16(uint32_t *T, const uint32_t *ks, const void **data, uint16_t *lens, - const uint32_t commonBits); + const uint64_t commonBits, + const uint64_t tag_size); IMB_DLL_LOCAL uint32_t asm_Eia3RemainderAVX512_16_VPCLMUL(uint32_t *T, const uint32_t *ks, @@ -1214,7 +1215,8 @@ IMB_DLL_LOCAL uint32_t asm_Eia3_256_RemainderAVX512_16_VPCLMUL(uint32_t *T, const uint32_t *ks, const void **data, uint16_t *lens, - const uint32_t commonBits); + const uint64_t commonBits, + const uint64_t tag_size); /** ****************************************************************************** -- GitLab From 758f057669b755f69d0d055304bb53b0a5eef6f5 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 3 May 2022 08:45:56 +0000 Subject: [PATCH 077/369] test: [HMAC-SHAx] Test using burst API in HMAC-SHAx tests --- test/hmac_sha1_test.c | 146 +++++++++++++++++++- test/hmac_sha256_sha512_test.c | 244 +++++++++++++++++++++++++++++++-- 2 files changed, 374 insertions(+), 16 deletions(-) diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c index 66ff7d98..ec8ad162 100644 --- a/test/hmac_sha1_test.c +++ b/test/hmac_sha1_test.c @@ -39,6 +39,7 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr); #define block_size 64 #define digest_size 20 #define digest96_size 12 +#define max_burst_jobs 32 /* * Test vectors from https://tools.ietf.org/html/rfc2202 @@ -347,17 +348,18 @@ hmac_sha1_job_ok(const struct hmac_sha1_rfc2202_vector *vec, static int test_hmac_sha1(struct IMB_MGR *mb_mgr, const struct hmac_sha1_rfc2202_vector *vec, - const int num_jobs) + const uint32_t num_jobs) { struct IMB_JOB *job; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; + uint32_t i = 0, jobs_rx = 0; + int ret = -1; uint8_t key[block_size]; uint8_t buf[block_size]; DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - int key_len = 0; + uint32_t key_len = 0; if (auths == NULL) { fprintf(stderr, "Can't allocate buffer memory\n"); @@ -477,9 +479,133 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, return ret; } +static int +test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, + const struct hmac_sha1_rfc2202_vector *vec, + const uint32_t num_jobs) +{ + struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1; + uint8_t key[block_size]; + uint8_t buf[block_size]; + DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); + DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); + uint32_t key_len = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->digest_len + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (vec->key_len <= block_size) { + memcpy(key, vec->key, vec->key_len); + key_len = (int) vec->key_len; + } else { + IMB_SHA1(mb_mgr, vec->key, vec->key_len, key); + key_len = digest_size; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->digest_len; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = vec->data; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->data_len; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + + job->user_data = auths[i]; + + } + + uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + static void test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, - const int num_jobs, + const uint32_t num_jobs, struct test_suite_context *ts) { const int vectors_cnt = DIM(hmac_sha1_vectors); @@ -506,6 +632,13 @@ test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, } else { test_suite_update(ts, 1, 0); } + if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], + num_jobs)) { + printf("error #%d - burst API\n", vect); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } } printf("\n"); } @@ -514,10 +647,11 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr) { struct test_suite_context ts; - int num_jobs, errors = 0; + int errors = 0; + uint32_t num_jobs; test_suite_start(&ts, "HMAC-SHA1"); - for (num_jobs = 1; num_jobs <= 17; num_jobs++) + for (num_jobs = 1; num_jobs <= max_burst_jobs; num_jobs++) test_hmac_sha1_std_vectors(mb_mgr, num_jobs, &ts); errors = test_suite_end(&ts); diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index 22eb3226..a1c8b6f4 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -34,6 +34,8 @@ #include "gcm_ctr_vectors_test.h" #include "utils.h" +#define max_burst_jobs 32 + int hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr); /* @@ -818,18 +820,19 @@ hmac_shax_job_ok(const struct hmac_rfc4231_vector *vec, static int test_hmac_shax(struct IMB_MGR *mb_mgr, const struct hmac_rfc4231_vector *vec, - const int num_jobs, + const uint32_t num_jobs, const int sha_type) { struct IMB_JOB *job; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); - int i = 0, jobs_rx = 0, ret = -1; + uint32_t i = 0, jobs_rx = 0; + int ret = -1; uint8_t key[IMB_SHA_512_BLOCK_SIZE]; uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); - int key_len = 0; + uint32_t key_len = 0; size_t digest_len = 0; size_t block_size = 0; @@ -1037,10 +1040,226 @@ test_hmac_shax(struct IMB_MGR *mb_mgr, return ret; } +static int +test_hmac_shax_burst(struct IMB_MGR *mb_mgr, + const struct hmac_rfc4231_vector *vec, + const uint32_t num_jobs, + const int sha_type) +{ + struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1; + uint8_t key[IMB_SHA_512_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint32_t key_len = 0; + size_t digest_len = 0; + size_t block_size = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + switch (sha_type) { + case 224: + digest_len = vec->hmac_sha224_len; + block_size = IMB_SHA_256_BLOCK_SIZE; + break; + case 256: + digest_len = vec->hmac_sha256_len; + block_size = IMB_SHA_256_BLOCK_SIZE; + break; + case 384: + digest_len = vec->hmac_sha384_len; + block_size = IMB_SHA_384_BLOCK_SIZE; + break; + case 512: + digest_len = vec->hmac_sha512_len; + block_size = IMB_SHA_512_BLOCK_SIZE; + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + digest_len + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (vec->key_len <= block_size) { + memcpy(key, vec->key, vec->key_len); + key_len = (int) vec->key_len; + } else { + switch (sha_type) { + case 224: + IMB_SHA224(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + break; + case 256: + IMB_SHA256(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + break; + case 384: + IMB_SHA384(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; + break; + case 512: + IMB_SHA512(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end; + } + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + + switch (sha_type) { + case 224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 512: + default: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + } + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + + switch (sha_type) { + case 224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 512: + default: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = digest_len; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = vec->data; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->data_len; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (sha_type) { + case 224: + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + } + + job->user_data = auths[i]; + + } + + uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + static void test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, const int sha_type, - const int num_jobs, + const uint32_t num_jobs, struct test_suite_context *ts) { const int vectors_cnt = DIM(hmac_sha256_sha512_vectors); @@ -1083,6 +1302,14 @@ test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, } else { test_suite_update(ts, 1, 0); } + if (test_hmac_shax_burst(mb_mgr, + &hmac_sha256_sha512_vectors[idx], + num_jobs, sha_type)) { + printf("error #%d - burst API\n", vect); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } } printf("\n"); } @@ -1096,19 +1323,16 @@ hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr) static const char * const sha_names_tab[] = { "HMAC-SHA224", "HMAC-SHA256", "HMAC-SHA384", "HMAC-SHA512" }; - const int num_jobs_tab[] = { - 1, 3, 4, 5, 7, 8, 9, 15, 16, 17 - }; - unsigned i, j; + unsigned i, num_jobs; int errors = 0; for (i = 0; i < DIM(sha_types_tab); i++) { struct test_suite_context ts; test_suite_start(&ts, sha_names_tab[i]); - for (j = 0; j < DIM(num_jobs_tab); j++) + for (num_jobs = 1; num_jobs <= max_burst_jobs; num_jobs++) test_hmac_shax_std_vectors(mb_mgr, sha_types_tab[i], - num_jobs_tab[j], &ts); + num_jobs, &ts); errors += test_suite_end(&ts); } -- GitLab From 2f7118523937c77ed5f9977f1987f1a5f4dc834e Mon Sep 17 00:00:00 2001 From: Julien Meunier Date: Tue, 5 Apr 2022 15:35:24 +0300 Subject: [PATCH 078/369] test: [snow3g] add random vectors and new linear tests In order to cover the bug #104, new random vectors and test cases are added. New test cases ============== - validate_snow3g_f8_n_blocks_linear_multi_keys Test all vectors from snow3g_f8_vectors. Each vector of each request (keys and plain text / cipher text / IV) are different. API tested: IMB_SNOW3G_F8_N_BUFFER_MULTIKEY (will test 1 / 2 / 4 / 8 buffers) - validate_snow3g_f8_n_blocks_linear Test all vectors from snow3g_f8_linear. Each vector of each request (keys and plain text / cipher text / IV) are different. API tested IMB_SNOW3G_F8_N_BUFFER (will test 1 / 2 / 4 / 8 buffers) New test vectors ================ - snow3g_f8_vectors, extending with random and simple vectors, in order to have more than 16 elements for the test case validate_snow3g_f8_n_blocks_linear_multi_keys - snow3g_f8_linear, random and simple vectors, with 16 elements containing the same key, used by validate_snow3g_f8_n_blocks_linear Without the fix provided in #104, these test cases failed with the AVX2 implementation. Reported-by: Damian Zajaczkowski Signed-off-by: Julien Meunier Signed-off-by: Clement Guedez --- test/snow3g_test.c | 555 ++++++++++++++++++++++++++++++++++++- test/snow3g_test_vectors.h | 438 ++++++++++++++++++++++++++++- 2 files changed, 981 insertions(+), 12 deletions(-) diff --git a/test/snow3g_test.c b/test/snow3g_test.c index fac42674..c0d93ca6 100644 --- a/test/snow3g_test.c +++ b/test/snow3g_test.c @@ -1,5 +1,6 @@ /***************************************************************************** Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2022, Nokia Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -70,6 +71,15 @@ validate_snow3g_f8_n_blocks(struct IMB_MGR *mb_mgr, uint32_t job_api, struct test_suite_context *uea2_ctx, struct test_suite_context *uia2_ctx); static void +validate_snow3g_f8_n_blocks_linear(struct IMB_MGR *mb_mgr, uint32_t job_api, + struct test_suite_context *uea2_ctx, + struct test_suite_context *uia2_ctx); +static void +validate_snow3g_f8_n_blocks_linear_mkeys(struct IMB_MGR *mb_mgr, + uint32_t job_api, + struct test_suite_context *uea2_ctx, + struct test_suite_context *uia2_ctx); +static void validate_snow3g_f8_n_blocks_multi(struct IMB_MGR *mb_mgr, uint32_t job_api, struct test_suite_context *uea2_ctx, @@ -99,6 +109,10 @@ struct { "validate_snow3g_f8_8_blocks_multi_key"}, {validate_snow3g_f8_n_blocks, "validate_snow3g_f8_n_blocks"}, + {validate_snow3g_f8_n_blocks_linear, + "validate_snow3g_f8_n_blocks_linear"}, + {validate_snow3g_f8_n_blocks_linear_mkeys, + "validate_snow3g_f8_n_blocks_linear_multi_keys"}, {validate_snow3g_f8_n_blocks_multi, "validate_snow3g_f8_n_blocks_multi"}, {validate_snow3g_f9, @@ -1885,21 +1899,21 @@ validate_snow3g_f8_n_blocks(struct IMB_MGR *mb_mgr, uint32_t job_api, for (i = 0; i < NUM_SUPPORTED_BUFFERS; i++) { /*Test the encrypt*/ - if (job_api) + if (job_api) { submit_uea2_jobs(mb_mgr, (uint8_t **)&pKeySched, pIV, pSrcBuff, pDstBuff, bitLens, bitOffsets, IMB_DIR_ENCRYPT, i + 1); - else + } else { IMB_SNOW3G_F8_N_BUFFER(mb_mgr, *pKeySched, (const void * const *)pIV, (const void * const *)pSrcBuff, (void **)pDstBuff, packetLen, i + 1); - - if (pDstBuff[0] == NULL) { - printf("N buffer failure\n"); - goto snow3g_f8_n_buffer_exit; + if (pDstBuff[0] == NULL) { + printf("N buffer failure\n"); + goto snow3g_f8_n_buffer_exit; + } } /*Compare the data in the pDstBuff with the cipher pattern*/ @@ -1913,20 +1927,21 @@ validate_snow3g_f8_n_blocks(struct IMB_MGR *mb_mgr, uint32_t job_api, } /*Test the Decrypt*/ - if (job_api) + if (job_api) { submit_uea2_jobs(mb_mgr, (uint8_t **)&pKeySched, pIV, pSrcBuff, pDstBuff, bitLens, bitOffsets, IMB_DIR_DECRYPT, i + 1); - else + } else { IMB_SNOW3G_F8_N_BUFFER(mb_mgr, *pKeySched, (const void * const *)pIV, (const void * const *)pDstBuff, (void **)pSrcBuff, packetLen, i + 1); - if (pSrcBuff[0] == NULL) { - printf("N buffer failure\n"); - goto snow3g_f8_n_buffer_exit; + if (pSrcBuff[0] == NULL) { + printf("N buffer failure\n"); + goto snow3g_f8_n_buffer_exit; + } } /*Compare the data in the pSrcBuff with the pDstBuff*/ @@ -1963,6 +1978,524 @@ snow3g_f8_n_buffer_exit: test_suite_update(uea2_ctx, 1, 0); } +static void +validate_snow3g_f8_n_blocks_linear(struct IMB_MGR *mb_mgr, uint32_t job_api, + struct test_suite_context *uea2_ctx, + struct test_suite_context *uia2_ctx) +{ + int length, numVectors, i, j, numPackets = 16; + size_t size = 0; + cipher_test_linear_vector_t *testVectors = + snow3g_cipher_test_vectors_linear[1]; + /* snow3g f8 test vectors are located at index 1 */ + numVectors = numSnow3gCipherTestVectorsLinear[1]; + + snow3g_key_schedule_t *pKeySched[NUM_SUPPORTED_BUFFERS]; + uint8_t *pKey[NUM_SUPPORTED_BUFFERS]; + uint8_t *pSrcBuff[NUM_SUPPORTED_BUFFERS]; + uint8_t *pDstBuff[NUM_SUPPORTED_BUFFERS]; + uint8_t *pSrcBuff_const[NUM_SUPPORTED_BUFFERS]; + uint8_t *pDstBuff_const[NUM_SUPPORTED_BUFFERS]; + uint8_t *pIV[NUM_SUPPORTED_BUFFERS]; + uint32_t packetLen[NUM_SUPPORTED_BUFFERS]; + uint32_t bitOffsets[NUM_SUPPORTED_BUFFERS]; + uint32_t bitLens[NUM_SUPPORTED_BUFFERS]; + int keyLen = MAX_KEY_LEN; + int status = 0; + + (void)uia2_ctx; +#ifdef DEBUG + printf("Testing IMB_SNOW3G_F8_N_BUFFER: (%s):\n", + job_api ? "Job API" : "Direct API"); +#endif + memset(pSrcBuff, 0, sizeof(pSrcBuff)); + memset(pSrcBuff_const, 0, sizeof(pSrcBuff_const)); + memset(pDstBuff, 0, sizeof(pDstBuff)); + memset(pDstBuff_const, 0, sizeof(pDstBuff_const)); + memset(pIV, 0, sizeof(pIV)); + memset(pKey, 0, sizeof(pKey)); + memset(packetLen, 0, sizeof(packetLen)); + memset(pKeySched, 0, sizeof(pKeySched)); + + if (!numVectors) { + printf("No Snow3G test vectors found !\n"); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + + size = IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr); + if (!size) { + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + + /* Create test Data for num Packets*/ + for (i = 0; i < numPackets; i++) { + /*vectors are in bits used to round up to bytes*/ + length = testVectors[0].dataLenInBytes[i]; + + packetLen[i] = length; + bitLens[i] = length * 8; + bitOffsets[i] = 0; + + pKey[i] = malloc(keyLen); + if (!pKey[i]) { + printf("malloc(pKey[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pKeySched[i] = malloc(size); + if (!pKeySched[i]) { + printf("malloc(pKeySched[%d]): failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pSrcBuff[i] = malloc(length); + if (!pSrcBuff[i]) { + printf("malloc(pSrcBuff[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pDstBuff[i] = malloc(length); + if (!pDstBuff[i]) { + printf("malloc(pDstBuff[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pSrcBuff_const[i] = malloc(length); + if (!pSrcBuff_const[i]) { + printf("malloc(pSrcBuff_const[%d]): failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pDstBuff_const[i] = malloc(length); + if (!pDstBuff_const[i]) { + printf("malloc(pDstBuff_const[%d]): failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + pIV[i] = malloc(SNOW3G_IV_LEN_IN_BYTES); + if (!pIV[i]) { + printf("malloc(pIV[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + + memcpy(pKey[i], testVectors[0].key[i], + testVectors[0].keyLenInBytes); + + memset(pSrcBuff[i], 0, length); + memcpy(pSrcBuff_const[i], testVectors[0].plaintext[i], length); + + memset(pDstBuff[i], 0, length); + memcpy(pDstBuff_const[i], testVectors[0].ciphertext[i], length); + + memcpy(pIV[i], testVectors[0].iv[i], + testVectors[0].ivLenInBytes); + + /* init key shed */ + if (IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, pKey[i], pKeySched[i])) { + printf("IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr) error\n"); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + } + + for (i = 0; i < numPackets; i++) { + const char *fn_name = job_api ? + "submit_uea2_jobs" : "IMB_SNOW3G_F8_N_BUFFER"; + + for (j = 0; j < i; j++) { + /* Cleanup previous values */ + memset(pSrcBuff[j], 0, length); + memset(pDstBuff[j], 0, length); + } + + /*Test the encrypt*/ + if (job_api) { + submit_uea2_jobs(mb_mgr, (uint8_t **)&pKeySched, pIV, + pSrcBuff_const, pDstBuff, bitLens, + bitOffsets, IMB_DIR_ENCRYPT, i + 1); + } else { + IMB_SNOW3G_F8_N_BUFFER(mb_mgr, *pKeySched, + (const void *const *)pIV, + (const void *const *) + pSrcBuff_const, + (void **)pDstBuff, packetLen, + i + 1); + if (pDstBuff[0] == NULL) { + printf("N buffer failure\n"); + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + } + + /*Compare the data in the pDstBuff with the cipher pattern*/ + for (j = 0; j < i; j++) { + if (memcmp(pDstBuff_const[j], pDstBuff[j], + packetLen[j]) != 0) { + printf("%s(Enc) %s nb_packets:%d vector:%d\n", + fn_name, __func__, i, j); + snow3g_hexdump("Actual:", pDstBuff[j], + packetLen[j]); + snow3g_hexdump("Expected:", pDstBuff_const[j], + packetLen[j]); + status = -1; + } + } + + /*Test the Decrypt*/ + if (job_api) { + submit_uea2_jobs(mb_mgr, (uint8_t **)&pKeySched, pIV, + pDstBuff_const, pSrcBuff, bitLens, + bitOffsets, IMB_DIR_DECRYPT, i + 1); + } else { + IMB_SNOW3G_F8_N_BUFFER(mb_mgr, *pKeySched, + (const void *const *)pIV, + (const void *const *) + pDstBuff_const, + (void **)pSrcBuff, packetLen, + i + 1); + if (pSrcBuff[0] == NULL) { + status = -1; + goto snow3g_f8_n_buffer_linear_exit; + } + } + + /*Compare the data in the pSrcBuff with the pDstBuff*/ + for (j = 0; j < i; j++) { + if (memcmp(pSrcBuff[j], pSrcBuff_const[j], + packetLen[j]) != 0) { + printf("%s(Dec) %s nb_packets:%d vector:%d\n", + fn_name, __func__, i, j); + snow3g_hexdump("Actual:", pSrcBuff[j], + packetLen[j]); + snow3g_hexdump("Expected:", pSrcBuff_const[j], + packetLen[j]); + status = -1; + } + } + } + +snow3g_f8_n_buffer_linear_exit: + for (i = 0; i < numPackets; i++) { + if (pKey[i] != NULL) + free(pKey[i]); + if (pKeySched[i] != NULL) + free(pKeySched[i]); + if (pSrcBuff[i] != NULL) + free(pSrcBuff[i]); + if (pDstBuff[i] != NULL) + free(pDstBuff[i]); + if (pSrcBuff_const[i] != NULL) + free(pSrcBuff_const[i]); + if (pDstBuff_const[i] != NULL) + free(pDstBuff_const[i]); + if (pIV[i] != NULL) + free(pIV[i]); + } + + if (status < 0) + test_suite_update(uea2_ctx, 0, 1); + else + test_suite_update(uea2_ctx, 1, 0); +} + +static void +validate_snow3g_f8_n_blocks_linear_mkeys(struct IMB_MGR *mb_mgr, + uint32_t job_api, + struct test_suite_context *uea2_ctx, + struct test_suite_context *uia2_ctx) +{ + int length, numVectors, i, j; + size_t size = 0; + cipher_test_vector_t *testVectors = snow3g_cipher_test_vectors[1]; + /* snow3g f8 random test vectors are located at index 1 */ + numVectors = numSnow3gCipherTestVectors[1]; + + snow3g_key_schedule_t **pKeySched; + uint8_t **pKey; + uint8_t **pSrcBuff; + uint8_t **pDstBuff; + uint8_t **pSrcBuff_const; + uint8_t **pDstBuff_const; + uint8_t **pIV; + uint32_t *packetLen; + uint32_t *bitOffsets; + uint32_t *bitLens; + int status = 0; + + (void)uia2_ctx; +#ifdef DEBUG + printf("Testing IMB_SNOW3G_F8_N_BUFFER_MULTI for usecase %s: (%s):\n", + __func__, job_api ? "Job API" : "Direct API"); +#endif + + pSrcBuff = malloc(sizeof(*pSrcBuff) * numVectors); + pSrcBuff_const = malloc(sizeof(*pSrcBuff_const) * numVectors); + pDstBuff = malloc(sizeof(*pDstBuff) * numVectors); + pDstBuff_const = malloc(sizeof(*pDstBuff_const) * numVectors); + pIV = malloc(sizeof(*pIV) * numVectors); + pKey = malloc(sizeof(*pKey) * numVectors); + pKeySched = malloc(sizeof(*pKeySched) * numVectors); + packetLen = malloc(sizeof(*packetLen) * numVectors); + bitOffsets = malloc(sizeof(*bitOffsets) * numVectors); + bitLens = malloc(sizeof(*bitLens) * numVectors); + + memset(pSrcBuff, 0, sizeof(*pSrcBuff) * numVectors); + memset(pSrcBuff_const, 0, sizeof(*pSrcBuff_const) * numVectors); + memset(pDstBuff, 0, sizeof(*pDstBuff) * numVectors); + memset(pDstBuff_const, 0, sizeof(*pDstBuff_const) * numVectors); + memset(pIV, 0, sizeof(*pIV) * numVectors); + memset(pKey, 0, sizeof(*pKey) * numVectors); + memset(pKeySched, 0, sizeof(*pKeySched) * numVectors); + memset(packetLen, 0, sizeof(*packetLen) * numVectors); + memset(bitOffsets, 0, sizeof(*bitOffsets) * numVectors); + memset(bitLens, 0, sizeof(*bitLens) * numVectors); + + if (!numVectors) { + printf("No Snow3G test vectors found !\n"); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + + size = IMB_SNOW3G_KEY_SCHED_SIZE(mb_mgr); + if (!size) { + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + + /* Create test Data for num Vectors */ + for (i = 0; i < numVectors; i++) { + /*vectors are in bits used to round up to bytes*/ + length = testVectors[i].dataLenInBytes; + + packetLen[i] = length; + bitLens[i] = length * 8; + bitOffsets[i] = 0; + + pKey[i] = malloc(testVectors[i].keyLenInBytes); + if (!pKey[i]) { + printf("malloc(pKey[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pKeySched[i] = malloc(size); + if (!pKeySched[i]) { + printf("malloc(pKeySched[%d]): failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pSrcBuff[i] = malloc(length); + if (!pSrcBuff[i]) { + printf("malloc(pSrcBuff[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pDstBuff[i] = malloc(length); + if (!pDstBuff[i]) { + printf("malloc(pDstBuff[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pSrcBuff_const[i] = malloc(length); + if (!pSrcBuff_const[i]) { + printf("malloc(pSrcBuff_const[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pDstBuff_const[i] = malloc(length); + if (!pDstBuff_const[i]) { + printf("malloc(pDstBuff_const[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + pIV[i] = malloc(testVectors[i].ivLenInBytes); + if (!pIV[i]) { + printf("malloc(pIV[%d]):failed !\n", i); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + + memcpy(pKey[i], testVectors[i].key, + testVectors[i].keyLenInBytes); + + memset(pSrcBuff[i], 0, length); + memcpy(pSrcBuff_const[i], testVectors[i].plaintext, length); + + memset(pDstBuff[i], 0, length); + memcpy(pDstBuff_const[i], testVectors[i].ciphertext, length); + + memcpy(pIV[i], testVectors[i].iv, testVectors[i].ivLenInBytes); + + /* init key shed */ + if (IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, pKey[i], pKeySched[i])) { + printf("IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr) error\n"); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + } + + for (i = 0; i < numVectors; i++) { + int nb_elem, nb_remain_elem = i + 1, idx = 0; + const char *fn_name = job_api ? + "submit_uea2_jobs" : "IMB_SNOW3G_F8_N_BUFFER_MULTIKEY"; + + for (j = 0; j <= i; j++) { + /* Cleanup previous values */ + memset(pSrcBuff[j], 0, packetLen[j]); + memset(pDstBuff[j], 0, packetLen[j]); + } + + /*Test the encrypt*/ + while (nb_remain_elem > 0) { + if (nb_remain_elem >= NUM_SUPPORTED_BUFFERS) { + nb_elem = NUM_SUPPORTED_BUFFERS; + nb_remain_elem -= NUM_SUPPORTED_BUFFERS; + } else { + nb_elem = nb_remain_elem; + nb_remain_elem = 0; + } + + if (job_api) { + submit_uea2_jobs(mb_mgr, + (uint8_t **)&pKeySched[idx], + &pIV[idx], + &pSrcBuff_const[idx], + &pDstBuff[idx], &bitLens[idx], + &bitOffsets[idx], + IMB_DIR_ENCRYPT, nb_elem); + } else { + IMB_SNOW3G_F8_N_BUFFER_MULTIKEY( + mb_mgr, + (const snow3g_key_schedule_t *const *) + &pKeySched[idx], + (const void *const *)&pIV[idx], + (const void *const *)&pSrcBuff_const[idx], + (void **)&pDstBuff[idx], &packetLen[idx], + nb_elem); + if (pDstBuff[idx] == NULL) { + printf("N buffer failure\n"); + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + } + + if (nb_elem == NUM_SUPPORTED_BUFFERS) + idx += NUM_SUPPORTED_BUFFERS; + } + + /*Compare the data in the pDstBuff with the cipher pattern*/ + for (j = 0; j <= i; j++) { + if (memcmp(pDstBuff_const[j], pDstBuff[j], + packetLen[j]) != 0) { + printf("%s(Enc) %s nb_packets:%d vector: %d\n", + fn_name, __func__, i, j); + snow3g_hexdump("Actual:", pDstBuff[j], + packetLen[j]); + snow3g_hexdump("Expected:", pDstBuff_const[j], + packetLen[j]); + status = -1; + } + } + + nb_remain_elem = i + 1; + idx = 0; + while (nb_remain_elem > 0) { + if (nb_remain_elem >= NUM_SUPPORTED_BUFFERS) { + nb_elem = NUM_SUPPORTED_BUFFERS; + nb_remain_elem -= NUM_SUPPORTED_BUFFERS; + } else { + nb_elem = nb_remain_elem; + nb_remain_elem = 0; + } + /*Test the Decrypt*/ + if (job_api) { + submit_uea2_jobs(mb_mgr, + (uint8_t **)&pKeySched[idx], + &pIV[idx], + &pDstBuff_const[idx], + &pSrcBuff[idx], &bitLens[idx], + &bitOffsets[idx], + IMB_DIR_DECRYPT, nb_elem); + } else { + IMB_SNOW3G_F8_N_BUFFER_MULTIKEY( + mb_mgr, + (const snow3g_key_schedule_t *const *) + &pKeySched[idx], + (const void *const *)&pIV[idx], + (const void *const *)&pDstBuff_const[idx], + (void **)&pSrcBuff[idx], &packetLen[idx], + nb_elem); + if (pSrcBuff[idx] == NULL) { + status = -1; + goto snow3g_f8_n_buff_linear_mkey_exit; + } + } + + if (nb_elem == NUM_SUPPORTED_BUFFERS) + idx += NUM_SUPPORTED_BUFFERS; + } + /*Compare the data in the pSrcBuff with the pDstBuff*/ + for (j = 0; j <= i; j++) { + if (memcmp(pSrcBuff[j], pSrcBuff_const[j], + packetLen[j]) != 0) { + printf("%s(Dec) %s nb_packets:%d vector: %d\n", + fn_name, __func__, i, j); + snow3g_hexdump("Actual:", pSrcBuff[j], + packetLen[j]); + snow3g_hexdump("Expected:", pSrcBuff_const[j], + packetLen[j]); + status = -1; + } + } + } + +snow3g_f8_n_buff_linear_mkey_exit: + for (i = 0; i < numVectors; i++) { + if (pKey[i] != NULL) + free(pKey[i]); + if (pKeySched[i] != NULL) + free(pKeySched[i]); + if (pSrcBuff[i] != NULL) + free(pSrcBuff[i]); + if (pDstBuff[i] != NULL) + free(pDstBuff[i]); + if (pSrcBuff_const[i] != NULL) + free(pSrcBuff_const[i]); + if (pDstBuff_const[i] != NULL) + free(pDstBuff_const[i]); + if (pIV[i] != NULL) + free(pIV[i]); + } + + if (pKey != NULL) + free(pKey); + if (pSrcBuff != NULL) + free(pSrcBuff); + if (pSrcBuff_const != NULL) + free(pSrcBuff_const); + if (pDstBuff != NULL) + free(pDstBuff); + if (pDstBuff_const != NULL) + free(pDstBuff_const); + if (pIV != NULL) + free(pIV); + if (packetLen != NULL) + free(packetLen); + if (pKeySched != NULL) + free(pKeySched); + if (bitOffsets != NULL) + free(bitOffsets); + if (bitLens != NULL) + free(bitLens); + + if (status < 0) + test_suite_update(uea2_ctx, 0, 1); + else + test_suite_update(uea2_ctx, 1, 0); +} + static void validate_snow3g_f8_n_blocks_multi(struct IMB_MGR *mb_mgr, uint32_t job_api, diff --git a/test/snow3g_test_vectors.h b/test/snow3g_test_vectors.h index bd444319..a95a9e6b 100644 --- a/test/snow3g_test_vectors.h +++ b/test/snow3g_test_vectors.h @@ -1,5 +1,6 @@ /***************************************************************************** Copyright (c) 2009-2022, Intel Corporation + Copyright (c) 2022, Nokia Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -56,6 +57,15 @@ typedef struct cipher_test_vector_s { uint8_t key[MAX_KEY_LEN]; uint8_t iv[MAX_IV_LEN]; } cipher_test_vector_t; +typedef struct cipher_test_linear_vector_s { + uint32_t dataLenInBytes[NUM_SUPPORTED_BUFFERS]; + uint32_t keyLenInBytes; + uint32_t ivLenInBytes; + uint8_t plaintext[NUM_SUPPORTED_BUFFERS][MAX_DATA_LEN]; + uint8_t ciphertext[NUM_SUPPORTED_BUFFERS][MAX_DATA_LEN]; + uint8_t key[NUM_SUPPORTED_BUFFERS][MAX_KEY_LEN]; + uint8_t iv[NUM_SUPPORTED_BUFFERS][MAX_IV_LEN]; +} cipher_test_linear_vector_t; typedef struct cipherbit_test_vector_s { uint32_t dataLenInBits; uint32_t keyLenInBytes; @@ -254,6 +264,176 @@ static cipherbit_test_linear_vector_t snow3g_f8_linear_bitvectors = { } }; +static cipher_test_linear_vector_t snow3g_f8_linear[] = { + { + /*dataLenInBytes*/ + { 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32 }, + /*keyLenInBytes*/ + 16, + /*ivLenInBytes*/ + 16, + { /*plaintext linear bit bit buffer*/ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { /*ciphertext linear bit buffer*/ + { 0xea, 0xe9, 0x11, 0x49, 0x21, 0x58, 0x44, 0x59, 0xc6, 0xec, 0x77, + 0x82, 0x71, 0x6f, 0x91, 0xd4, 0xe1, 0xf6, 0xbf, 0xfa, 0x45, 0xfc, + 0x80, 0x94, 0xc2, 0x3d, 0x5c, 0x5f, 0x17, 0x72, 0x44, 0x92 }, + { 0xe3, 0x96, 0x7f, 0x63, 0x88, 0x74, 0x2d, 0x7c, 0x3b, 0x3d, 0x5a, + 0x08, 0xc0, 0x8d, 0x1a, 0xc2, 0xa1, 0xf2, 0x38, 0x96, 0xc7, 0x41, + 0x1b, 0xc1, 0xde, 0xac, 0x24, 0x57, 0x9c, 0x45, 0x0d, 0xb0 }, + { 0x4d, 0x50, 0xd2, 0xe3, 0xa6, 0xe7, 0x97, 0xee, 0xf3, 0xe6, 0x2a, + 0x32, 0xf3, 0x66, 0xea, 0x99, 0xbc, 0x54, 0x24, 0xa6, 0x3f, 0xc3, + 0xbb, 0x27, 0xc7, 0xba, 0x42, 0xd8, 0x2f, 0x86, 0xfc, 0xdf }, + { 0x4a, 0x98, 0xa3, 0x04, 0xad, 0xcf, 0xb9, 0xfa, 0xdb, 0x3b, 0xe7, + 0x7c, 0xd1, 0x69, 0x59, 0x6c, 0x7e, 0x44, 0x73, 0x8d, 0x96, 0xd0, + 0x51, 0x1f, 0x55, 0x32, 0x55, 0x59, 0xc2, 0xe9, 0x0a, 0x21 }, + { 0xe4, 0xf5, 0x71, 0x42, 0xaf, 0x93, 0x5e, 0x6d, 0x7d, 0xb8, 0x44, + 0xf7, 0xcb, 0x0d, 0x08, 0xd3, 0x3a, 0xe6, 0x20, 0xcd, 0xc5, 0x33, + 0xa3, 0x5a, 0x5e, 0x5a, 0x41, 0x92, 0x96, 0x44, 0x28, 0xc1 }, + { 0x1b, 0x6e, 0x3a, 0x76, 0x18, 0x77, 0x0b, 0x77, 0xf2, 0xda, 0xca, + 0x1d, 0xd2, 0x9c, 0xa9, 0xc2, 0x10, 0x6b, 0xe7, 0x4b, 0xdb, 0x30, + 0x79, 0xc5, 0x56, 0xf4, 0xcb, 0xb7, 0x19, 0xdf, 0xe5, 0xcb }, + { 0x10, 0x4e, 0x19, 0x91, 0x09, 0x91, 0xd0, 0x9d, 0x41, 0x99, 0x33, + 0x1b, 0xfd, 0xa0, 0xe0, 0xf7, 0x80, 0x6c, 0x1f, 0x7b, 0x12, 0x78, + 0x3d, 0x46, 0x65, 0x86, 0x7e, 0xdf, 0x9f, 0xac, 0x31, 0x42 }, + { 0x59, 0x02, 0x9f, 0xe6, 0x98, 0x24, 0xe3, 0xc5, 0x1b, 0x19, 0x61, + 0x08, 0x37, 0x79, 0xd0, 0xfe, 0xe4, 0xa1, 0xd8, 0xd4, 0x04, 0xf2, + 0x43, 0x60, 0xa4, 0x68, 0x08, 0x9b, 0x06, 0x9c, 0xb3, 0x51 }, + { 0x60, 0x59, 0x72, 0x8f, 0xd0, 0x0b, 0x69, 0x67, 0xdf, 0x89, 0xb0, + 0xd1, 0xc3, 0x02, 0x3e, 0xe4, 0x5b, 0xde, 0xe2, 0x0a, 0xea, 0x67, + 0xc6, 0x05, 0x40, 0x4f, 0xe9, 0x38, 0xd4, 0x10, 0x1d, 0x5a }, + { 0x46, 0x63, 0x8d, 0x94, 0x5a, 0xcc, 0x87, 0x2d, 0x57, 0x4f, 0xcb, + 0x55, 0x71, 0xaa, 0x6c, 0x17, 0xcb, 0x7c, 0x7a, 0x6d, 0x19, 0xbf, + 0xba, 0xcd, 0xe1, 0x3c, 0x77, 0xed, 0x0f, 0x10, 0x4f, 0x38 }, + { 0xe5, 0xdd, 0xa1, 0x58, 0xbe, 0x47, 0x6e, 0xa6, 0x3b, 0xea, 0x90, + 0x02, 0xf6, 0x87, 0xbd, 0x69, 0x1b, 0x15, 0xb7, 0x3c, 0x6f, 0xa1, + 0x28, 0xde, 0x0f, 0xf3, 0x80, 0xfb, 0xef, 0x89, 0xa5, 0xd5 }, + { 0xae, 0x68, 0x9e, 0x4e, 0x8d, 0x49, 0x2a, 0x59, 0x5c, 0x02, 0x53, + 0x73, 0x91, 0xb2, 0x92, 0x94, 0xc3, 0x7e, 0x09, 0xae, 0xe3, 0x6d, + 0x3c, 0xea, 0xac, 0x57, 0x34, 0x19, 0x82, 0x56, 0x10, 0x2e }, + { 0xd4, 0x35, 0x4b, 0x73, 0x1b, 0x6e, 0xa3, 0x4c, 0xd5, 0x9b, 0x10, + 0xd0, 0x93, 0x64, 0xc9, 0xdf, 0xe0, 0xe5, 0x2e, 0xa0, 0x8c, 0x2d, + 0x85, 0x38, 0x53, 0xaa, 0x79, 0x92, 0xad, 0xae, 0x36, 0xb3 }, + { 0xdb, 0x9b, 0x66, 0x28, 0xf7, 0x9d, 0x37, 0x32, 0xbe, 0x36, 0x78, + 0x8d, 0xa3, 0xd0, 0xc2, 0x73, 0x68, 0x0d, 0x47, 0xf7, 0x13, 0x8f, + 0x3d, 0x83, 0x2a, 0xcf, 0x5e, 0xe1, 0xb8, 0x6d, 0x86, 0xb8 }, + { 0xd5, 0xbf, 0x80, 0x07, 0x53, 0x33, 0xad, 0x71, 0x9a, 0x05, 0x70, + 0xd9, 0xe7, 0xdd, 0x45, 0xf0, 0xd0, 0x2d, 0xaf, 0xdc, 0xf1, 0x12, + 0x67, 0x8f, 0x46, 0x20, 0xac, 0xb4, 0xd7, 0xf8, 0x98, 0x41 }, + { 0xd5, 0xbf, 0x80, 0x07, 0x53, 0x33, 0xad, 0x71, 0x9a, 0x05, 0x70, + 0xd9, 0xe7, 0xdd, 0x45, 0xf0, 0xd0, 0x2d, 0xaf, 0xdc, 0xf1, 0x12, + 0x67, 0x8f, 0x46, 0x20, 0xac, 0xb4, 0xd7, 0xf8, 0x98, 0x41 } }, + { /*key buffers*/ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00 } }, + { /* IV buffers*/ + { 0x00, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x01, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x02, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x03, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x04, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x05, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x06, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x07, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x08, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x09, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x09, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0A, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0A, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0B, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0B, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0C, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0C, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0D, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0D, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0E, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0E, 0x1C, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x0E, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0E, 0x1C, 0x00, 0x00, 0x00 } } } }; + static cipher_test_vector_t snow3g_f8_vectors[] = { {/* SCPM test vector */ /*dataLenInBytes*/ @@ -432,7 +612,254 @@ static cipher_test_vector_t snow3g_f8_vectors[] = { 0x2B, 0xE8, 0xE3, 0x65, 0x66}, /*iv*/ {0x72, 0xA4, 0xF2, 0x0F, 0x48, 0x00, 0x00, 0x00, 0x72, 0xA4, 0xF2, - 0x0F, 0x48, 0x00, 0x00, 0x00} } }; + 0x0F, 0x48, 0x00, 0x00, 0x00} }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xea, 0xe9, 0x11, 0x49, 0x21, 0x58, 0x44, 0x59, 0xc6, 0xec, 0x77, + 0x82, 0x71, 0x6f, 0x91, 0xd4, 0xe1, 0xf6, 0xbf, 0xfa, 0x45, 0xfc, + 0x80, 0x94, 0xc2, 0x3d, 0x5c, 0x5f, 0x17, 0x72, 0x44, 0x92}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xe3, 0x96, 0x7f, 0x63, 0x88, 0x74, 0x2d, 0x7c, 0x3b, 0x3d, 0x5a, + 0x08, 0xc0, 0x8d, 0x1a, 0xc2, 0xa1, 0xf2, 0x38, 0x96, 0xc7, 0x41, + 0x1b, 0xc1, 0xde, 0xac, 0x24, 0x57, 0x9c, 0x45, 0x0d, 0xb0}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x01, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x4d, 0x50, 0xd2, 0xe3, 0xa6, 0xe7, 0x97, 0xee, 0xf3, 0xe6, 0x2a, + 0x32, 0xf3, 0x66, 0xea, 0x99, 0xbc, 0x54, 0x24, 0xa6, 0x3f, 0xc3, + 0xbb, 0x27, 0xc7, 0xba, 0x42, 0xd8, 0x2f, 0x86, 0xfc, 0xdf}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x02, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x4a, 0x98, 0xa3, 0x04, 0xad, 0xcf, 0xb9, 0xfa, 0xdb, 0x3b, 0xe7, + 0x7c, 0xd1, 0x69, 0x59, 0x6c, 0x7e, 0x44, 0x73, 0x8d, 0x96, 0xd0, + 0x51, 0x1f, 0x55, 0x32, 0x55, 0x59, 0xc2, 0xe9, 0x0a, 0x21}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x03, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xe4, 0xf5, 0x71, 0x42, 0xaf, 0x93, 0x5e, 0x6d, 0x7d, 0xb8, 0x44, + 0xf7, 0xcb, 0x0d, 0x08, 0xd3, 0x3a, 0xe6, 0x20, 0xcd, 0xc5, 0x33, + 0xa3, 0x5a, 0x5e, 0x5a, 0x41, 0x92, 0x96, 0x44, 0x28, 0xc1}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x04, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x1b, 0x6e, 0x3a, 0x76, 0x18, 0x77, 0x0b, 0x77, 0xf2, 0xda, 0xca, + 0x1d, 0xd2, 0x9c, 0xa9, 0xc2, 0x10, 0x6b, 0xe7, 0x4b, 0xdb, 0x30, + 0x79, 0xc5, 0x56, 0xf4, 0xcb, 0xb7, 0x19, 0xdf, 0xe5, 0xcb}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x05, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x10, 0x4e, 0x19, 0x91, 0x09, 0x91, 0xd0, 0x9d, 0x41, 0x99, 0x33, + 0x1b, 0xfd, 0xa0, 0xe0, 0xf7, 0x80, 0x6c, 0x1f, 0x7b, 0x12, 0x78, + 0x3d, 0x46, 0x65, 0x86, 0x7e, 0xdf, 0x9f, 0xac, 0x31, 0x42}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x06, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x59, 0x02, 0x9f, 0xe6, 0x98, 0x24, 0xe3, 0xc5, 0x1b, 0x19, 0x61, + 0x08, 0x37, 0x79, 0xd0, 0xfe, 0xe4, 0xa1, 0xd8, 0xd4, 0x04, 0xf2, + 0x43, 0x60, 0xa4, 0x68, 0x08, 0x9b, 0x06, 0x9c, 0xb3, 0x51}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x07, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x07, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x60, 0x59, 0x72, 0x8f, 0xd0, 0x0b, 0x69, 0x67, 0xdf, 0x89, 0xb0, + 0xd1, 0xc3, 0x02, 0x3e, 0xe4, 0x5b, 0xde, 0xe2, 0x0a, 0xea, 0x67, + 0xc6, 0x05, 0x40, 0x4f, 0xe9, 0x38, 0xd4, 0x10, 0x1d, 0x5a}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x08, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x46, 0x63, 0x8d, 0x94, 0x5a, 0xcc, 0x87, 0x2d, 0x57, 0x4f, 0xcb, + 0x55, 0x71, 0xaa, 0x6c, 0x17, 0xcb, 0x7c, 0x7a, 0x6d, 0x19, 0xbf, + 0xba, 0xcd, 0xe1, 0x3c, 0x77, 0xed, 0x0f, 0x10, 0x4f, 0x38}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x09, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x09, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xe5, 0xdd, 0xa1, 0x58, 0xbe, 0x47, 0x6e, 0xa6, 0x3b, 0xea, 0x90, + 0x02, 0xf6, 0x87, 0xbd, 0x69, 0x1b, 0x15, 0xb7, 0x3c, 0x6f, 0xa1, + 0x28, 0xde, 0x0f, 0xf3, 0x80, 0xfb, 0xef, 0x89, 0xa5, 0xd5}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0A, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0A, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xae, 0x68, 0x9e, 0x4e, 0x8d, 0x49, 0x2a, 0x59, 0x5c, 0x02, 0x53, + 0x73, 0x91, 0xb2, 0x92, 0x94, 0xc3, 0x7e, 0x09, 0xae, 0xe3, 0x6d, + 0x3c, 0xea, 0xac, 0x57, 0x34, 0x19, 0x82, 0x56, 0x10, 0x2e}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0B, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0B, 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xd4, 0x35, 0x4b, 0x73, 0x1b, 0x6e, 0xa3, 0x4c, 0xd5, 0x9b, 0x10, + 0xd0, 0x93, 0x64, 0xc9, 0xdf, 0xe0, 0xe5, 0x2e, 0xa0, 0x8c, 0x2d, + 0x85, 0x38, 0x53, 0xaa, 0x79, 0x92, 0xad, 0xae, 0x36, 0xb3}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0C, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0C, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xdb, 0x9b, 0x66, 0x28, 0xf7, 0x9d, 0x37, 0x32, 0xbe, 0x36, 0x78, + 0x8d, 0xa3, 0xd0, 0xc2, 0x73, 0x68, 0x0d, 0x47, 0xf7, 0x13, 0x8f, + 0x3d, 0x83, 0x2a, 0xcf, 0x5e, 0xe1, 0xb8, 0x6d, 0x86, 0xb8}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0D, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0D, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0xd5, 0xbf, 0x80, 0x07, 0x53, 0x33, 0xad, 0x71, 0x9a, 0x05, 0x70, + 0xd9, 0xe7, 0xdd, 0x45, 0xf0, 0xd0, 0x2d, 0xaf, 0xdc, 0xf1, 0x12, + 0x67, 0x8f, 0x46, 0x20, 0xac, 0xb4, 0xd7, 0xf8, 0x98, 0x41}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0E, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0E, + 0x1C, 0x00, 0x00, 0x00} + }, + { + 32, + 16, + 16, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x62, 0x45, 0xff, 0xfa, 0x89, 0x47, 0x18, 0x63, 0x28, 0x4e, 0xd5, + 0xf2, 0x94, 0xb0, 0x54, 0x43, 0xd5, 0xae, 0xb5, 0x68, 0x2f, 0xf6, + 0x7c, 0x81, 0xe3, 0xc5, 0x81, 0x32, 0x0e, 0x59, 0xc5, 0x60}, + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x00, 0x0F, 0x1C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0F, 0x1C, 0x00, 0x00, 0x00} + } +}; static hash_test_vector_t snow_f9_vectors[] = { { @@ -794,6 +1221,15 @@ uint32_t numSnow3gCipherTestVectors[] = { sizeof(snow3g_f8_linear_bitvectors) / sizeof(cipherbit_test_vector_t) }; +cipher_test_linear_vector_t *snow3g_cipher_test_vectors_linear[] = { + snow3g_f8_linear, snow3g_f8_linear +}; + +uint32_t numSnow3gCipherTestVectorsLinear[] = { + sizeof(snow3g_f8_linear) / sizeof(cipher_test_linear_vector_t), + sizeof(snow3g_f8_linear) / sizeof(cipher_test_linear_vector_t) +}; + hash_test_vector_t *snow3g_hash_test_vectors[] = { snow_f9_vectors, snow_f9_vectors, snow_f9_vectors }; -- GitLab From 39dfb2867cfb5ace0d71d26f6339d847d44eeb1d Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 6 May 2022 10:35:43 +0000 Subject: [PATCH 079/369] [CI]: update workflow names --- .../{freebsd_make.yml => freebsd_build_shared_clang.yml} | 2 +- .../{freebsd_gcc_make.yml => freebsd_build_shared_gcc.yml} | 2 +- .../workflows/{clang_make.yml => linux_build_shared_clang.yml} | 2 +- ...ng_aesni_make.yml => linux_build_shared_clang_aesni_emu.yml} | 2 +- .../workflows/{linux_make.yml => linux_build_shared_gcc.yml} | 2 +- .../{clang_static_make.yml => linux_build_static_clang.yml} | 2 +- .../{linux_static_make.yml => linux_build_static_gcc.yml} | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{freebsd_make.yml => freebsd_build_shared_clang.yml} (94%) rename .github/workflows/{freebsd_gcc_make.yml => freebsd_build_shared_gcc.yml} (94%) rename .github/workflows/{clang_make.yml => linux_build_shared_clang.yml} (90%) rename .github/workflows/{clang_aesni_make.yml => linux_build_shared_clang_aesni_emu.yml} (87%) rename .github/workflows/{linux_make.yml => linux_build_shared_gcc.yml} (90%) rename .github/workflows/{clang_static_make.yml => linux_build_static_clang.yml} (91%) rename .github/workflows/{linux_static_make.yml => linux_build_static_gcc.yml} (91%) diff --git a/.github/workflows/freebsd_make.yml b/.github/workflows/freebsd_build_shared_clang.yml similarity index 94% rename from .github/workflows/freebsd_make.yml rename to .github/workflows/freebsd_build_shared_clang.yml index d821219a..3ecc0e46 100644 --- a/.github/workflows/freebsd_make.yml +++ b/.github/workflows/freebsd_build_shared_clang.yml @@ -1,4 +1,4 @@ -name: FreeBSD make +name: FreeBSD Build Shared clang on: push: diff --git a/.github/workflows/freebsd_gcc_make.yml b/.github/workflows/freebsd_build_shared_gcc.yml similarity index 94% rename from .github/workflows/freebsd_gcc_make.yml rename to .github/workflows/freebsd_build_shared_gcc.yml index 106c7d27..4ae7e405 100644 --- a/.github/workflows/freebsd_gcc_make.yml +++ b/.github/workflows/freebsd_build_shared_gcc.yml @@ -1,4 +1,4 @@ -name: FreeBSD GCC make +name: FreeBSD Build Shared gcc on: push: diff --git a/.github/workflows/clang_make.yml b/.github/workflows/linux_build_shared_clang.yml similarity index 90% rename from .github/workflows/clang_make.yml rename to .github/workflows/linux_build_shared_clang.yml index afa58dbc..193fdefc 100644 --- a/.github/workflows/clang_make.yml +++ b/.github/workflows/linux_build_shared_clang.yml @@ -1,4 +1,4 @@ -name: Clang make +name: Linux Build Shared clang on: push: diff --git a/.github/workflows/clang_aesni_make.yml b/.github/workflows/linux_build_shared_clang_aesni_emu.yml similarity index 87% rename from .github/workflows/clang_aesni_make.yml rename to .github/workflows/linux_build_shared_clang_aesni_emu.yml index de5292e3..2454c76a 100644 --- a/.github/workflows/clang_aesni_make.yml +++ b/.github/workflows/linux_build_shared_clang_aesni_emu.yml @@ -1,4 +1,4 @@ -name: Clang aesni make +name: Linux Build Shared clang AESNI emulation on: push: diff --git a/.github/workflows/linux_make.yml b/.github/workflows/linux_build_shared_gcc.yml similarity index 90% rename from .github/workflows/linux_make.yml rename to .github/workflows/linux_build_shared_gcc.yml index c490dec4..f356cfd0 100644 --- a/.github/workflows/linux_make.yml +++ b/.github/workflows/linux_build_shared_gcc.yml @@ -1,4 +1,4 @@ -name: Linux make +name: Linux Build Shared gcc on: push: diff --git a/.github/workflows/clang_static_make.yml b/.github/workflows/linux_build_static_clang.yml similarity index 91% rename from .github/workflows/clang_static_make.yml rename to .github/workflows/linux_build_static_clang.yml index 78c27115..0e5e678a 100644 --- a/.github/workflows/clang_static_make.yml +++ b/.github/workflows/linux_build_static_clang.yml @@ -1,4 +1,4 @@ -name: Clang static make +name: Linux Build Static clang on: push: diff --git a/.github/workflows/linux_static_make.yml b/.github/workflows/linux_build_static_gcc.yml similarity index 91% rename from .github/workflows/linux_static_make.yml rename to .github/workflows/linux_build_static_gcc.yml index 7655ab15..a4941bbc 100644 --- a/.github/workflows/linux_static_make.yml +++ b/.github/workflows/linux_build_static_gcc.yml @@ -1,4 +1,4 @@ -name: Linux static make +name: Linux Build Static gcc on: push: -- GitLab From 0c3832f3696205e690897593f26716cdd2237b64 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 6 May 2022 09:00:00 +0000 Subject: [PATCH 080/369] doc: github workflow badges added to README - mark down fixes for text README version --- README | 17 +++++++++++------ README.md | 42 +++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/README b/README index 3499ac1a..0d522121 100644 --- a/README +++ b/README @@ -1,4 +1,7 @@ -[Coverity Status] [Linux Build Status] +[Coverity Status] [Linux Build Shared gcc] [Linux Build Static gcc] +[Linux Build Shared clang] [Linux Build Static clang] [Linux Build +Shared clang AESNI emulation] [FreeBSD Build Shared clang] [FreeBSD +Build Shared gcc] @@ -364,8 +367,9 @@ Build without safety features: registers or internal data structures - SAFE_PARAM adds extra checks on input parameters - SAFE_LOOKUP uses constant-time lookups (enabled by default) -- SAFE_OPTIONS additional option to disable all safe options. -disable to turn off SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP (enabled by default) +- SAFE_OPTIONS additional option to disable all safe options. Enabled by +default. +Disable to turn off: SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP. > nmake /f win_x64.mak SAFE_DATA=n SAFE_PARAM=n > nmake /f win_x64.mak SAFE_OPTIONS=n @@ -521,9 +525,10 @@ which could expose the cryptographic key. SAFE_OPTIONS -SAFE_OPTIONS is a parameter that can be used to disable -all other safe options(SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). By just -setting this parameter (e.g. SAFE_OPTIONS=n). +SAFE_OPTIONS is a parameter that can be used to disable/enable all +supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). It is +set to y by default and all safe options are enabled. SAFE_OPTIONS=n +disables all safe options. Security API diff --git a/README.md b/README.md index a928cf5b..22557bc3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ [![Coverity Status](https://scan.coverity.com/projects/16449/badge.svg)](https://scan.coverity.com/projects/intel-ipsec-mb) -![Linux Build Status](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_make.yml/badge.svg) +![Linux Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_make.yml/badge.svg) +![Linux Build Static gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_static_make.yml/badge.svg) +![Linux Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_make.yml/badge.svg) +![Linux Build Static clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_static_make.yml/badge.svg) +![Linux Build Shared clang AESNI emulation](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_aesni_make.yml/badge.svg) +![FreeBSD Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_make.yml/badge.svg) +![FreeBSD Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_gcc_make.yml/badge.svg) # Intel(R) Multi-Buffer Crypto for IPsec Library @@ -66,9 +72,9 @@ Table 1. List of supported cipher algorithms and their implementations. | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | -| AES128-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | -| AES192-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | -| AES256-ECB | N | Y(1) | Y by4 | Y(10) | N | Y by16 | +| AES128-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | +| AES192-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | +| AES256-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | | NULL | Y | N | N | N | N | N | | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | @@ -95,9 +101,9 @@ Notes: (6) - decryption is by16 and encryption is x16 (7) - same as AES128-CBC for AVX, combines cipher and CRC32 (8) - decryption is by16 and encryption is x16 -(9) - currently 1:9 crypt:skip pattern supported -(10) - by default, decryption and encryption are AVX by8. - On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16. +(9) - currently 1:9 crypt:skip pattern supported +(10) - by default, decryption and encryption are AVX by8. + On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16. Legend: ` byY` - single buffer Y blocks at a time @@ -160,8 +166,8 @@ Notes: - CRC8: WIMAX OFDMA HCS - CRC7: FP header - CRC6: IUUP header -(7) - used only with PON-AES128-CTR cipher -(8) - x16 for init keystream generation, then by32 +(7) - used only with PON-AES128-CTR cipher +(8) - x16 for init keystream generation, then by32 Legend: ` byY`- single buffer Y blocks at a time @@ -312,8 +318,8 @@ or Build with debugging information: `> make DEBUG=y` -Build with AESNI emulation support (disabled by default): - > make AESNI_EMU=y +Build with AESNI emulation support (disabled by default): +`> make AESNI_EMU=y` **Note:** Building with debugging information is not advised for production use. @@ -347,7 +353,8 @@ Build without safety features: - SAFE_DATA clears sensitive information stored temporarily on stack, registers or internal data structures - SAFE_PARAM adds extra checks on input parameters - SAFE_LOOKUP uses constant-time lookups (enabled by default) -- SAFE_OPTIONS additional option to disable all safe options. disable to turn off SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP (enabled by default) +- SAFE_OPTIONS additional option to disable all safe options. Enabled by default. + Disable to turn off: SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP. `> nmake /f win_x64.mak SAFE_DATA=n SAFE_PARAM=n` `> nmake /f win_x64.mak SAFE_OPTIONS=n` @@ -355,8 +362,8 @@ Build without safety features: Build with debugging information: `> nmake /f win_x64.mak DEBUG=y` -Build with AESNI emulation support (disabled by default): - > nmake /f win_x64.mak AESNI_EMU=y +Build with AESNI emulation support (disabled by default): +`> nmake /f win_x64.mak AESNI_EMU=y` **Note:** Building with debugging information is not advised for production use. @@ -487,9 +494,10 @@ algorithms listed above may be susceptible to timing attacks which could expose the cryptographic key. ### SAFE_OPTIONS -SAFE_OPTIONS is a parameter that can be used to disable -all other safe options(SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). By just -setting this parameter (e.g. SAFE_OPTIONS=n). +SAFE_OPTIONS is a parameter that can be used to disable/enable +all supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). +It is set to `y` by default and all safe options are enabled. +`SAFE_OPTIONS=n` disables all safe options. ### Security API **Force clearing/zeroing of memory** -- GitLab From 430a383c7d9e0a8ed605203294a722ce302a6527 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 6 May 2022 13:40:56 +0000 Subject: [PATCH 081/369] doc: algorithm recommendations update --- README | 24 +++++++++++++----------- README.md | 24 +++++++++++++----------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/README b/README index 0d522121..f737a0d5 100644 --- a/README +++ b/README @@ -260,17 +260,19 @@ Legacy or to be avoided algorithms listed in the table below are implemented in the library in order to support legacy applications. Please use corresponding alternative algorithms instead. - +-------------------------------------------------------------+ - | # | Algorithm | Recommendation | Alternative | - |---+--------------------+----------------+-------------------| - | 1 | DES encryption | Avoid | AES encryption | - |---+--------------------+----------------+-------------------| - | 2 | 3DES encryption | Avoid | AES encryption | - |---+--------------------+----------------+-------------------| - | 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA1 | - |---+--------------------+----------------+-------------------| - | 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR | - +-------------------------------------------------------------+ + +--------------------------------------------------------------+ + | # | Algorithm | Recommendation | Alternative | + |---+---------------------+----------------+-------------------| + | 1 | DES encryption | Avoid | AES encryption | + |---+---------------------+----------------+-------------------| + | 2 | 3DES encryption | Avoid | AES encryption | + |---+---------------------+----------------+-------------------| + | 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA256 | + |---+---------------------+----------------+-------------------| + | 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR | + |---+---------------------+----------------+-------------------| + | 3 | HMAC-SHA1 integrity | Avoid | HMAC-SHA256 | + +--------------------------------------------------------------+ Intel(R) Multi-Buffer Crypto for IPsec Library depends on C library and it is recommended to use its latest version. diff --git a/README.md b/README.md index 22557bc3..afe6f554 100644 --- a/README.md +++ b/README.md @@ -253,17 +253,19 @@ Legacy or to be avoided algorithms listed in the table below are implemented in the library in order to support legacy applications. Please use corresponding alternative algorithms instead. ``` -+-------------------------------------------------------------+ -| # | Algorithm | Recommendation | Alternative | -|---+--------------------+----------------+-------------------| -| 1 | DES encryption | Avoid | AES encryption | -|---+--------------------+----------------+-------------------| -| 2 | 3DES encryption | Avoid | AES encryption | -|---+--------------------+----------------+-------------------| -| 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA1 | -|---+--------------------+----------------+-------------------| -| 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR | -+-------------------------------------------------------------+ ++--------------------------------------------------------------+ +| # | Algorithm | Recommendation | Alternative | +|---+---------------------+----------------+-------------------| +| 1 | DES encryption | Avoid | AES encryption | +|---+---------------------+----------------+-------------------| +| 2 | 3DES encryption | Avoid | AES encryption | +|---+---------------------+----------------+-------------------| +| 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA256 | +|---+---------------------+----------------+-------------------| +| 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR | +|---+---------------------+----------------+-------------------| +| 3 | HMAC-SHA1 integrity | Avoid | HMAC-SHA256 | ++--------------------------------------------------------------+ ``` Intel(R) Multi-Buffer Crypto for IPsec Library depends on C library and it is recommended to use its latest version. -- GitLab From 629498db4a1f8733e7e5e6743feb6e9603b45be1 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 14:59:18 +0000 Subject: [PATCH 082/369] test: [GCM] rename MAX_JOBS macro Rename MAX_JOBS to GCM_MAX_JOBS to avoid macro redefinition (MAX_JOBS is still defined in intel-ipsec-mb.h header). --- test/gcm_test.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/gcm_test.c b/test/gcm_test.c index b6887b45..8a2ddbd9 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -43,7 +43,7 @@ #define IV_SZ 12 #define DIGEST_SZ 16 #define MAX_KEY_SZ 32 -#define MAX_JOBS 32 +#define GCM_MAX_JOBS 32 /* * 60-Byte Packet Encryption Using GCM-AES-128 @@ -1644,7 +1644,7 @@ aes_gcm_burst(IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher_mode, const IMB_SGL_STATE sgl_state, const uint32_t num_jobs) { - IMB_JOB *job, jobs[MAX_JOBS]; + IMB_JOB *job, jobs[GCM_MAX_JOBS]; uint32_t i; for (i = 0; i < num_jobs; i++) { @@ -2094,7 +2094,7 @@ test_gcm_vectors_burst(struct gcm_ctr_vector const *vector, uint8_t **T_test = NULL; const uint8_t *iv = vector->IV; uint64_t iv_len = vector->IVlen; - uint32_t i, j, num_jobs = MAX_JOBS; + uint32_t i, j, num_jobs = GCM_MAX_JOBS; /* Allocate space for the calculated ciphertext */ ct_test = malloc(num_jobs * sizeof(void *)); -- GitLab From 44d5cc0c8feb8a02c162ea3e70941937ef512ef9 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 9 May 2022 13:50:07 +0000 Subject: [PATCH 083/369] lib: Burst API complete_job() fix Fix for a case when complete_job() completes multiple jobs before returning. In this case, the burst API would not mark all jobs as completed as it assumed only a single job was completed by the complete_job() function. --- lib/include/mb_mgr_code.h | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 05ccc96e..02612e28 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2809,8 +2809,14 @@ IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) } __forceinline -void complete_job(IMB_MGR *state, IMB_JOB *job) +uint32_t complete_job(IMB_MGR *state, IMB_JOB *job) { + uint32_t completed_jobs = 0; + + /** + * complete as many jobs as necessary + * until specified 'job' has completed + */ if (job->chain_order == IMB_ORDER_CIPHER_HASH) { /* while() loop optimized for cipher_hash order */ while (job->status < IMB_STATUS_COMPLETED) { @@ -2820,6 +2826,7 @@ void complete_job(IMB_MGR *state, IMB_JOB *job) tmp = FLUSH_JOB_HASH(state, job); (void) RESUBMIT_JOB(state, tmp); + completed_jobs++; } } else { /* while() loop optimized for hash_cipher order */ @@ -2830,8 +2837,11 @@ void complete_job(IMB_MGR *state, IMB_JOB *job) tmp = FLUSH_JOB_AES(state, job); (void) RESUBMIT_JOB(state, tmp); + completed_jobs++; } } + + return completed_jobs; } __forceinline @@ -2872,7 +2882,7 @@ submit_job_and_check(IMB_MGR *state, const int run_check) if (state->earliest_job == state->next_job) { /* Full */ job = JOBS(state, state->earliest_job); - complete_job(state, job); + (void) complete_job(state, job); ADV_JOBS(&state->earliest_job); goto exit; } @@ -2952,7 +2962,7 @@ FLUSH_JOB(IMB_MGR *state) SAVE_XMMS(xmm_save); #endif job = JOBS(state, state->earliest_job); - complete_job(state, job); + (void) complete_job(state, job); ADV_JOBS(&state->earliest_job); @@ -3098,8 +3108,7 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, if (job->status < IMB_STATUS_COMPLETED) { /* force job to completion */ - complete_job(state, job); - completed_jobs++; + completed_jobs += complete_job(state, job); } } -- GitLab From 1b5c5103b9f7772e120baf4fc5bc431bad1a0f3c Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 9 May 2022 14:03:49 +0000 Subject: [PATCH 084/369] test: Improve burst API tests error checking --- test/aes_test.c | 9 +++++++++ test/chacha20_poly1305_test.c | 9 +++++++++ test/gcm_test.c | 13 +++++++++++++ test/hmac_sha1_test.c | 9 +++++++++ test/hmac_sha256_sha512_test.c | 9 +++++++++ 5 files changed, 49 insertions(+) diff --git a/test/aes_test.c b/test/aes_test.c index ff8fab15..fddbdd2a 100644 --- a/test/aes_test.c +++ b/test/aes_test.c @@ -2044,12 +2044,21 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; } } for (i = 0; i < num_jobs; i++) { job = &jobs[i]; + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (!aes_job_ok(job, out_text, job->user_data, padding, sizeof(padding), text_len)) goto end; diff --git a/test/chacha20_poly1305_test.c b/test/chacha20_poly1305_test.c index a584e4da..09da80ac 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/chacha20_poly1305_test.c @@ -510,12 +510,21 @@ test_aead(struct IMB_MGR *mb_mgr, printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; } } for (i = 0; i < num_jobs; i++) { job = &jobs[i]; + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (!aead_job_ok(mb_mgr, vec, job, job->user_data, padding, sizeof(padding))) goto end; diff --git a/test/gcm_test.c b/test/gcm_test.c index 8a2ddbd9..071624ae 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -1687,6 +1687,19 @@ aes_gcm_burst(IMB_MGR *mb_mgr, printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); return -1; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + return -1; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + return -1; } } diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c index ec8ad162..5e4f4b21 100644 --- a/test/hmac_sha1_test.c +++ b/test/hmac_sha1_test.c @@ -572,12 +572,21 @@ test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; } } for (i = 0; i < num_jobs; i++) { job = &jobs[i]; + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (!hmac_sha1_job_ok(vec, job, job->user_data, padding, sizeof(padding))) goto end; diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index a1c8b6f4..3403598d 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -1224,12 +1224,21 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; } } for (i = 0; i < num_jobs; i++) { job = &jobs[i]; + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (!hmac_shax_job_ok(vec, job, sha_type, job->user_data, padding, sizeof(padding))) -- GitLab From c46a896dc3b318eb7f359548602fd970aefe9672 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 20 Apr 2022 09:04:39 +0000 Subject: [PATCH 085/369] avx512: [ZUC-EIA3] enable ZUC-EIA3 functions Switch from SSE to AVX512 ZUC-EIA3 functions. --- lib/avx512/mb_mgr_avx512.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 044c9d66..3c070cf4 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -141,16 +141,16 @@ IMB_JOB *aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job); IMB_JOB *aes_cntr_ccm_256_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, +IMB_JOB *submit_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, +IMB_JOB *flush_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, +IMB_JOB *submit_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, +IMB_JOB *flush_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); void aes_cmac_256_subkey_gen_avx512(const void *key_exp, @@ -897,12 +897,12 @@ static IMB_JOB * static IMB_JOB * (*submit_job_zuc256_eia3_avx512) (MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz) = - submit_job_zuc256_eia3_no_gfni_sse; + submit_job_zuc256_eia3_no_gfni_avx512; static IMB_JOB * (*flush_job_zuc256_eia3_avx512) (MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) = - flush_job_zuc256_eia3_no_gfni_sse; + flush_job_zuc256_eia3_no_gfni_avx512; static IMB_JOB * (*submit_job_aes_xcbc_avx512) @@ -1396,12 +1396,12 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(zuc256_eia3_ooo->lens)); memset(zuc256_eia3_ooo->job_in_lane, 0, sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; + zuc256_eia3_ooo->unused_lanes = 0xFEDCBA9876543210; zuc256_eia3_ooo->num_lanes_inuse = 0; memset(&zuc256_eia3_ooo->state, 0, sizeof(zuc256_eia3_ooo->state)); zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; + zuc256_eia3_ooo->unused_lane_bitmask = 0xffff; memset(zuc256_eia3_ooo->args.digest, 0, sizeof(zuc256_eia3_ooo->args.digest)); @@ -1890,9 +1890,9 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) flush_job_zuc256_eea3_avx512 = flush_job_zuc256_eea3_gfni_avx512; submit_job_zuc256_eia3_avx512 = - submit_job_zuc256_eia3_gfni_sse; + submit_job_zuc256_eia3_gfni_avx512; flush_job_zuc256_eia3_avx512 = - flush_job_zuc256_eia3_gfni_sse; + flush_job_zuc256_eia3_gfni_avx512; } if (reset_mgrs) { -- GitLab From 54cb8658ca3db8b29935c0a5ca2bfb38c7f35b6b Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:38:13 +0000 Subject: [PATCH 086/369] avx512: [ZUC-EIA3-256] Initialize 8-byte tags --- lib/avx512/zuc_x16_avx512.asm | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index a00d0364..7eaf563c 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -362,6 +362,16 @@ db 10111010b, 10111011b, 10111110b, 10111111b db 11101010b, 11101011b, 11101110b, 11101111b db 11111010b, 11111011b, 11111110b, 11111111b +align 64 +idx_tags_64_0_7: +dd 0x00, 0x10, 0x01, 0x11, 0x02, 0x12, 0x03, 0x13 +dd 0x04, 0x14, 0x05, 0x15, 0x06, 0x16, 0x07, 0x17 + +align 64 +idx_tags_64_8_15: +dd 0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B +dd 0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F + ;; Calculate address for next bytes of keystream (KS) ;; Memory for KS is laid out in the following way: ;; - There are 128 bytes of KS for each buffer spread in chunks of 16 bytes, @@ -1106,6 +1116,9 @@ align 64 %define %%SHIFT_KMASK k3 ; Mask to shift 4 bytes only in the 15th dword %define %%IV_KMASK k4 ; Mask to read 10 bytes of IV +%define %%TMP_KMASK1 k3 +%define %%TMP_KMASK2 k4 + kmovw %%INIT_LANE_KMASK, DWORD(%%LANE_MASK) %if %%KEY_SIZE == 256 @@ -1207,10 +1220,23 @@ align 64 vmovdqa32 [%%STATE + OFS_R1]{%%INIT_LANE_KMASK}, %%R1 vmovdqa32 [%%STATE + OFS_R2]{%%INIT_LANE_KMASK}, %%R2 + ; Transpose (if needed) the keystream generated and store it + ; for each lane as their initial digest %if %%TAG_SIZE == 4 vmovdqa32 [%%TAGS]{%%INIT_LANE_KMASK}, %%KSTR1 REORDER_LFSR %%STATE, 1, %%INIT_LANE_KMASK -%elif %%TAG_SIZE == 8 ;; TODO +%elif %%TAG_SIZE == 8 + mov DWORD(%%TMP), 0xff + kmovd %%TMP_KMASK1, DWORD(%%TMP) + kandd %%TMP_KMASK1, %%TMP_KMASK1, %%INIT_LANE_KMASK ; First 8 lanes + kshiftrd %%TMP_KMASK2, %%INIT_LANE_KMASK, 8 ; Second 8 lanes + vmovdqa64 %%ZTMP1, [rel idx_tags_64_0_7] + vmovdqa64 %%ZTMP2, [rel idx_tags_64_8_15] + vpermi2d %%ZTMP1, %%KSTR1, %%KSTR2 + vpermi2d %%ZTMP2, %%KSTR1, %%KSTR2 + vmovdqa64 [%%TAGS]{%%TMP_KMASK1}, %%ZTMP1 + vmovdqa64 [%%TAGS + 64]{%%TMP_KMASK2}, %%ZTMP2 + REORDER_LFSR %%STATE, 2, %%INIT_LANE_KMASK %elif %%TAG_SIZE == 16 ;; TODO %endif @@ -1259,9 +1285,11 @@ ZUC256_INIT: or tag_sz, tag_sz jz init_for_cipher - ;; TODO: Check for 8B and 16B tags - cmp tag_sz, 4 - je init_for_auth_tag_4B + ;; TODO: Check for 16B tags + cmp tag_sz, 8 + je init_for_auth_tag_8B + jb init_for_auth_tag_4B + init_for_cipher: FUNC_SAVE @@ -1281,6 +1309,15 @@ init_for_auth_tag_4B: ret +init_for_auth_tag_8B: + FUNC_SAVE + + INIT_16_AVX512 pKe, pIv, pState, lane_mask, r12, r13, 256, 8, tags + + FUNC_RESTORE + + ret + ; ; Generate N*4 bytes of keystream ; for 16 buffers (where N is number of rounds) -- GitLab From 6d0c7a326075e9a57ef1f90c29451fab1feeacb6 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:41:16 +0000 Subject: [PATCH 087/369] avx512: [ZUC-EIA3] Add tag size parameter to EIA3_N64_16 function --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 7 +++-- lib/avx512/zuc_x16_avx512.asm | 27 ++++++++++++++++--- lib/include/zuc_internal.h | 7 +++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index b5c9671b..02347b4e 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -93,6 +93,7 @@ extern asm_Eia3_Nx64B_AVX512_16_VPCLMUL %define arg4 rcx %define arg5 r8 %define arg6 r9 +%define arg7 qword [rsp] %else %define arg1 rcx %define arg2 rdx @@ -100,6 +101,7 @@ extern asm_Eia3_Nx64B_AVX512_16_VPCLMUL %define arg4 r9 %define arg5 qword [rsp + 32] %define arg6 dword [rsp + 40] +%define arg7 qword [rsp + 48] %endif %define state arg1 @@ -679,7 +681,7 @@ FLUSH_JOB_ZUC256_EEA3: %%_above_eq_16_loop: ; Generate next 16 KS words and digest 64 bytes of data - RESERVE_STACK_SPACE 6 + RESERVE_STACK_SPACE 7 mov DWORD(%%TMP), %%L shr DWORD(%%TMP), 4 ; Number of rounds of 64 bytes @@ -700,10 +702,11 @@ FLUSH_JOB_ZUC256_EEA3: lea %%TMP, [%%OOO + _zuc_lens] mov [rsp + 32], %%TMP %endif + mov arg7, 4 ; Hardcoded to 4 (tag size) call ZUC_EIA3_N64B - RESTORE_STACK_SPACE 6 + RESTORE_STACK_SPACE 7 and %%L, 0xf ; Remaining words of KS left to generate diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 7eaf563c..acfb6c1f 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -394,6 +394,7 @@ align 64 %define arg4 rcx %define arg5 r8 %define arg6 r9d +%define arg7 qword [rsp + 8] %else %define arg1 rcx %define arg2 rdx @@ -401,6 +402,7 @@ align 64 %define arg4 r9 %define arg5 qword [rsp + 40] %define arg6 qword [rsp + 48] +%define arg7 qword [rsp + 56] %endif %define OFS_R1 (16*(4*16)) @@ -1560,13 +1562,14 @@ init_for_auth_tag_8B: ; Generate 64 bytes of keystream ; for 16 buffers and authenticate 64 bytes of data ; -%macro ZUC_EIA3_16_64B_AVX512 6 +%macro ZUC_EIA3_16_64B_AVX512 7 %define %%STATE %1 ; [in] ZUC state %define %%KS %2 ; [in] Pointer to keystream (128x16 bytes) %define %%T %3 ; [in] Pointer to digests %define %%DATA %4 ; [in] Pointer to array of pointers to data buffers %define %%LEN %5 ; [in] Pointer to array of remaining length to digest %define %%NROUNDS %6 ; [in/clobbered] Number of rounds of 64 bytes of data to digest +%define %%TAG_SIZE %7 ; [in] Tag size (4 or 8 bytes) %define %%TMP r12 %define %%DATA_ADDR0 rbx @@ -1844,7 +1847,9 @@ ZUC_KEYGEN64B_16: ;; uint32_t *pKeyStr, ;; uint32_t *T, ;; const void **data, -;; uint16_t *len); +;; uint16_t *len, +;; const uint64_t numRounds, +;; const uint64_t tag_size); MKGLOBAL(ZUC_EIA3_N64B,function,internal) ZUC_EIA3_N64B: %define STATE arg1 @@ -1859,6 +1864,8 @@ ZUC_EIA3_N64B: %define LEN r10 %define NROUNDS r11 %endif +%define TAG_SIZE arg7 + endbranch64 %ifndef LINUX @@ -1866,9 +1873,23 @@ ZUC_EIA3_N64B: mov NROUNDS, arg6 %endif + cmp TAG_SIZE, 8 + je Eia3_N64B_tag_8B + + ; Fall-through for 4 bytes +Eia3_N64B_tag_4B: + FUNC_SAVE + + ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS, 4 + + FUNC_RESTORE + + ret + +Eia3_N64B_tag_8B: FUNC_SAVE - ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS + ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS, 8 FUNC_RESTORE diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 4bd82b9b..eb1a5af9 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -1229,6 +1229,7 @@ IMB_DLL_LOCAL uint32_t asm_Eia3_256_RemainderAVX512_16_VPCLMUL(uint32_t *T, * @param[in] data Array of 16 pointers to data for 16 buffers * @param[in] len Array of lengths for 16 buffers * @param[in] numRounds Number of 64B rounds to perform + * @param[in] tagSize Tag size (4 or 8 bytes) * *****************************************************************************/ IMB_DLL_LOCAL @@ -1237,14 +1238,16 @@ void asm_Eia3_Nx64B_AVX512_16(ZucState16_t *pState, uint32_t *T, const void **data, uint16_t *len, - const uint32_t numRounds); + const uint64_t numRounds, + const uint64_t tagSize); IMB_DLL_LOCAL void asm_Eia3_Nx64B_AVX512_16_VPCLMUL(ZucState16_t *pState, uint32_t *pKeyStr, uint32_t *T, const void **data, uint16_t *len, - const uint32_t numRounds); + const uint64_t numRounds, + const uint64_t tagSize); IMB_DLL_LOCAL void zuc_eia3_4_buffer_job_gfni_sse(const void * const pKey[4], const uint8_t *ivs, -- GitLab From 75f39644342e372eef41b92958e29ac6fd7b3417 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:44:59 +0000 Subject: [PATCH 088/369] avx512: [ZUC-EIA3-256] Extend DIGEST_DATA to produce 8-byte tags --- lib/avx/zuc_x4_avx.asm | 8 ++- lib/avx512/zuc_x16_avx512.asm | 128 ++++++++++++++++++++++++---------- 2 files changed, 95 insertions(+), 41 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index ae3f56d6..dbe8336e 100755 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -1508,7 +1508,7 @@ exit_cipher: %define %%BIT_REV_H %3 ; [in] Bit reverse high table (XMM) %define %%BIT_REV_AND %4 ; [in] Bit reverse and table (XMM) %define %%XDIGEST %5 ; [in/out] Temporary digest (XMM) -%define %%XTMP1 %6 ; [clobbered] Temporary XMM register +%define %%XDATA %6 ; [in/clobbered] Input data (16 bytes) / Temporary XMM register %define %%XTMP2 %7 ; [clobbered] Temporary XMM register %define %%XTMP3 %8 ; [clobbered] Temporary XMM register %define %%XTMP4 %9 ; [clobbered] Temporary XMM register @@ -1521,9 +1521,11 @@ exit_cipher: %define %%OFF %16 ; [in] Offset into KS %define %%TAG_SZ %17 ; [in] Tag size (4, 8 or 16) - vpand %%XTMP2, %%XTMP1, %%BIT_REV_AND +%define %%XTMP1 %%XDATA - vpandn %%XTMP3, %%BIT_REV_AND, %%XTMP1 + vpand %%XTMP2, %%XDATA, %%BIT_REV_AND + + vpandn %%XTMP3, %%BIT_REV_AND, %%XDATA vpsrld %%XTMP3, 4 vpshufb %%XTMP4, %%BIT_REV_H, %%XTMP2 diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index acfb6c1f..77f33bd5 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -372,6 +372,14 @@ idx_tags_64_8_15: dd 0x08, 0x18, 0x09, 0x19, 0x0A, 0x1A, 0x0B, 0x1B dd 0x0C, 0x1C, 0x0D, 0x1D, 0x0E, 0x1E, 0x0F, 0x1F +align 64 +bits_32_63: +times 4 dd 0x00000000, 0xffffffff, 0x00000000, 0x00000000 + +align 64 +shuf_mask_0_0_0_dw1: +times 4 db 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + ;; Calculate address for next bytes of keystream (KS) ;; Memory for KS is laid out in the following way: ;; - There are 128 bytes of KS for each buffer spread in chunks of 16 bytes, @@ -1517,18 +1525,21 @@ init_for_auth_tag_8B: ;; To use it with YMM and ZMM registers, VPCMULQDQ must be ;; supported. ;; -%macro DIGEST_DATA 11 -%define %%DATA %1 ; [in] Input data (16 bytes) -%define %%KS_L %2 ; [in] Lower 16 bytes of KS -%define %%KS_H %3 ; [in] Higher 16 bytes of KS -%define %%IN_OUT %4 ; [in/out] Accumulated digest -%define %%KMASK %5 ; [in] Shuffle mask register -%define %%TMP1 %6 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TMP2 %7 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TMP3 %8 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TMP4 %9 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TMP5 %10 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TMP6 %11 ; [clobbered] Temporary XMM/YMM/ZMM register +%macro DIGEST_DATA 14 +%define %%DATA %1 ; [in] Input data (16 bytes) per buffer +%define %%KS_L %2 ; [in] Lower 16 bytes of KS per buffer +%define %%KS_H %3 ; [in] Higher 16 bytes of KS per buffer +%define %%KS_M1 %4 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%KS_M2 %5 ; [cloberred] Temporary XMM/YMM/ZMM register +%define %%IN_OUT %6 ; [in/out] Accumulated digest +%define %%KMASK %7 ; [in] Shuffle mask register +%define %%TMP1 %8 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TMP2 %9 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TMP3 %10 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TMP4 %11 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TMP5 %12 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TMP6 %13 ; [clobbered] Temporary XMM/YMM/ZMM register +%define %%TAG_SIZE %14 ; [in] Tag size (4 or 8 bytes) ;; Set up KS ;; @@ -1537,25 +1548,54 @@ init_for_auth_tag_8B: ;; TMP1 to contain bytes in the following order [7:4 11:8 3:0 7:4] ;; TMP2 to contain bytes in the following order [15:12 19:16 11:8 15:12] vpalignr %%TMP1, %%KS_H, %%KS_L, 8 - vpshufd %%TMP2, %%KS_L, 0x61 - vpshufd %%TMP1, %%TMP1, 0x61 +%if %%TAG_SIZE != 4 ;; TAG_SIZE == 8 or 16 + vpshufd %%KS_M2, %%KS_H, 0x61 ; KS bits [191:160 159:128 223:192 191:160] +%endif +%if %%TAG_SIZE == 16 + vpshufd %%KS_H, %%KS_H, 0xBB ; KS bits [255:224 223:192 255:224 223:192] +%endif + vpshufd %%KS_L, %%KS_L, 0x61 + vpshufd %%KS_M1, %%TMP1, 0x61 ;; Set up DATA ;; ;; DATA contains 16 bytes of input data (for 1, 2 or 4 buffers) ;; TMP3 to contain bytes in the following order [4*0's 7:4 4*0's 3:0] ;; TMP3 to contain bytes in the following order [4*0's 15:12 4*0's 11:8] - vpshufd %%TMP3{%%KMASK}{z}, %%DATA, 0x10 - vpshufd %%TMP4{%%KMASK}{z}, %%DATA, 0x32 + vpshufd %%TMP1{%%KMASK}{z}, %%DATA, 0x10 + vpshufd %%TMP2{%%KMASK}{z}, %%DATA, 0x32 ;; PCMUL the KS's with the DATA ;; XOR the results from 4 32-bit words together - vpclmulqdq %%TMP5, %%TMP3, %%TMP2, 0x00 - vpclmulqdq %%TMP3, %%TMP3, %%TMP2, 0x11 - vpclmulqdq %%TMP6, %%TMP4, %%TMP1, 0x00 - vpclmulqdq %%TMP4, %%TMP4, %%TMP1, 0x11 - vpternlogq %%TMP5, %%TMP3, %%TMP6, 0x96 - vpternlogq %%IN_OUT, %%TMP5, %%TMP4, 0x96 + vpclmulqdq %%TMP3, %%TMP1, %%KS_L, 0x00 + vpclmulqdq %%TMP4, %%TMP1, %%KS_L, 0x11 + vpclmulqdq %%TMP5, %%TMP2, %%KS_M1, 0x00 + vpclmulqdq %%TMP6, %%TMP2, %%KS_M1, 0x11 + vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 +%if %%TAG_SIZE == 4 + vpternlogq %%IN_OUT, %%TMP5, %%TMP6, 0x96 +%endif ; %%TAG_SIZE == 4 +%if %%TAG_SIZE >= 8 + ; Move previous result to low 32 bits and XOR with previous digest + vpxorq %%TMP5, %%TMP5, %%TMP6 + vpshufb %%TMP5, %%TMP5, [rel shuf_mask_0_0_0_dw1] + + vpxorq %%IN_OUT, %%IN_OUT, %%TMP5 + + vpclmulqdq %%TMP3, %%TMP1, %%KS_L, 0x10 + vpclmulqdq %%TMP4, %%TMP1, %%KS_M1, 0x01 + vpclmulqdq %%TMP5, %%TMP2, %%KS_M1, 0x10 + vpclmulqdq %%TMP6, %%TMP2, %%KS_M2, 0x01 + + ; XOR all the products and keep only 32-63 bits + vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 + vpxorq %%TMP5, %%TMP5, %%TMP6 + vpandq %%TMP5, %%TMP5, [rel bits_32_63] + + ; XOR with bits 32-63 of previous digest + vpxorq %%IN_OUT, %%TMP5 + +%endif ; %%TAG_SIZE >= 8 %endmacro ; @@ -1592,6 +1632,7 @@ init_for_auth_tag_8B: %define %%ZTMP7 zmm7 %define %%ZTMP8 zmm8 %define %%ZTMP9 zmm9 +%define %%ZTMP10 zmm0 %define %%ZKS_L %%ZTMP9 %define %%ZKS_H zmm21 @@ -1603,7 +1644,9 @@ init_for_auth_tag_8B: %define %%XTMP5 xmm5 %define %%XTMP6 xmm6 %define %%XTMP7 xmm7 +%define %%XTMP8 xmm8 %define %%XTMP9 xmm9 +%define %%XTMP10 xmm0 %define %%KS_L %%XTMP9 %define %%KS_H xmm21 %define %%XDIGEST_0 xmm13 @@ -1657,9 +1700,6 @@ init_for_auth_tag_8B: vpxorq %%DIGEST_2, %%DIGEST_2 vpxorq %%DIGEST_3, %%DIGEST_3 - ; Load read-only registers - vmovdqa64 %%MASK31, [rel mask31] - %if USE_GFNI_VAES_VPCLMUL == 0 vmovdqa64 %%REV_TABLE_L, [rel bit_reverse_table_l] vmovdqa64 %%REV_TABLE_H, [rel bit_reverse_table_h] @@ -1691,6 +1731,9 @@ init_for_auth_tag_8B: %assign %%idx 0 %rep 4 + ; Load read-only registers + vmovdqa64 %%MASK31, [rel mask31] + BITS_REORG16 %%STATE, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KS_, %%idx) NONLIN_FUN16 %%STATE, %%ALL_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ @@ -1727,8 +1770,9 @@ init_for_auth_tag_8B: vgf2p8affineqb %%ZTMP7, %%ZTMP1, [rel bit_reverse_table], 0x00 ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA %%ZTMP7, %%ZKS_L, %%ZKS_H, APPEND(%%DIGEST_, %%LANE_GROUP), %%SHUF_DATA_KMASK, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 + DIGEST_DATA %%ZTMP7, %%ZKS_L, %%ZKS_H, %%ZTMP8, %%ZTMP10, \ + APPEND(%%DIGEST_, %%LANE_GROUP), %%SHUF_DATA_KMASK, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%TAG_SIZE %else ; USE_GFNI_VAES_VPCLMUL == 1 ;; If VPCMUL is NOT available, read chunks of 16 bytes of data @@ -1751,8 +1795,9 @@ init_for_auth_tag_8B: %%REV_AND_TABLE, %%XTMP2, %%XTMP3 ; Digest 16 bytes of data with 24 bytes of KS, for one buffer - DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, APPEND(%%XDIGEST_, %%J), %%SHUF_DATA_KMASK, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, %%XTMP8, %%XTMP10, APPEND(%%XDIGEST_, %%J), \ + %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE ; Once all 64 bytes of data have been digested, insert them in temporary ZMM register %if %%idx == 3 @@ -2352,6 +2397,7 @@ _no_final_rounds: %define %%ZTMP6 zmm5 %define %%ZTMP7 zmm6 %define %%ZTMP8 zmm7 +%define %%ZTMP9 zmm8 %define %%YTMP1 YWORD(%%ZTMP1) @@ -2387,8 +2433,10 @@ _no_final_rounds: vgf2p8affineqb %%ZTMP1, APPEND(%%DATA_TRANS,%%I), [rel bit_reverse_table], 0x00 ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA %%ZTMP1, APPEND(%%KS_TRANS, %%I), APPEND(%%KS_TRANS, %%J), APPEND(%%DIGEST_, %%IDX), \ - %%SHUF_DATA_KMASK, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7 + DIGEST_DATA %%ZTMP1, APPEND(%%KS_TRANS, %%I), APPEND(%%KS_TRANS, %%J), \ + %%ZTMP8, %%ZTMP9, APPEND(%%DIGEST_, %%IDX), %%SHUF_DATA_KMASK, \ + %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ + %%ZTMP6, %%ZTMP7, %%TAG_SIZE %assign %%J (%%J + 1) %assign %%I (%%I + 1) @@ -2475,6 +2523,8 @@ _no_final_rounds: %define %%XTMP4 xmm10 %define %%XTMP5 xmm11 %define %%XTMP6 xmm12 +%define %%XTMP7 xmm13 +%define %%XTMP8 xmm14 %define %%ZTMP1 zmm24 %define %%ZTMP2 zmm25 @@ -2532,8 +2582,8 @@ _no_final_rounds: vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (16*4)] %endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE %assign %%K (%%K + 1) %endrep @@ -2689,6 +2739,8 @@ round_4B: %define %%XTMP4 xmm10 %define %%XTMP5 xmm11 %define %%XTMP6 xmm12 +%define %%XTMP7 xmm13 +%define %%XTMP8 xmm14 %define %%ZTMP1 zmm7 %define %%ZTMP2 zmm8 @@ -2784,8 +2836,8 @@ round_4B: vmovdqu %%KS_H, [%%KS + (16*I + J*512) + (16*4)] %endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers - DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE add %%OFFSET, 16 sub %%LEN_BUF, 128 %assign K (K + 1) @@ -2844,9 +2896,9 @@ APPEND3(%%Eia3RoundsAVX512_dq_end,I,J): vmovdqu %%KS_H, [%%KS + (16*I + J*512) + %%OFFSET + 16*4] shr %%OFFSET, 2 - ; Digest 16 bytes of data with 24 bytes of %%KS, for 4 buffers - DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%TEMP_DIGEST, k2, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6 + ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers + DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE APPEND3(%%Eia3RoundsAVX_end,I,J): vinserti32x4 APPEND(%%DIGEST_, I), %%TEMP_DIGEST, J %assign J (J + 1) -- GitLab From 204bb24b6dbfd3767f7a3181df69838ef5fe85ec Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:49:41 +0000 Subject: [PATCH 089/369] avx512: [ZUC-EIA3-256] Update 8-byte tags Add code to permute data to update 8-byte tags, with new data being digested. --- lib/avx512/zuc_x16_avx512.asm | 224 ++++++++++++++++++++-------------- 1 file changed, 135 insertions(+), 89 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 77f33bd5..41708115 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -213,15 +213,35 @@ bit_reverse_table: times 8 db 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 align 64 -shuf_mask_tags_0_1_2_3: +shuf_mask_4B_tags_0_1_2_3: dd 0x01, 0x05, 0x09, 0x0D, 0x11, 0x15, 0x19, 0x1D, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF dd 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x05, 0x09, 0x0D, 0x11, 0x15, 0x19, 0x1D align 64 -shuf_mask_tags_0_4_8_12: +shuf_mask_4B_tags_0_4_8_12: dd 0x01, 0x11, 0xFF, 0xFF, 0x05, 0x15, 0xFF, 0xFF, 0x09, 0x19, 0xFF, 0xFF, 0x0D, 0x1D, 0xFF, 0xFF dd 0xFF, 0xFF, 0x01, 0x11, 0xFF, 0xFF, 0x05, 0x15, 0xFF, 0xFF, 0x09, 0x19, 0xFF, 0xFF, 0x0D, 0x1D +align 64 +shuf_mask_8B_tags_0_1_4_5: +dq 0x00, 0x08, 0xFF, 0xFF, 0x02, 0x0A, 0xFF, 0xFF + +align 64 +shuf_mask_8B_tags_2_3_6_7: +dq 0xFF, 0xFF, 0x00, 0x08, 0xFF, 0xFF, 0x02, 0x0A + +align 64 +shuf_mask_8B_tags_8_9_12_13: +dq 0x04, 0x0C, 0xFF, 0xFF, 0x06, 0x0E, 0xFF, 0xFF + +align 64 +shuf_mask_8B_tags_10_11_14_15: +dq 0xFF, 0xFF, 0x04, 0x0C, 0xFF, 0xFF, 0x06, 0x0E + +align 64 +shuf_mask_8B_tags: +dq 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E + align 64 all_ffs: dw 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff @@ -1598,6 +1618,89 @@ init_for_auth_tag_8B: %endif ; %%TAG_SIZE >= 8 %endmacro +%macro UPDATE_TAGS 13-16 +%define %%T %1 ; [in] Pointer to digests +%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8 bytes) +%define %%ORDER_TAGS %3 ; [constant] Order of tags (order_0_4_8_12 or order_0_1_2_3) +%define %%TMP %4 ; [clobbered] Temporary GP register +%define %%PERM_DIGEST_KMASK1 %5 ; [clobbered] Permutation mask for digests +%define %%PERM_DIGEST_KMASK2 %6 ; [clobbered] Permulation mask for digests +%define %%DIGEST_0 %7 ; [in/clobbered] Digests for lanes 0,4,8,12 or 0,1,2,3 +%define %%DIGEST_1 %8 ; [in] Digests for lanes 1,5,9,13 or 4,5,6,7 +%define %%DIGEST_2 %9 ; [in/clobbered] Digests for lanes 2,6,10,14 or 8,9,10,11 +%define %%DIGEST_3 %10 ; [in] Digests for lanes 3,7,11,15 or 12,13,14,15 +%define %%ZTMP1 %11 ; [clobbered] Temporary ZMM register +%define %%ZTMP2 %12 ; [clobbered] Temporary ZMM register +%define %%ZTMP3 %13 ; [clobbered] Temporary ZMM register +%define %%ZTMP4 %14 ; [clobbered] Temporary ZMM register +%define %%ZTMP5 %15 ; [clobbered] Temporary ZMM register +%define %%ZTMP6 %16 ; [clobbered] Temporary ZMM register + +%if %%TAG_SIZE == 4 +%ifidn %%ORDER_TAGS, order_0_4_8_12 + mov DWORD(%%TMP), 0x3333 + kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) + kshiftld %%PERM_DIGEST_KMASK2, %%PERM_DIGEST_KMASK1, 2 + vmovdqa64 %%ZTMP2, [rel shuf_mask_4B_tags_0_4_8_12] + vmovdqa64 %%ZTMP3, [rel shuf_mask_4B_tags_0_4_8_12 + 64] +%else + mov DWORD(%%TMP), 0x00FF + kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) + kshiftld %%PERM_DIGEST_KMASK2, %%PERM_DIGEST_KMASK1, 8 + vmovdqa64 %%ZTMP2, [rel shuf_mask_4B_tags_0_1_2_3] + vmovdqa64 %%ZTMP3, [rel shuf_mask_4B_tags_0_1_2_3 + 64] +%endif + vmovdqu64 %%ZTMP1, [%%T] ; Input tags + ; Get result tags for 16 buffers in different position in each lane + ; and blend these tags into an ZMM register. + ; Then, XOR the results with the previous tags and write out the result. + vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 + vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 + vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C + vmovdqu64 [%%T], %%ZTMP1 + +%elif %%TAG_SIZE == 8 + ; Input tags + vmovdqu64 %%ZTMP1, [%%T] + vmovdqu64 %%ZTMP2, [%%T + 64] +%ifidn %%ORDER_TAGS, order_0_4_8_12 + mov DWORD(%%TMP), 0x33 + kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) + kshiftld %%PERM_DIGEST_KMASK2, %%PERM_DIGEST_KMASK1, 2 + + vmovdqa64 %%ZTMP3, [rel shuf_mask_8B_tags_0_1_4_5] + vmovdqa64 %%ZTMP4, [rel shuf_mask_8B_tags_2_3_6_7] + vmovdqa64 %%ZTMP5, [rel shuf_mask_8B_tags_8_9_12_13] + vmovdqa64 %%ZTMP6, [rel shuf_mask_8B_tags_10_11_14_15] + + ; Get result tags for 16 buffers in different positions in each lane + ; and blend these tags into two ZMM registers + ; Then, XOR the results with the previous tags and write out the result. + + vpermi2q %%ZTMP3{%%PERM_DIGEST_KMASK1}{z}, %%DIGEST_0, %%DIGEST_1 + vpermi2q %%ZTMP4{%%PERM_DIGEST_KMASK2}{z}, %%DIGEST_2, %%DIGEST_3 + vpermi2q %%ZTMP5{%%PERM_DIGEST_KMASK1}{z}, %%DIGEST_0, %%DIGEST_1 + vpermi2q %%ZTMP6{%%PERM_DIGEST_KMASK2}{z}, %%DIGEST_2, %%DIGEST_3 + + vpternlogq %%ZTMP1, %%ZTMP3, %%ZTMP4, 0x96 ; A XOR B XOR C + vpternlogq %%ZTMP2, %%ZTMP5, %%ZTMP6, 0x96 ; A XOR B XOR C + +%else ; %%ORDER_TAGS == order_0_1_2_3 + ; Input tags + vmovdqa64 %%ZTMP3, [rel shuf_mask_8B_tags] + ; Get result tags for 16 buffers in different position in each lane + ; and blend these tags into an ZMM register. + ; Then, XOR the results with the previous tags and write out the result. + vpermt2q %%DIGEST_0, %%ZTMP3, %%DIGEST_1 + vpermt2q %%DIGEST_2, %%ZTMP3, %%DIGEST_3 + vpxorq %%ZTMP1, %%ZTMP1, %%DIGEST_0 + vpxorq %%ZTMP2, %%ZTMP2, %%DIGEST_2 +%endif + vmovdqu64 [%%T], %%ZTMP1 + vmovdqu64 [%%T + 64], %%ZTMP2 +%else ;; %%TAG_SIZE == 16 +%endif +%endmacro ; ; Generate 64 bytes of keystream ; for 16 buffers and authenticate 64 bytes of data @@ -1675,9 +1778,8 @@ init_for_auth_tag_8B: %define %%BLEND_KMASK k1 ; Mask to blend LFSRs 14&15 %define %%ALL_KMASK k2 ; Mask with all 1's %define %%SHUF_DATA_KMASK k3 ; Mask to shuffle data -%define %%PERM_DIGEST_KMASK1 k4 ; Permutation mask for digests -%define %%PERM_DIGEST_KMASK2 k5 ; Permulation mask for digests -%define %%TMP_KMASK k6 +%define %%TMP_KMASK1 k4 +%define %%TMP_KMASK2 k5 xor %%OFFSET, %%OFFSET @@ -1690,11 +1792,6 @@ init_for_auth_tag_8B: mov DWORD(%%TMP), 0x55555555 kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP) - mov DWORD(%%TMP), 0x3333 - kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) - mov DWORD(%%TMP), 0xCCCC - kmovd %%PERM_DIGEST_KMASK2, DWORD(%%TMP) - vpxorq %%DIGEST_0, %%DIGEST_0 vpxorq %%DIGEST_1, %%DIGEST_1 vpxorq %%DIGEST_2, %%DIGEST_2 @@ -1837,17 +1934,9 @@ init_for_auth_tag_8B: dec %%NROUNDS jnz %%_loop - ;; - update tags - vmovdqu64 %%ZTMP1, [%%T] ; Input tags - vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_4_8_12] - vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] - ; Get result tags for 16 buffers in different position in each lane - ; and blend these tags into an ZMM register. - ; Then, XOR the results with the previous tags and write out the result. - vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 - vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 - vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [%%T], %%ZTMP1 + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP, %%TMP_KMASK1, %%TMP_KMASK2, \ + %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 ; Update R1/R2 vmovdqa64 [%%STATE + OFS_R1], %%R1 @@ -1864,10 +1953,10 @@ init_for_auth_tag_8B: ; Update array of lengths (if lane is valid, so length < UINT16_MAX) vmovdqa64 YWORD(%%ZTMP2), [%%LEN] - vpcmpw %%TMP_KMASK, YWORD(%%ZTMP2), [rel all_ffs], 4 ; valid lanes + vpcmpw %%TMP_KMASK1, YWORD(%%ZTMP2), [rel all_ffs], 4 ; valid lanes shl %%OFFSET, 3 ; Convert to bits vpbroadcastw YWORD(%%ZTMP1), DWORD(%%OFFSET) - vpsubw YWORD(%%ZTMP2){%%TMP_KMASK}, YWORD(%%ZTMP1) + vpsubw YWORD(%%ZTMP2){%%TMP_KMASK1}, YWORD(%%ZTMP1) vmovdqa64 [%%LEN], YWORD(%%ZTMP2) %endmacro @@ -2355,9 +2444,8 @@ _no_final_rounds: %define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data -%define %%PERM_DIGEST_KMASK1 k2 ; Permutation mask for digests -%define %%PERM_DIGEST_KMASK2 k3 ; Permulation mask for digests -%define %%TMP_KMASK k4 +%define %%TMP_KMASK1 k2 +%define %%TMP_KMASK2 k3 %define %%DATA_ADDR0 %%TMP3 %define %%DATA_ADDR1 %%TMP4 @@ -2455,22 +2543,9 @@ _no_final_rounds: %assign %%IDX (%%IDX + 1) %endrep - ;; - update tags - mov %%TMP1, 0x3333 - mov %%TMP2, 0xCCCC - kmovq %%PERM_DIGEST_KMASK1, %%TMP1 - kmovq %%PERM_DIGEST_KMASK2, %%TMP2 - - vmovdqu64 %%ZTMP1, [%%T] ; Input tags - vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_4_8_12] - vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] - ; Get result tags for 16 buffers in different position in each lane - ; and blend these tags into an ZMM register. - ; Then, XOR the results with the previous tags and write out the result. - vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 - vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 - vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [%%T], %%ZTMP1 + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ + %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3 ; Update data pointers vmovdqu64 %%ZTMP1, [%%DATA] @@ -2482,8 +2557,8 @@ _no_final_rounds: ; Update array of lengths (subtract 512 bits from all lengths if valid lane) vmovdqa %%YTMP1, [LEN] - vpcmpw %%TMP_KMASK, %%YTMP1, [rel all_ffs], 4 - vpsubw %%YTMP1{%%TMP_KMASK}, [rel all_512w] + vpcmpw %%TMP_KMASK1, %%YTMP1, [rel all_ffs], 4 + vpsubw %%YTMP1{%%TMP_KMASK1}, [rel all_512w] vmovdqa [%%LEN], %%YTMP1 %endmacro @@ -2506,9 +2581,8 @@ _no_final_rounds: %define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data -%define %%PERM_DIGEST_KMASK1 k2 ; Permutation mask for digests -%define %%PERM_DIGEST_KMASK2 k3 ; Permulation mask for digests -%define %%TMP_KMASK k4 +%define %%TMP_KMASK1 k2 +%define %%TMP_KMASK2 k3 %define %%REV_TABLE_L xmm0 %define %%REV_TABLE_H xmm1 @@ -2603,22 +2677,9 @@ _no_final_rounds: %assign %%I (%%I + 1) %endrep - ;; - update tags - mov %%TMP1, 0x3333 - mov %%TMP2, 0xCCCC - kmovq %%PERM_DIGEST_KMASK1, %%TMP1 - kmovq %%PERM_DIGEST_KMASK2, %%TMP2 - - vmovdqu64 %%ZTMP1, [%%T] ; Input tags - vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_4_8_12] - vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_4_8_12 + 64] - ; Get result tags for 16 buffers in different position in each lane - ; and blend these tags into an ZMM register. - ; Then, XOR the results with the previous tags and write out the result. - vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 - vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 - vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [%%T], %%ZTMP1 + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ + %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3 ; Update data pointers vmovdqu64 %%ZTMP2, [%%DATA] @@ -2630,8 +2691,8 @@ _no_final_rounds: ; Update array of lengths (if lane is valid, so length < UINT16_MAX) vmovdqa64 %%YTMP1, [%%LEN] - vpcmpw %%TMP_KMASK, %%YTMP1, [rel all_ffs], 4 ; valid lanes - vpsubw %%YTMP1{%%TMP_KMASK}, [rel all_512w] + vpcmpw %%TMP_KMASK1, %%YTMP1, [rel all_ffs], 4 ; valid lanes + vpsubw %%YTMP1{%%TMP_KMASK1}, [rel all_512w] vmovdqa64 [%%LEN], %%YTMP1 %endmacro @@ -2749,9 +2810,8 @@ round_4B: %define %%VALID_KMASK k1 ; Mask with valid lanes %define %%SHUF_DATA_KMASK k2 ; Mask to shuffle data -%define %%PERM_DIGEST_KMASK1 k3 ; Permutation mask for digests -%define %%PERM_DIGEST_KMASK2 k4 ; Permulation mask for digests -%define %%TEMP_KMASK k5 +%define %%TMP_KMASK1 k3 +%define %%TMP_KMASK2 k4 vpbroadcastw %%YTMP1, DWORD(%%MIN_LEN) ; Get mask of non-NULL lanes (lengths not set to UINT16_MAX, indicating that lane is not valid) @@ -2763,11 +2823,11 @@ round_4B: vpandq %%YTMP1, [rel all_ffe0w] ; Calculate remaining bits to authenticate after function call - vpcmpuw %%TEMP_KMASK, %%YTMP2, %%YTMP1, 1 ; Get mask of lengths that will be < 0 after subtracting + vpcmpuw %%TMP_KMASK1, %%YTMP2, %%YTMP1, 1 ; Get mask of lengths that will be < 0 after subtracting vpsubw %%YTMP3{%%VALID_KMASK}, %%YTMP2, %%YTMP1 vpxorq %%YTMP4, %%YTMP4 ; Set to zero the lengths of the lanes which are going to be completed - vmovdqu16 %%YTMP3{%%TEMP_KMASK}, %%YTMP4 ; YMM2 contain final lengths + vmovdqu16 %%YTMP3{%%TMP_KMASK1}, %%YTMP4 ; YMM2 contain final lengths vmovdqu16 [%%LEN]{%%VALID_KMASK}, %%YTMP3 ; Update in memory the final updated lengths ; Calculate number of bits to authenticate (up to 511 bits), @@ -2853,10 +2913,10 @@ APPEND3(%%Eia3RoundsAVX512_dq_end,I,J): shr %%N_BYTES, 3 lea %%TMP1, [rel byte64_len_to_mask_table] - kmovq %%TEMP_KMASK, [%%TMP1 + %%N_BYTES*8] + kmovq %%TMP_KMASK1, [%%TMP1 + %%N_BYTES*8] ;; read up to 16 bytes of data, zero bits not needed if partial byte and bit-reverse - vmovdqu8 %%XTMP1{%%TEMP_KMASK}{z}, [%%DATA_ADDR + %%OFFSET] + vmovdqu8 %%XTMP1{%%TMP_KMASK1}{z}, [%%DATA_ADDR + %%OFFSET] ; check if there is a partial byte (less than 8 bits in last byte) mov %%TMP2, %%LEN_BUF and %%TMP2, 0x7 @@ -2906,23 +2966,9 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): %assign I (I + 1) %endrep - ;; - update tags - mov %%TMP1, 0x00FF - kmovq %%PERM_DIGEST_KMASK1, %%TMP1 - mov %%TMP1, 0xFF00 - kmovq %%PERM_DIGEST_KMASK2, %%TMP1 - - vmovdqu64 %%ZTMP1, [%%T] ; Input tags - vmovdqa64 %%ZTMP2, [rel shuf_mask_tags_0_1_2_3] - vmovdqa64 %%ZTMP3, [rel shuf_mask_tags_0_1_2_3 + 64] - ; Get result tags for 16 buffers in different position in each lane - ; and blend these tags into an ZMM register. - ; Then, XOR the results with the previous tags and write out the result. - vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 - vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 - vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - - vmovdqa64 [%%T], %%ZTMP1 ; Store temporary digests + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_1_2_3, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ + %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3 ; These last steps should be done only for the buffers that ; have no more data to authenticate -- GitLab From 6e7e4056cca87e983bdf13cf36170c65da582373 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 2 May 2022 20:04:06 +0000 Subject: [PATCH 090/369] avx512: [ZUC-EIA-256] Support 8-byte tags in Round64B function --- lib/avx512/zuc_x16_avx512.asm | 47 +++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 41708115..16477f9d 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2441,7 +2441,7 @@ _no_final_rounds: %define %%TMP4 %8 ; [clobbered] Temporary GP register %define %%TMP5 %9 ; [clobbered] Temporary GP register %define %%TMP6 %10 ; [clobbered] Temporary GP register -%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) +%define %%TAG_SIZE %11 ; [constant] Tag size (4 or 8 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data %define %%TMP_KMASK1 k2 @@ -2545,7 +2545,7 @@ _no_final_rounds: UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 ; Update data pointers vmovdqu64 %%ZTMP1, [%%DATA] @@ -2578,7 +2578,7 @@ _no_final_rounds: %define %%TMP4 %8 ; [clobbered] Temporary GP register %define %%TMP5 %9 ; [clobbered] Temporary GP register %define %%TMP6 %10 ; [clobbered] Temporary GP register -%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) +%define %%TAG_SIZE %11 ; [constant] Tag size (4 or 8 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data %define %%TMP_KMASK1 k2 @@ -2600,10 +2600,12 @@ _no_final_rounds: %define %%XTMP7 xmm13 %define %%XTMP8 xmm14 -%define %%ZTMP1 zmm24 -%define %%ZTMP2 zmm25 -%define %%ZTMP3 zmm26 -%define %%ZTMP4 zmm27 +%define %%ZTMP1 zmm22 +%define %%ZTMP2 zmm23 +%define %%ZTMP3 zmm24 +%define %%ZTMP4 zmm25 +%define %%ZTMP5 zmm26 +%define %%ZTMP6 zmm27 %define %%DIGEST_0 zmm28 %define %%DIGEST_1 zmm29 %define %%DIGEST_2 zmm30 @@ -2679,7 +2681,7 @@ _no_final_rounds: UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 ; Update data pointers vmovdqu64 %%ZTMP2, [%%DATA] @@ -2721,18 +2723,31 @@ _no_final_rounds: align 64 MKGLOBAL(ZUC_ROUND64B_16,function,internal) ZUC_ROUND64B_16: -%define T arg1 -%define KS arg2 -%define DATA arg3 -%define LEN arg4 -%define TAG_SZ arg5 +%define T arg1 +%define KS arg2 +%define DATA arg3 +%define LEN arg4 +%define TAG_SIZE arg5 endbranch64 - ; TODO: 8-byte and 16-byte digests - cmp TAG_SZ, 4 - je round_4B + cmp TAG_SIZE, 8 + je round_8B + jb round_4B +round_8B: + + FUNC_SAVE + +%if USE_GFNI_VAES_VPCLMUL == 1 + ROUND64B_16_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 8 +%else + ROUND64B_16_NO_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 8 +%endif + + FUNC_RESTORE + + ret round_4B: FUNC_SAVE -- GitLab From 7e9ae007f6e6938713ec6e938b5d091947913078 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 3 May 2022 07:01:33 +0000 Subject: [PATCH 091/369] avx512: [ZUC-EIA3-256] Support 8-byte tags in Remainder function --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 2 +- lib/avx512/zuc_x16_avx512.asm | 99 ++++++++++++++++--- 2 files changed, 89 insertions(+), 12 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index 02347b4e..fa3b64c5 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -100,7 +100,7 @@ extern asm_Eia3_Nx64B_AVX512_16_VPCLMUL %define arg3 r8 %define arg4 r9 %define arg5 qword [rsp + 32] -%define arg6 dword [rsp + 40] +%define arg6 qword [rsp + 40] %define arg7 qword [rsp + 48] %endif diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 16477f9d..4e14ba1a 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2776,7 +2776,7 @@ round_4B: %define %%TMP6 %11 ; [clobbered] Temporary GP register %define %%TMP7 %12 ; [clobbered] Temporary GP register %define %%KEY_SIZE %13 ; [constant] Key size (128 or 256) -%define %%TAG_SIZE %14 ; [constant] Tag size (4, 8 or 16 bytes) +%define %%TAG_SIZE %14 ; [constant] Tag size (4 or 8 bytes) %define %%DIGEST_0 zmm28 %define %%DIGEST_1 zmm29 @@ -2822,6 +2822,8 @@ round_4B: %define %%ZTMP2 zmm8 %define %%ZTMP3 zmm9 %define %%ZTMP4 zmm10 +%define %%ZTMP5 zmm11 +%define %%ZTMP6 zmm12 %define %%VALID_KMASK k1 ; Mask with valid lanes %define %%SHUF_DATA_KMASK k2 ; Mask to shuffle data @@ -2983,7 +2985,7 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): UPDATE_TAGS %%T, %%TAG_SIZE, order_0_1_2_3, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 ; These last steps should be done only for the buffers that ; have no more data to authenticate @@ -2997,20 +2999,22 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): cmp word [%%LEN + 2*%%IDX], 0 jnz %%skip_comput + ; Load base address of keystream for lane %%IDX + ; Fist, find the offset for the 512-byte set (containing the 128-byte KS for 4 lanes) mov %%TMP1, %%IDX and %%TMP1, 0x3 shl %%TMP1, 9 ; * 512 + ; Then, find the offset within the 512-byte set, based on the lane, + ; and add to the previous offset mov %%TMP2, %%IDX shr %%TMP2, 2 shl %%TMP2, 4 ; * 16 add %%TMP1, %%TMP2 + ;; Load pointer to the base address of keystream for lane %%IDX lea %%KS_ADDR, [%%KS + %%TMP1] - ; Read digest - mov DWORD(%%DIGEST), [%%T + 4*%%IDX] - - ; Read keyStr[MIN_LEN / 32] + ; Read keyStr[MIN_LEN / 32] (last dwords of KS, based on tag_size) movzx %%LEN_BUF, word [rsp + 2*%%IDX] mov %%TMP2, %%LEN_BUF shr %%TMP2, 5 @@ -3019,9 +3023,15 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): shl %%TMP2, (4+2) and %%TMP3, 0x3 shl %%TMP3, 2 - add %%TMP2, %%TMP3 + add %%TMP2, %%TMP3 ;; Offset to last dwords of KS, from base address mov %%TMP3, %%TMP2 and %%TMP3, 0xf +%if %%TAG_SIZE == 4 + ; Read 4-byte digest + mov DWORD(%%DIGEST), [%%T + 4*%%IDX] + + ; Read last two dwords of KS, which can be scattered or contiguous + ; (First dword can be at the end of a 16-byte chunk) cmp %%TMP3, 12 je %%_read_2dwords mov %%TMP1, [%%KS_ADDR + %%TMP2] @@ -3062,6 +3072,65 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): ; byte swap and write digest out bswap DWORD(%%DIGEST) mov [%%T + 4*%%IDX], DWORD(%%DIGEST) +%elif %%TAG_SIZE == 8 + ; Read 8-byte digest + mov %%DIGEST, [%%T + 8*%%IDX] + + ; Read last three dwords of KS, which can be scattered or contiguous + ; (First dword can be at the end of a 16-byte chunk and the other + ; two dwords in the next chunk; first two dwords can be at the end of + ; a 16-byte chunk and the other dword in the next chunk; or all three + ; dwords can be in the same 16-byte chunk) + cmp %%TMP3, 8 + je %%_read_8B_4B + cmp %%TMP3, 12 + je %%_read_4B_8B + + ;; All 12 bytes of KS are contiguous +%%_read_12B: + mov %%TMP1, [%%KS_ADDR + %%TMP2] + mov %%TMP2, [%%KS_ADDR + %%TMP2 + 4] + jmp %%_ks_qwords_read + + ;; The first 8 bytes of KS are contiguous, the other 4 are separated +%%_read_8B_4B: + mov %%TMP1, [%%KS_ADDR + %%TMP2] + ; Read last 4 bytes of first segment and first 4 bytes of second segment + mov DWORD(%%TMP3), [%%KS_ADDR + %%TMP2 + 4] + mov DWORD(%%TMP2), [%%KS_ADDR + %%TMP2 + (8+48)] + shl %%TMP2, 32 + or %%TMP2, %%TMP3 + + jmp %%_ks_qwords_read + ;; The first 8 bytes of KS are separated, the other 8 are contiguous +%%_read_4B_8B: + mov DWORD(%%TMP1), [%%KS_ADDR + %%TMP2] + mov DWORD(%%TMP3), [%%KS_ADDR + %%TMP2 + (4+48)] + shl %%TMP3, 32 + or %%TMP1, %%TMP3 + mov %%TMP2, [%%KS_ADDR + %%TMP2 + (4+48)] +%%_ks_qwords_read: + ; Rotate left by MIN_LEN % 32 + mov %%TMP3, rcx + mov rcx, %%LEN_BUF + and rcx, 0x1F + rol %%TMP1, cl + rol %%TMP2, cl + mov rcx, %%TMP3 + + shl %%TMP2, 32 + mov DWORD(%%TMP1), DWORD(%%TMP1) ; Clear top 32 bits + or %%TMP2, %%TMP1 + + ; XOR with current digest + xor %%DIGEST, %%TMP2 + + ; byte swap and write digest out + bswap %%DIGEST + ror %%DIGEST, 32 + mov [%%T + 8*%%IDX], %%DIGEST +%else ; %%TAG_SIZE == 16 +%endif %%skip_comput: inc %%IDX @@ -3168,16 +3237,24 @@ ZUC256_REMAINDER_16: %define N_BITS r10 -%define TAG_SZ arg6 +%define TAG_SIZE arg6 endbranch64 mov N_BITS, arg5 - ; TODO: 8-byte and 16-byte digests - cmp TAG_SZ, 4 - je remainder_4B + cmp TAG_SIZE, 8 + je remainder_8B + jb remainder_4B + +remainder_8B: + FUNC_SAVE + + REMAINDER_16 T, KS, DATA, LEN, N_BITS, rax, rbx, r11, r12, r13, r14, r15, 256, 8 + + FUNC_RESTORE + ret remainder_4B: FUNC_SAVE -- GitLab From dc032a0ee8d50d1f00d172fba652e054a1195147 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 5 May 2022 17:19:38 +0000 Subject: [PATCH 092/369] avx512: [ZUC-EIA3-256] add functions to skip first 4 KS bytes For old buffers, 4 bytes of keystream are saved from previous iteration of ZUC-EIA3-256 (for 4-byte tags), so to avoid extra memcpy's, skip the first 4 bytes of KS and write the rest for these buffers. --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 6 + .../mb_mgr_zuc_submit_flush_gfni_avx512.asm | 2 + lib/avx512/zuc_x16_avx512.asm | 180 +++++++++++------- lib/avx512/zuc_x16_vaes_avx512.asm | 3 +- lib/include/zuc_internal.h | 79 ++++++++ 5 files changed, 204 insertions(+), 66 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index fa3b64c5..831d6759 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -50,6 +50,8 @@ %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_avx512 +%define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_avx512 +%define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_avx512 %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_avx512 %define ZUC_ROUND64B asm_Eia3Round64BAVX512_16 @@ -77,6 +79,10 @@ extern asm_ZucGenKeystream_16_skip8_avx512 extern asm_ZucGenKeystream_16_skip8_gfni_avx512 extern asm_ZucGenKeystream64B_16_skip8_avx512 extern asm_ZucGenKeystream64B_16_skip8_gfni_avx512 +extern asm_ZucGenKeystream_16_skip4_avx512 +extern asm_ZucGenKeystream_16_skip4_gfni_avx512 +extern asm_ZucGenKeystream64B_16_skip4_avx512 +extern asm_ZucGenKeystream64B_16_skip4_gfni_avx512 extern asm_ZucGenKeystream_16_avx512 extern asm_ZucGenKeystream_16_gfni_avx512 extern asm_ZucGenKeystream64B_16_avx512 diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm index 17c7a522..cddf54e2 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm @@ -39,6 +39,8 @@ %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16_VPCLMUL %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_gfni_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_gfni_avx512 +%define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_gfni_avx512 +%define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_gfni_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_gfni_avx512 %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_gfni_avx512 %define ZUC_ROUND64B asm_Eia3Round64B_16_VPCLMUL diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 4e14ba1a..6992c4bc 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -46,8 +46,9 @@ %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_avx512 -%define ZUC_KEYGEN8B_SKIP8_16 asm_ZucGenKeystream8B_16_skip8_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_avx512 +%define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_avx512 +%define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_avx512 %define ZUC_ROUND64B_16 asm_Eia3Round64BAVX512_16 %define ZUC_EIA3_N64B asm_Eia3_Nx64B_AVX512_16 %endif @@ -376,11 +377,17 @@ dw 0x0000, 0xffff, 0x0000, 0x0000 dw 0xffff, 0x0000, 0x0000, 0x0000 dw 0x0000, 0x0000, 0x0000, 0x0000 -mov_mask: -db 10101010b, 10101011b, 10101110b, 10101111b -db 10111010b, 10111011b, 10111110b, 10111111b -db 11101010b, 11101011b, 11101110b, 11101111b -db 11111010b, 11111011b, 11111110b, 11111111b +mov_8B_mask: +dw 1100110011001100b, 1100110011001111b, 1100110011111100b, 1100110011111111b +dw 1100111111001100b, 1100111111001111b, 1100111111111100b, 1100111111111111b +dw 1111110011001100b, 1111110011001111b, 1111110011111100b, 1111110011111111b +dw 1111111111001100b, 1111111111001111b, 1111111111111100b, 1111111111111111b + +mov_4B_mask: +dw 1110111011101110b, 1110111011101111b, 1110111011111110b, 1110111011111111b +dw 1110111111101110b, 1110111111101111b, 1110111111111110b, 1110111111111111b +dw 1111111011101110b, 1111111011101111b, 1111111011111110b, 1111111011111111b +dw 1111111111101110b, 1111111111101111b, 1111111111111110b, 1111111111111111b align 64 idx_tags_64_0_7: @@ -806,7 +813,7 @@ align 64 ; then there is a gap of 256 bytes and then the next 256 bytes ; are written, and so on. ; -%macro STORE_KSTR16 18-24 +%macro STORE_KSTR16 18-25 %define %%KS %1 ; [in] Pointer to keystream %define %%DATA64B_L0 %2 ; [in] 64 bytes of keystream for lane 0 %define %%DATA64B_L1 %3 ; [in] 64 bytes of keystream for lane 1 @@ -831,6 +838,7 @@ align 64 %define %%TMP %22 ; [in] Temporary GP register %define %%KMASK1 %23 ; [clobbered] Temporary K mask %define %%KMASK2 %24 ; [clobbered] Temporary K mask +%define %%SKIP_ROUNDS %25 ; [constant] Number of rounds to skip %if (%0 == 18) vmovdqu64 [%%KS + %%KEY_OFF*4], %%DATA64B_L0 @@ -855,52 +863,52 @@ align 64 %else pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_0_4_8_12] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] + kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] ; Shifting left 8 bytes of KS for lanes which first 8 bytes are skipped - vpalignr %%DATA64B_L3{%%KMASK1}, %%DATA64B_L3, %%DATA64B_L2, 8 - vpalignr %%DATA64B_L2{%%KMASK1}, %%DATA64B_L2, %%DATA64B_L1, 8 - vpalignr %%DATA64B_L1{%%KMASK1}, %%DATA64B_L1, %%DATA64B_L0, 8 - vpalignr %%DATA64B_L0{%%KMASK1}, %%DATA64B_L0, %%DATA64B_L3, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4]{%%KMASK2}, %%DATA64B_L0 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 + vpalignr %%DATA64B_L3{%%KMASK1}, %%DATA64B_L3, %%DATA64B_L2, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L2{%%KMASK1}, %%DATA64B_L2, %%DATA64B_L1, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L1{%%KMASK1}, %%DATA64B_L1, %%DATA64B_L0, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L0{%%KMASK1}, %%DATA64B_L0, %%DATA64B_L3, (16 - %%SKIP_ROUNDS * 4) + vmovdqu32 [%%KS + %%KEY_OFF*4]{%%KMASK2}, %%DATA64B_L0 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 3*64], %%DATA64B_L3 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_1_5_9_13] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L7{%%KMASK1}, %%DATA64B_L7, %%DATA64B_L6, 8 - vpalignr %%DATA64B_L6{%%KMASK1}, %%DATA64B_L6, %%DATA64B_L5, 8 - vpalignr %%DATA64B_L5{%%KMASK1}, %%DATA64B_L5, %%DATA64B_L4, 8 - vpalignr %%DATA64B_L4{%%KMASK1}, %%DATA64B_L4, %%DATA64B_L7, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512]{%%KMASK2}, %%DATA64B_L4 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*2], %%DATA64B_L6 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512 + 64*3], %%DATA64B_L7 + kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] + vpalignr %%DATA64B_L7{%%KMASK1}, %%DATA64B_L7, %%DATA64B_L6, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L6{%%KMASK1}, %%DATA64B_L6, %%DATA64B_L5, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L5{%%KMASK1}, %%DATA64B_L5, %%DATA64B_L4, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L4{%%KMASK1}, %%DATA64B_L4, %%DATA64B_L7, (16 - %%SKIP_ROUNDS * 4) + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512]{%%KMASK2}, %%DATA64B_L4 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512 + 64*2], %%DATA64B_L6 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512 + 64*3], %%DATA64B_L7 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_2_6_10_14] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L11{%%KMASK1}, %%DATA64B_L11, %%DATA64B_L10, 8 - vpalignr %%DATA64B_L10{%%KMASK1}, %%DATA64B_L10, %%DATA64B_L9, 8 - vpalignr %%DATA64B_L9{%%KMASK1}, %%DATA64B_L9, %%DATA64B_L8, 8 - vpalignr %%DATA64B_L8{%%KMASK1}, %%DATA64B_L8, %%DATA64B_L11, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2]{%%KMASK2}, %%DATA64B_L8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 + kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] + vpalignr %%DATA64B_L11{%%KMASK1}, %%DATA64B_L11, %%DATA64B_L10, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L10{%%KMASK1}, %%DATA64B_L10, %%DATA64B_L9, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L9{%%KMASK1}, %%DATA64B_L9, %%DATA64B_L8, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L8{%%KMASK1}, %%DATA64B_L8, %%DATA64B_L11, (16 - %%SKIP_ROUNDS * 4) + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2]{%%KMASK2}, %%DATA64B_L8 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2 + 64*3], %%DATA64B_L11 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_3_7_11_15] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] - kmovb %%KMASK2, [%%MOV_MASK + %%TMP] - vpalignr %%DATA64B_L15{%%KMASK1}, %%DATA64B_L15, %%DATA64B_L14, 8 - vpalignr %%DATA64B_L14{%%KMASK1}, %%DATA64B_L14, %%DATA64B_L13, 8 - vpalignr %%DATA64B_L13{%%KMASK1}, %%DATA64B_L13, %%DATA64B_L12, 8 - vpalignr %%DATA64B_L12{%%KMASK1}, %%DATA64B_L12, %%DATA64B_L15, 8 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3]{%%KMASK2}, %%DATA64B_L12 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 - vmovdqu64 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 + kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] + vpalignr %%DATA64B_L15{%%KMASK1}, %%DATA64B_L15, %%DATA64B_L14, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L14{%%KMASK1}, %%DATA64B_L14, %%DATA64B_L13, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L13{%%KMASK1}, %%DATA64B_L13, %%DATA64B_L12, (16 - %%SKIP_ROUNDS * 4) + vpalignr %%DATA64B_L12{%%KMASK1}, %%DATA64B_L12, %%DATA64B_L15, (16 - %%SKIP_ROUNDS * 4) + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3]{%%KMASK2}, %%DATA64B_L12 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 + vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3 + 64*3], %%DATA64B_L15 %endif %endmacro @@ -1352,10 +1360,11 @@ init_for_auth_tag_8B: ; Generate N*4 bytes of keystream ; for 16 buffers (where N is number of rounds) ; -%macro KEYGEN_16_AVX512 2-3 +%macro KEYGEN_16_AVX512 3-4 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds %define %%KEY_OFF %2 ; [in] Offset to start writing Keystream -%define %%LANE_MASK %3 ; [in] Lane mask with lanes to generate all keystream words +%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip +%define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate all keystream words %define pState arg1 %define pKS arg2 @@ -1418,7 +1427,7 @@ init_for_auth_tag_8B: mov DWORD(%%TMP1), 0xAAAAAAAA kmovd %%BLEND_KMASK, DWORD(%%TMP1) -%if (%0 == 3) +%if (%0 == 4) kmovd %%FULL_LANE_KMASK, DWORD(%%LANE_MASK) knotd %%SKIP_LANE_KMASK, %%FULL_LANE_KMASK mov DWORD(%%TMP1), 0x0000FFFF @@ -1450,7 +1459,7 @@ init_for_auth_tag_8B: ; Generate first bytes of KS for all lanes %assign %%N 1 %assign %%IDX 1 -%rep (%%NUM_ROUNDS-2) +%rep (%%NUM_ROUNDS-%%SKIP_ROUNDS) BITS_REORG16 pState, %%N, %%ALL_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%IDX) NONLIN_FUN16 pState, %%ALL_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ @@ -1462,13 +1471,13 @@ init_for_auth_tag_8B: %assign %%N %%N+1 %assign %%IDX (%%IDX + 1) %endrep -%if (%%NUM_ROUNDS > 2) +%if (%%NUM_ROUNDS > %%SKIP_ROUNDS) vmovdqa32 [pState + OFS_R1]{%%ALL_KMASK}, %%R1 vmovdqa32 [pState + OFS_R2]{%%ALL_KMASK}, %%R2 %endif ; Generate rest of the KS bytes (last 8 bytes) for selected lanes -%rep 2 +%rep %%SKIP_ROUNDS BITS_REORG16 pState, %%N, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, APPEND(%%KSTR, %%IDX) NONLIN_FUN16 pState, %%FULL_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ @@ -1491,22 +1500,26 @@ init_for_auth_tag_8B: %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ %%ZTMP9, %%ZTMP10 -%if (%0 == 3) +%if (%0 == 4) lea %%TMP1, [rel alignr_mask] - lea %%TMP2, [rel mov_mask] +%if %%SKIP_ROUNDS == 1 + lea %%TMP2, [rel mov_4B_mask] +%else ; %%SKIP_ROUNDS == 2 + lea %%TMP2, [rel mov_8B_mask] +%endif STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF, \ - %%LANE_MASK, %%TMP1, %%TMP2, %%TMP3, %%TMP_KMASK1, %%TMP_KMASK2 + %%LANE_MASK, %%TMP1, %%TMP2, %%TMP3, %%TMP_KMASK1, %%TMP_KMASK2, %%SKIP_ROUNDS %else STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF %endif ; Reorder LFSR registers -%if (%0 == 3) +%if (%0 == 4) REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK -%if (%%NUM_ROUNDS >= 2) - REORDER_LFSR pState, (%%NUM_ROUNDS - 2), %%SKIP_LANE_KMASK ; 2 less rounds for "old" buffers +%if (%%NUM_ROUNDS >= %%SKIP_ROUNDS) + REORDER_LFSR pState, (%%NUM_ROUNDS - %%SKIP_ROUNDS), %%SKIP_LANE_KMASK ; 2 less rounds for "old" buffers %endif %else REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK @@ -1971,7 +1984,7 @@ ZUC_KEYGEN64B_16: FUNC_SAVE - KEYGEN_16_AVX512 16, arg3 + KEYGEN_16_AVX512 16, arg3, 0 FUNC_RESTORE @@ -2040,7 +2053,23 @@ ZUC_KEYGEN64B_SKIP8_16: FUNC_SAVE - KEYGEN_16_AVX512 16, arg3, arg4 + KEYGEN_16_AVX512 16, arg3, 2, arg4 + + FUNC_RESTORE + + ret + +;; void asm_ZucGenKeystream64B_16_skip4_avx512(state16_t *pSta, u32* pKeyStr[16], +;; const u32 key_off, +;; const u16 lane_mask) +;; +MKGLOBAL(ZUC_KEYGEN64B_SKIP4_16,function,internal) +ZUC_KEYGEN64B_SKIP4_16: + endbranch64 + + FUNC_SAVE + + KEYGEN_16_AVX512 16, arg3, 1, arg4 FUNC_RESTORE @@ -2056,16 +2085,17 @@ ZUC_KEYGEN8B_16: FUNC_SAVE - KEYGEN_16_AVX512 2, arg3 + KEYGEN_16_AVX512 2, arg3, 0 FUNC_RESTORE ret -%macro KEYGEN_VAR_16_AVX512 2-3 +%macro KEYGEN_VAR_16_AVX512 3-4 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds (GP dowrd register) %define %%KEY_OFF %2 ; [in] Offset to start writing Keystream -%define %%LANE_MASK %3 ; [in] Lane mask with lanes to generate full keystream (rest 2 words less) +%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip +%define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate full keystream (rest 1-2 words less) cmp %%NUM_ROUNDS, 16 je %%_num_rounds_is_16 @@ -2110,10 +2140,10 @@ ZUC_KEYGEN8B_16: %assign %%I 1 %rep 16 APPEND(%%_num_rounds_is_,%%I): -%if (%0 == 3) - KEYGEN_16_AVX512 %%I, %%KEY_OFF, %%LANE_MASK +%if (%0 == 4) + KEYGEN_16_AVX512 %%I, %%KEY_OFF, %%SKIP_ROUNDS, %%LANE_MASK %else - KEYGEN_16_AVX512 %%I, %%KEY_OFF + KEYGEN_16_AVX512 %%I, %%KEY_OFF, 0 %endif jmp %%_done @@ -2134,7 +2164,7 @@ ZUC_KEYGEN_16: FUNC_SAVE - KEYGEN_VAR_16_AVX512 arg4, arg3 + KEYGEN_VAR_16_AVX512 arg4, arg3, 0 FUNC_RESTORE @@ -2154,7 +2184,27 @@ ZUC_KEYGEN_SKIP8_16: FUNC_SAVE - KEYGEN_VAR_16_AVX512 r10d, arg3, arg4 + KEYGEN_VAR_16_AVX512 r10d, arg3, 2, arg4 + + FUNC_RESTORE + + ret + +;; +;; void asm_ZucGenKeystream_16_skip4_avx512(state16_t *pSta, u32* pKeyStr[16], +;; const u32 key_off, +;; const u16 lane_mask, +;; u32 numRounds) +;; +MKGLOBAL(ZUC_KEYGEN_SKIP4_16,function,internal) +ZUC_KEYGEN_SKIP4_16: + endbranch64 + + mov r10, arg5 + + FUNC_SAVE + + KEYGEN_VAR_16_AVX512 r10d, arg3, 1, arg4 FUNC_RESTORE diff --git a/lib/avx512/zuc_x16_vaes_avx512.asm b/lib/avx512/zuc_x16_vaes_avx512.asm index 138d0584..65f40001 100644 --- a/lib/avx512/zuc_x16_vaes_avx512.asm +++ b/lib/avx512/zuc_x16_vaes_avx512.asm @@ -37,8 +37,9 @@ %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_gfni_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_gfni_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_gfni_avx512 -%define ZUC_KEYGEN8B_SKIP8_16 asm_ZucGenKeystream8B_16_skip8_gfni_avx512 +%define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_gfni_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_gfni_avx512 +%define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_gfni_avx512 %define ZUC_ROUND64B_16 asm_Eia3Round64B_16_VPCLMUL %define ZUC_EIA3_N64B asm_Eia3_Nx64B_AVX512_16_VPCLMUL %include "avx512/zuc_x16_avx512.asm" diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index eb1a5af9..39e3b0f3 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -700,6 +700,43 @@ asm_ZucGenKeystream64B_16_skip8_gfni_avx512(ZucState16_t *pState, uint32_t *pKeyStr, const unsigned key_off, const uint16_t lane_mask); +/** + ****************************************************************************** + * + * @description + * Definition of the external function that implements the working + * stage of the ZUC algorithm. The function will generate 64 bytes of + * keystream for 16 packets in parallel, except for selected lanes, + * which will have 60 bytes of keystream generated instead. + * + * @param[in] pState Pointer to a ZUC state structure of type + * @ref ZucState16_t + * + * @param[in,out] pKeyStr Array of pointers to 16 input buffers + * that will contain the generated keystream + * for these 16 packets. + * + * @param[in] key_off Starting offset for writing KS. + * + * @param[in] lane_mask Mask containing lanes which will have 64 + * bytes of KS generated (4 bytes less for + * the rest) + * + * @pre + * A successful call to @ref asm_ZucInitialization_16_avx512 to initialize + * the ZUC state. + * + *****************************************************************************/ +IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_skip4_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); + +IMB_DLL_LOCAL void +asm_ZucGenKeystream64B_16_skip4_gfni_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); /** ****************************************************************************** * @@ -940,6 +977,48 @@ asm_ZucGenKeystream_16_skip8_gfni_avx512(ZucState16_t *pState, const unsigned key_off, const uint16_t lane_mask, const uint32_t numRounds); + +/** + ****************************************************************************** + * + * @description + * Definition of the external function that implements the working + * stage of the ZUC algorithm. The function will generate N*4 bytes of + * keystream for sixteen packets in parallel. + * + * @param[in] pState Pointer to a ZUC state structure of type + * @ref ZucState16_t + * + * @param[in,out] pKeyStr Array of pointers to 16 input buffers + * that will contain the generated keystream + * for these 16 packets. + * + * @param[in] key_off Starting offset for writing KS. + * + * @param[in] lane_mask Mask containing lanes which will have N*4 + * bytes of KS generated (4 bytes less for + * the rest) + * + * @param[in] numRounds Number of 4-byte rounds (1 to 16 rounds) + * + * @pre + * A successful call to @ref asm_ZucInitialization to initialize the ZUC + * state. + * + *****************************************************************************/ +IMB_DLL_LOCAL void +asm_ZucGenKeystream_16_skip4_avx512(ZucState16_t *pState, + uint32_t *pKstr, + const unsigned key_off, + const uint16_t lane_mask, + const uint32_t numRounds); + +IMB_DLL_LOCAL void +asm_ZucGenKeystream_16_skip4_gfni_avx512(ZucState16_t *pState, + uint32_t *pKstr, + const unsigned key_off, + const uint16_t lane_mask, + const uint32_t numRounds); /** ****************************************************************************** * -- GitLab From e268c2f0c6d557b873935b21faa739b3934c557a Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 5 May 2022 19:15:48 +0000 Subject: [PATCH 093/369] avx512: [ZUC-EIA3-256] Copy last 4 KS bytes to start Copy last 4 bytes of keystream if there is only one round to copy, when generating 4-byte tags with ZUC-EIA3-256. --- lib/avx512/zuc_x16_avx512.asm | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 6992c4bc..e1909c96 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -3189,16 +3189,27 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): add rsp, 32 - ; Memcpy last 8 bytes of %%KS into start add DWORD(%%MIN_LEN), 31 shr DWORD(%%MIN_LEN), 5 - shl DWORD(%%MIN_LEN), 2 ; Offset where to copy the last 8 bytes from + shl DWORD(%%MIN_LEN), 2 ; Offset where to copy the last 4/8 bytes from + +%if %%KEY_SIZE == 128 +%define %%KS_WORDS_TO_COPY 2 +%else ;; %%KEY_SIZE == 256 +%if %%TAG_SIZE == 4 +%define %%KS_WORDS_TO_COPY 1 +%elif %%TAG_SIZE == 8 +%define %%KS_WORDS_TO_COPY 2 +%endif +%endif ;; %%KEY_SIZE mov DWORD(%%TMP1), DWORD(%%MIN_LEN) shr DWORD(%%MIN_LEN), 4 shl DWORD(%%MIN_LEN), (4+2) and DWORD(%%TMP1), 0xf add DWORD(%%MIN_LEN), DWORD(%%TMP1) +%if %%KS_WORDS_TO_COPY == 2 + ; Memcpy last 8 bytes of KS into start cmp DWORD(%%TMP1), 12 je %%_copy_2dwords @@ -3228,6 +3239,19 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): %endrep %assign %%i (%%i + 1) %endrep +%elif %%KS_WORDS_TO_COPY == 1 + ; Memcpy last 4 bytes of KS into start +%assign %%i 0 +%rep 4 +%assign %%j 0 +%rep 4 + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], DWORD(%%TMP1) +%assign %%j (%%j + 1) +%endrep +%assign %%i (%%i + 1) +%endrep +%endif ; %%KS_WORDS_TO_COPY %%_ks_copied: vzeroupper %endmacro ; REMAINDER_16 -- GitLab From a227b4905161f1e12a99935c97f300c7d5d18ec0 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 09:03:52 +0000 Subject: [PATCH 094/369] avx512: [ZUC-EIA3-256] Support 8-byte tags in submit/flush --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 104 ++++++++++++++---- lib/include/ipsec_ooo_mgr.h | 2 +- lib/include/mb_mgr_datastruct.asm | 2 +- 3 files changed, 83 insertions(+), 25 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index 831d6759..74602159 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -601,12 +601,13 @@ FLUSH_JOB_ZUC256_EEA3: endbranch64 FLUSH_JOB_ZUC_EEA3 256 -%macro ZUC_EIA3_16_BUFFER 5 +%macro ZUC_EIA3_16_BUFFER 6 %define %%OOO %1 ; [in] Pointer to ZUC OOO manager %define %%KEY_SIZE %2 ; [constant] Key size (16 or 32) -%define %%L %3 ; [clobbered] Temporary GP register (dword) -%define %%REMAIN_BITS %4 ; [clobbered] Temporary GP register (dword) -%define %%TMP %5 ; [clobbered] Temporary GP register +%define %%TAG_SIZE %3 ; [constant] Tag size (4 or 8 bytes) +%define %%L %4 ; [clobbered] Temporary GP register (dword) +%define %%REMAIN_BITS %5 ; [clobbered] Temporary GP register (dword) +%define %%TMP %6 ; [clobbered] Temporary GP register ; Find minimum length vmovdqa xmm0, [%%OOO + _zuc_lens] @@ -618,8 +619,13 @@ FLUSH_JOB_ZUC256_EEA3: cmp DWORD(%%TMP), %%REMAIN_BITS cmovbe %%REMAIN_BITS, DWORD(%%TMP) - ; Get number of KS 32-bit words to generate ([length/32] + 2)) - lea %%L, [%%REMAIN_BITS + 31 + (2 << 5)] + ; Get number of KS 32-bit words to generate ([length/32] + tag_size)) +%if %%KEY_SIZE == 128 + lea %%L, [%%REMAIN_BITS + 31 + 2*(8*%%TAG_SIZE)] +%else ; %%KEY_SIZE == 256 + lea %%L, [%%REMAIN_BITS + 31 + (8*%%TAG_SIZE)] +%endif + shr %%L, 5 cmp %%L, 16 @@ -640,7 +646,15 @@ FLUSH_JOB_ZUC256_EEA3: mov [rsp + 32], %%L %endif +%if %%KEY_SIZE == 128 + call ZUC_KEYGEN_SKIP8_16 +%else ; %%KEY_SIZE == 256 +%if %%TAG_SIZE == 4 + call ZUC_KEYGEN_SKIP4_16 +%else ;; %%TAG_SIZE = 8 call ZUC_KEYGEN_SKIP8_16 +%endif +%endif ; %%KEY_SIZE RESTORE_STACK_SPACE 5 @@ -654,7 +668,15 @@ FLUSH_JOB_ZUC256_EEA3: xor arg3, arg3 ; offset = 0 movzx DWORD(arg4), word [%%OOO + _zuc_init_not_done] +%if %%KEY_SIZE == 128 + call ZUC_KEYGEN64B_SKIP8_16 +%else ; %%KEY_SIZE == 256 +%if %%TAG_SIZE == 4 + call ZUC_KEYGEN64B_SKIP4_16 +%else ;; %%TAG_SIZE = 8 call ZUC_KEYGEN64B_SKIP8_16 +%endif +%endif sub %%L, 16 %%_loop: @@ -677,7 +699,7 @@ FLUSH_JOB_ZUC256_EEA3: lea arg2, [%%OOO + _zuc_args_KS] lea arg3, [%%OOO + _zuc_args_in] lea arg4, [%%OOO + _zuc_lens] - mov arg5, 4 + mov arg5, %%TAG_SIZE call ZUC_ROUND64B @@ -708,7 +730,7 @@ FLUSH_JOB_ZUC256_EEA3: lea %%TMP, [%%OOO + _zuc_lens] mov [rsp + 32], %%TMP %endif - mov arg7, 4 ; Hardcoded to 4 (tag size) + mov arg7, %%TAG_SIZE call ZUC_EIA3_N64B @@ -746,7 +768,7 @@ FLUSH_JOB_ZUC256_EEA3: %if %%KEY_SIZE == 128 call ZUC_REMAINDER_16 %else - mov arg6, 4 ; Hardcoded to 4-byte digest for now + mov arg6, %%TAG_SIZE call ZUC256_REMAINDER_16 %endif @@ -755,8 +777,9 @@ FLUSH_JOB_ZUC256_EEA3: mov word [%%OOO + _zuc_init_not_done], 0 %endmacro -%macro SUBMIT_JOB_ZUC_EIA3 1 +%macro SUBMIT_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8) ; idx needs to be in rbp %define len rbp @@ -844,7 +867,11 @@ FLUSH_JOB_ZUC256_EEA3: not tmp and [state + _zuc_unused_lane_bitmask], WORD(tmp) ; Reset temporary digest for the lane +%if %%TAG_SIZE == 4 mov dword [state + _zuc_args_digest + lane*4], 0 +%else + mov qword [state + _zuc_args_digest + lane*8], 0 +%endif mov tmp, [job + _src] add tmp, [job + _hash_start_src_offset_in_bytes] @@ -899,14 +926,14 @@ FLUSH_JOB_ZUC256_EEA3: lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] movzx DWORD(arg4), word [r12 + _zuc_init_not_done] - mov r10, 4 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + mov r10, %%TAG_SIZE ; Argument 5 hardcoded to r10, as INIT is expecting it in that register lea r11, [r12 + _zuc_args_digest] ; Argument 6 hardcoded to r11 call ZUC256_INIT_16 %endif ;; %%KEY_SIZE == 128 - ZUC_EIA3_16_BUFFER r12, %%KEY_SIZE, DWORD(tmp), DWORD(tmp2), tmp3 + ZUC_EIA3_16_BUFFER r12, %%KEY_SIZE, %%TAG_SIZE, DWORD(tmp), DWORD(tmp2), tmp3 mov state, [rsp + _gpr_save + 8*8] mov job, [rsp + _gpr_save + 8*9] @@ -920,9 +947,14 @@ FLUSH_JOB_ZUC256_EEA3: mov qword [state + _zuc_job_in_lane + idx*8], 0 or dword [job_rax + _status], IMB_STATUS_COMPLETED_AUTH ; Copy digest to auth tag output - mov r10d, [state + _zuc_args_digest + idx*4] mov r11, [job_rax + _auth_tag_output] +%if %%TAG_SIZE == 4 + mov r10d, [state + _zuc_args_digest + idx*4] mov [r11], r10d +%elif %%TAG_SIZE == 8 + mov r10, [state + _zuc_args_digest + idx*8] + mov [r11], r10 +%endif shl unused_lanes, 4 or unused_lanes, idx mov [state + _zuc_unused_lanes], unused_lanes @@ -963,8 +995,9 @@ FLUSH_JOB_ZUC256_EEA3: jmp %%return_submit_eia3 %endmacro -%macro FLUSH_JOB_ZUC_EIA3 1 +%macro FLUSH_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) +%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8) %define unused_lanes rbx %define tmp1 rbx @@ -1074,7 +1107,7 @@ FLUSH_JOB_ZUC256_EEA3: lea arg2, [r12 + _zuc_args_IV] lea arg3, [r12 + _zuc_state] movzx DWORD(arg4), word [r12 + _zuc_init_not_done] - mov r10, 4 ; Argument 5 hardcoded to r10, as INIT is expecting it in that register + mov r10, %%TAG_SIZE ; Argument 5 hardcoded to r10, as INIT is expecting it in that register lea r11, [r12 + _zuc_args_digest] ; Argument 6 hardcoded to r11 call ZUC256_INIT_16 @@ -1082,7 +1115,7 @@ FLUSH_JOB_ZUC256_EEA3: %endif ;; %%KEY_SIZE == 128 %%skip_init_flush_eia3: - ZUC_EIA3_16_BUFFER r12, %%KEY_SIZE, DWORD(tmp), DWORD(tmp2), tmp4 + ZUC_EIA3_16_BUFFER r12, %%KEY_SIZE, %%TAG_SIZE, DWORD(tmp), DWORD(tmp2), tmp4 mov state, [rsp + _gpr_save + 8*8] @@ -1113,9 +1146,14 @@ FLUSH_JOB_ZUC256_EEA3: mov qword [state + _zuc_job_in_lane + idx*8], 0 or dword [job_rax + _status], IMB_STATUS_COMPLETED_AUTH ; Copy digest to auth tag output - mov r10d, [state + _zuc_args_digest + idx*4] mov r11, [job_rax + _auth_tag_output] +%if %%TAG_SIZE == 4 + mov r10d, [state + _zuc_args_digest + idx*4] mov [r11], r10d +%elif %%TAG_SIZE == 8 + mov r10, [state + _zuc_args_digest + idx*8] + mov [r11], r10 +%endif shl unused_lanes, 4 or unused_lanes, idx mov [state + _zuc_unused_lanes], unused_lanes @@ -1162,28 +1200,48 @@ FLUSH_JOB_ZUC256_EEA3: MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: endbranch64 - SUBMIT_JOB_ZUC_EIA3 128 + SUBMIT_JOB_ZUC_EIA3 128, 4 -; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) +; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job, +; const uint64_t tag_sz) ; arg 1 : state ; arg 2 : job +; arg 3 : tag size (4 or 8 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: endbranch64 - SUBMIT_JOB_ZUC_EIA3 256 + cmp arg3, 8 + je submit_tag_8B + jb submit_tag_4B + +submit_tag_8B: + SUBMIT_JOB_ZUC_EIA3 256, 8 + +submit_tag_4B: + SUBMIT_JOB_ZUC_EIA3 256, 4 ; JOB* FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: endbranch64 - FLUSH_JOB_ZUC_EIA3 128 + FLUSH_JOB_ZUC_EIA3 128, 4 -; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state) +; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, +; const uint64_t tag_sz) ; arg 1 : state +; arg 2 : tag size (4 or 8 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: endbranch64 - FLUSH_JOB_ZUC_EIA3 256 + cmp arg2, 8 + je flush_tag_8B + jb flush_tag_4B + +flush_tag_8B: + FLUSH_JOB_ZUC_EIA3 256, 8 + +flush_tag_4B: + FLUSH_JOB_ZUC_EIA3 256, 4 mksection stack-noexec diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index ae42aeaf..632d58a9 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -155,7 +155,7 @@ typedef struct { DECLARE_ALIGNED(uint8_t *out[16], 64); const uint8_t *keys[16]; DECLARE_ALIGNED(uint8_t iv[16*32], 32); - DECLARE_ALIGNED(uint32_t digest[16], 64); + DECLARE_ALIGNED(uint32_t digest[16*2], 64); /* Memory for 128 bytes of KS for 16 buffers */ DECLARE_ALIGNED(uint32_t ks[16 * 2 * 16], 64); } ZUC_ARGS_x16; diff --git a/lib/include/mb_mgr_datastruct.asm b/lib/include/mb_mgr_datastruct.asm index f61595cf..eadf0668 100644 --- a/lib/include/mb_mgr_datastruct.asm +++ b/lib/include/mb_mgr_datastruct.asm @@ -239,7 +239,7 @@ FIELD _zucarg_in, 16*8, 64 ; array of 16 pointers to in text FIELD _zucarg_out, 16*8, 64 ; array of 16 pointers to out text FIELD _zucarg_keys, 16*8, 8 ; array of 16 pointers to keys FIELD _zucarg_IV, 16*32, 32 ; array of IVs (up to 25 bytes each) -FIELD _zucarg_digest, 16*4, 64 ; array of 16 digests +FIELD _zucarg_digest, 16*8, 64 ; array of 16 digests FIELD _zucarg_KS, 16*128, 64 ; array of 128-byte keystream of 16 buffers END_FIELDS %assign _ZUC_ARGS_X16_size _FIELD_OFFSET -- GitLab From 14355326ad2fa2804fa9d3944bf835d7f31955b0 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 09:08:30 +0000 Subject: [PATCH 095/369] test: [ZUC-EIA3-256] Disable 16-byte tag tests for now Disable 16-byte tag tests until AVX512 supports them. --- test/zuc_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/zuc_test.c b/test/zuc_test.c index 1b150da6..306a32bb 100644 --- a/test/zuc_test.c +++ b/test/zuc_test.c @@ -1088,7 +1088,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, memcpy(pSrcData[j], vector->message, byteLength); iv_lens[j] = vector->iv_length; } - for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { + for (tag_sz = 4; tag_sz <= 8; tag_sz *= 2) { submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, @@ -1117,7 +1117,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, iv_lens[i] = vector->iv_length; } - for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { + for (tag_sz = 4; tag_sz <= 8; tag_sz *= 2) { submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, -- GitLab From 940336dc060ca8bb6993781e498788992e8c625d Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Mon, 9 May 2022 12:56:32 +0000 Subject: [PATCH 096/369] doc: update workflow badges in README --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index afe6f554..d8a37612 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ [![Coverity Status](https://scan.coverity.com/projects/16449/badge.svg)](https://scan.coverity.com/projects/intel-ipsec-mb) -![Linux Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_make.yml/badge.svg) -![Linux Build Static gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_static_make.yml/badge.svg) -![Linux Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_make.yml/badge.svg) -![Linux Build Static clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_static_make.yml/badge.svg) -![Linux Build Shared clang AESNI emulation](https://github.com/intel/intel-ipsec-mb/actions/workflows/clang_aesni_make.yml/badge.svg) -![FreeBSD Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_make.yml/badge.svg) -![FreeBSD Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_gcc_make.yml/badge.svg) +![Linux Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_gcc.yml/badge.svg) +![Linux Build Static gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_static_gcc.yml/badge.svg) +![Linux Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_clang.yml/badge.svg) +![Linux Build Static clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_static_clang.yml/badge.svg) +![Linux Build Shared clang AESNI emulation](https://github.com/intel/intel-ipsec-mb/actions/workflows/linux_build_shared_clang_aesni_emu.yml/badge.svg) +![FreeBSD Build Shared clang](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_build_shared_clang.yml/badge.svg) +![FreeBSD Build Shared gcc](https://github.com/intel/intel-ipsec-mb/actions/workflows/freebsd_build_shared_gcc.yml/badge.svg) # Intel(R) Multi-Buffer Crypto for IPsec Library -- GitLab From 2cd0f6851f09862ec81ab998c674aa2f2fe77d3b Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 21 Apr 2022 10:45:39 +0000 Subject: [PATCH 097/369] sse: [SHA1] initial C implementation --- lib/include/ipsec_ooo_mgr.h | 2 +- lib/include/mb_mgr_code.h | 285 +++++++++++++++++++++++++++++++++++- lib/intel-ipsec-mb.h | 1 + lib/sse/mb_mgr_sse.c | 12 ++ lib/sse/sha1_x4_sse.asm | 75 +++++++++- lib/x86_64/alloc.c | 1 + 6 files changed, 370 insertions(+), 6 deletions(-) diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 632d58a9..4fb683f9 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -114,7 +114,7 @@ typedef struct { typedef struct { DECLARE_ALIGNED(uint32_t digest[SHA1_DIGEST_SZ], 32); - uint8_t *data_ptr[AVX512_NUM_SHA1_LANES]; + const uint8_t *data_ptr[AVX512_NUM_SHA1_LANES]; } SHA1_ARGS; typedef struct { diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 02612e28..7c67bc84 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1,3 +1,4 @@ + /******************************************************************************* Copyright (c) 2012-2022, Intel Corporation @@ -46,6 +47,7 @@ #include "intel-ipsec-mb.h" #include "error.h" #include "include/snow3g_submit.h" +#include "constants.h" #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -53,6 +55,10 @@ #define BSWAP64 _byteswap_uint64 #endif +extern void sha1_mult_sse(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void sha1_block_sse(const void *, void *); +extern void sha1_block_avx(const void *, void *); + #define CRC(func, state, job) *((uint32_t *)job->auth_tag_output) = \ func(state, job->src + job->hash_start_src_offset_in_bytes, \ job->msg_len_to_hash_in_bytes) @@ -77,6 +83,263 @@ void ADV_JOBS(int *ptr) *ptr = 0; } +/* ========================================================================= */ +/* SHA1 */ +/* ========================================================================= */ +__forceinline +void +sha_generic_one_block(const void *inp, void *digest, + const int is_avx, const int sha_type) +{ + if (sha_type == 1) { + if (is_avx) + sha1_block_avx(inp, digest); + else + sha1_block_sse(inp, digest); + } +} + +__forceinline +uint32_t mbswap4(const uint32_t val) +{ + return ((val >> 24) | + ((val & 0xff0000) >> 8) | + ((val & 0xff00) << 8) | + (val << 24)); +} + +__forceinline +uint64_t mbswap8(const uint64_t val) +{ + return (((uint64_t) mbswap4((uint32_t) val)) << 32) | + (((uint64_t) mbswap4((uint32_t) (val >> 32)))); +} + +__forceinline +void copy_bswap4_array(void *dst, const void *src, const size_t num) +{ + uint32_t *outp = (uint32_t *) dst; + const uint32_t *inp = (const uint32_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = mbswap4(inp[i]); +} + +__forceinline +void copy_bswap8_array(void *dst, const void *src, const size_t num) +{ + uint64_t *outp = (uint64_t *) dst; + const uint64_t *inp = (const uint64_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = mbswap8(inp[i]); +} + +__forceinline +void sha1_init_digest(void *d, unsigned lane) +{ + uint32_t *digest = (uint32_t *)d; + + digest[lane + 0*16] = H0; + digest[lane + 1*16] = H1; + digest[lane + 2*16] = H2; + digest[lane + 3*16] = H3; + digest[lane + 4*16] = H4; +} + +__forceinline +void +sha_generic_init(void *digest, const int sha_type, unsigned lane) +{ + if (sha_type == 1) + sha1_init_digest(digest, lane); +} + +__forceinline +void sha1_sc_digest(void *p, void *d, unsigned lane) +{ + uint32_t *p_digest = (uint32_t *)p; + uint32_t *d_digest = (uint32_t *)d; + + p_digest[0] = d_digest[lane + 0*16]; + p_digest[1] = d_digest[lane + 1*16]; + p_digest[2] = d_digest[lane + 2*16]; + p_digest[3] = d_digest[lane + 3*16]; + p_digest[4] = d_digest[lane + 4*16]; +} + +__forceinline +void +sha_generic_sc(void *digest, void *d, unsigned lane, const int sha_type) +{ + if (sha_type == 1) + sha1_sc_digest(digest, d, lane); +} + +__forceinline +void sha_generic_write_digest(void *dst, const void *src, const int sha_type) +{ + if (sha_type == 1) + copy_bswap4_array(dst, src, NUM_SHA_DIGEST_WORDS); +} + +__forceinline +void store8_be(void *outp, const uint64_t val) +{ + *((uint64_t *)outp) = mbswap8(val); +} + +__forceinline +void var_memcpy(void *dst, const void *src, const uint64_t len) +{ + uint64_t i; + const uint8_t *src8 = (const uint8_t *)src; + uint8_t *dst8 = (uint8_t *)dst; + + for (i = 0; i < len; i++) + dst8[i] = src8[i]; +} + +__forceinline +IMB_JOB * +submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, + const unsigned max_jobs, const int is_submit, + const int is_avx, const int sha_type, + const uint64_t blk_size, const uint64_t pad_size) +{ + unsigned lane, min_len, min_idx; + IMB_JOB *ret_job = NULL; + unsigned i; + uint8_t cb[IMB_SHA_512_BLOCK_SIZE]; + union { + uint32_t digest1[NUM_SHA_256_DIGEST_WORDS]; + uint64_t digest2[NUM_SHA_512_DIGEST_WORDS]; + } local_digest; + void *ld = (void *) &local_digest; + uint64_t r; + + if (is_submit) { + /* + * SUBMIT + * - get a free lane id + */ + + lane = state->unused_lanes & 15; + state->unused_lanes >>= 4; + state->args.data_ptr[lane] = + job->src + job->hash_start_src_offset_in_bytes; + + sha_generic_init(state->args.digest, sha_type, lane); + + /* copy job data in and set up initial blocks */ + state->ldata[lane].job_in_lane = job; + state->lens[lane] = (uint16_t)job->msg_len_to_hash_in_bytes; + + /* enough jobs to start processing? */ + if (state->unused_lanes != 0xf) + return NULL; + } else { + /* + * FLUSH + * - find 1st non null job + */ + for (lane = 0; lane < max_jobs; lane++) + if (state->ldata[lane].job_in_lane != NULL) + break; + if (lane >= max_jobs) + return NULL; /* no not null job */ + } + + if (is_submit) { + /* + * SUBMIT + * - find min common length to process + */ + min_idx = 0; + min_len = state->lens[0]; + + for (i = 1; i < max_jobs; i++) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } + } else { + /* + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across not null lanes + */ + min_idx = lane; + min_len = state->lens[lane]; + + for (i = 0; i < max_jobs; i++) { + if (i == lane) + continue; + + if (state->ldata[i].job_in_lane != NULL) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } else { + state->args.data_ptr[i] = + state->args.data_ptr[lane]; + state->lens[i] = UINT16_MAX; + } + } + } + + /* subtract min len from all lanes */ + for (i = 0; i < max_jobs; i++) + state->lens[i] -= (uint16_t)((min_len/blk_size)*blk_size); + + r = min_len % blk_size; + + /* run the algorithmic code on full selected blocks */ + if(min_len >= blk_size) + sha1_mult_sse(&state->args, (uint32_t)(min_len/blk_size)); + + /* process partial block */ + memset(cb, 0, sizeof(cb)); + var_memcpy(cb, state->args.data_ptr[min_idx], r); + cb[r] = 0x80; + + sha_generic_sc(ld, state->args.digest, min_idx, sha_type); + + if (r >= (blk_size - pad_size)) { + /* length will be encoded in the next block */ + sha_generic_one_block(cb, ld, is_avx, sha_type); + memset(cb, 0, sizeof(cb)); + } + + ret_job = state->ldata[min_idx].job_in_lane; + + /* encode bit length */ + store8_be(&cb[blk_size - 8], ret_job->msg_len_to_hash_in_bytes * 8); + sha_generic_one_block(cb, ld, is_avx, sha_type); + + /* put back processed packet into unused lanes, set job as complete */ + state->unused_lanes = (state->unused_lanes << 4) | min_idx; + sha_generic_write_digest(ret_job->auth_tag_output, ld, sha_type); + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; + state->ldata[min_idx].job_in_lane = NULL; + +#ifdef SAFE_DATA + clear_mem(cb, sizeof(cb)); + clear_mem(&local_digest, sizeof(local_digest)); + clear_scratch_gps(); + if (is_avx) + clear_scratch_xmms_avx(); + else + clear_scratch_xmms_sse(); +#endif + + return ret_job; +} + /* ========================================================================= */ /* Lower level "out of order" schedulers */ /* ========================================================================= */ @@ -1006,6 +1269,9 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; +#ifndef AVX2 + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -1063,11 +1329,17 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: + #ifdef AVX2 IMB_SHA1(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; + #else + return submit_flush_job_sha_1(sha_1_ooo, job, 4, 1, 0, 1, + IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE); + #endif case IMB_AUTH_SHA_224: IMB_SHA224(state, job->src + job->hash_start_src_offset_in_bytes, @@ -1211,6 +1483,9 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; +#ifndef AVX2 + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -1240,6 +1515,12 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); + #ifndef AVX2 + case IMB_AUTH_SHA_1: + return submit_flush_job_sha_1(sha_1_ooo, job, 4, 0, 0, 1, + IMB_SHA1_BLOCK_SIZE, + SHA1_PAD_SIZE); + #endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index beb06dda..9fc02ec6 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1188,6 +1188,7 @@ typedef struct IMB_MGR { void *aes256_cmac_ooo; void *snow3g_uea2_ooo; void *snow3g_uia2_ooo; + void *sha_1_ooo; } IMB_MGR; /** diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index a7529cda..1f29deba 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -714,6 +714,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, sizeof(aes128_ooo->lens)); @@ -1168,8 +1169,19 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(aes128_cbcs_ooo->job_in_lane)); aes128_cbcs_ooo->num_lanes_inuse = 0; aes128_cbcs_ooo->unused_lanes = 0xF3210; + + /* Init SHA1 out-of-order fields */ + sha_1_ooo->lens[0] = 0; + sha_1_ooo->lens[1] = 0; + sha_1_ooo->lens[2] = 0; + sha_1_ooo->lens[3] = 0; + sha_1_ooo->unused_lanes = 0xF3210; + for (j = 0; j < SSE_NUM_SHA1_LANES; j++) + sha_1_ooo->ldata[j].job_in_lane = NULL; } + + IMB_DLL_LOCAL void init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) { diff --git a/lib/sse/sha1_x4_sse.asm b/lib/sse/sha1_x4_sse.asm index f88c077c..8ffe44e9 100644 --- a/lib/sse/sha1_x4_sse.asm +++ b/lib/sse/sha1_x4_sse.asm @@ -78,6 +78,12 @@ mksection .text ; r0 = {d2 c2 b2 a2} ; r3 = {d3 c3 b3 a3} ; +%define XMM_STORAGE 16*10 +%define GP_STORAGE 8*3 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + %macro TRANSPOSE 6 %define %%r0 %1 %define %%r1 %2 @@ -224,7 +230,7 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; FRAMESZ must be an odd multiple of 8 -%define FRAMESZ 16*16 + 8 +%define FRAMESZ 16*16 %define MOVPS movdqu @@ -288,6 +294,69 @@ mksection .text %xdefine W14 TMP_ %endm +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + movdqa [rsp + 0*16], xmm6 + movdqa [rsp + 1*16], xmm7 + movdqa [rsp + 2*16], xmm8 + movdqa [rsp + 3*16], xmm9 + movdqa [rsp + 4*16], xmm10 + movdqa [rsp + 5*16], xmm11 + movdqa [rsp + 6*16], xmm12 + movdqa [rsp + 7*16], xmm13 + movdqa [rsp + 8*16], xmm14 + movdqa [rsp + 9*16], xmm15 +%endif +%ifidn __OUTPUT_FORMAT__, win64 + ; rdi and rdx need to be maintained for Windows + mov [rsp + GP_OFFSET], rdi + mov [rsp + GP_OFFSET + 8], r11 ;; rsp pointer +%else + mov [rsp + GP_OFFSET], rsi + mov [rsp + GP_OFFSET + 8], r11 ;; rsp pointer +%endif +%endmacro + +%macro FUNC_RESTORE 0 +%ifndef LINUX + movdqa xmm6, [rsp + 0*16] + movdqa xmm7, [rsp + 1*16] + movdqa xmm8, [rsp + 2*16] + movdqa xmm9, [rsp + 3*16] + movdqa xmm10, [rsp + 4*16] + movdqa xmm11, [rsp + 5*16] + movdqa xmm12, [rsp + 6*16] + movdqa xmm13, [rsp + 7*16] + movdqa xmm14, [rsp + 8*16] + movdqa xmm15, [rsp + 9*16] +%endif + + pxor xmm5, xmm5 + movdqa [rsp + 0*16], xmm5 + movdqa [rsp + 1*16], xmm5 + movdqa [rsp + 2*16], xmm5 + movdqa [rsp + 3*16], xmm5 + movdqa [rsp + 4*16], xmm5 + movdqa [rsp + 5*16], xmm5 + movdqa [rsp + 6*16], xmm5 + movdqa [rsp + 7*16], xmm5 + movdqa [rsp + 8*16], xmm5 + movdqa [rsp + 9*16], xmm5 + +%ifidn __OUTPUT_FORMAT__, win64 + ; rdi and rdx need to be maintained for Windows + mov rdi, [rsp + GP_OFFSET] + mov rsp, [rsp + GP_OFFSET + 8] ;; rsp pointer +%else + ; rsi needs to be maintained for Linux + mov rsi, [rsp + GP_OFFSET] + mov rsp, [rsp + GP_OFFSET + 8] ;; rsp pointer +%endif +%endmacro + align 32 ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -297,7 +366,7 @@ align 32 ; arg 2 : rdx : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_mult_sse,function,internal) sha1_mult_sse: - + FUNC_SAVE sub rsp, FRAMESZ ;; Initialize digests @@ -429,7 +498,7 @@ lloop: %endif add rsp, FRAMESZ - + FUNC_RESTORE ret mksection stack-noexec diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 257a99cd..c0f66ebf 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -84,6 +84,7 @@ const struct { OOO_INFO(aes256_cmac_ooo, MB_MGR_CMAC_OOO), OOO_INFO(snow3g_uea2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), + OOO_INFO(sha_1_ooo, MB_MGR_HMAC_SHA_1_OOO) }; /** -- GitLab From ad638b5d336735558cf4c4020ff6c055b3938c70 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 11 May 2022 15:29:41 +0000 Subject: [PATCH 098/369] sse: [SHA1] MB handling of extra blocks --- lib/Makefile | 1 + lib/include/mb_mgr_code.h | 285 ++------------------------------------ lib/include/sha_mb_mgr.h | 216 +++++++++++++++++++++++++++++ lib/sse/mb_mgr_sse.c | 10 +- lib/sse/sha1_x4_sse.asm | 41 +++--- lib/sse/sha_mb_sse.c | 52 +++++++ lib/win_x64.mak | 1 + 7 files changed, 305 insertions(+), 301 deletions(-) create mode 100644 lib/include/sha_mb_mgr.h create mode 100644 lib/sse/sha_mb_sse.c diff --git a/lib/Makefile b/lib/Makefile index ad98bcc4..195245df 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -256,6 +256,7 @@ c_lib_objs := \ aes_xcbc_expand_key.o \ md5_one_block.o \ sha_sse.o \ + sha_mb_sse.o \ sha_avx.o \ des_key.o \ des_basic.o \ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 7c67bc84..28d6f52d 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1,4 +1,3 @@ - /******************************************************************************* Copyright (c) 2012-2022, Intel Corporation @@ -47,7 +46,6 @@ #include "intel-ipsec-mb.h" #include "error.h" #include "include/snow3g_submit.h" -#include "constants.h" #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -55,10 +53,6 @@ #define BSWAP64 _byteswap_uint64 #endif -extern void sha1_mult_sse(SHA1_ARGS *args, uint32_t size_in_blocks); -extern void sha1_block_sse(const void *, void *); -extern void sha1_block_avx(const void *, void *); - #define CRC(func, state, job) *((uint32_t *)job->auth_tag_output) = \ func(state, job->src + job->hash_start_src_offset_in_bytes, \ job->msg_len_to_hash_in_bytes) @@ -83,263 +77,6 @@ void ADV_JOBS(int *ptr) *ptr = 0; } -/* ========================================================================= */ -/* SHA1 */ -/* ========================================================================= */ -__forceinline -void -sha_generic_one_block(const void *inp, void *digest, - const int is_avx, const int sha_type) -{ - if (sha_type == 1) { - if (is_avx) - sha1_block_avx(inp, digest); - else - sha1_block_sse(inp, digest); - } -} - -__forceinline -uint32_t mbswap4(const uint32_t val) -{ - return ((val >> 24) | - ((val & 0xff0000) >> 8) | - ((val & 0xff00) << 8) | - (val << 24)); -} - -__forceinline -uint64_t mbswap8(const uint64_t val) -{ - return (((uint64_t) mbswap4((uint32_t) val)) << 32) | - (((uint64_t) mbswap4((uint32_t) (val >> 32)))); -} - -__forceinline -void copy_bswap4_array(void *dst, const void *src, const size_t num) -{ - uint32_t *outp = (uint32_t *) dst; - const uint32_t *inp = (const uint32_t *) src; - size_t i; - - for (i = 0; i < num; i++) - outp[i] = mbswap4(inp[i]); -} - -__forceinline -void copy_bswap8_array(void *dst, const void *src, const size_t num) -{ - uint64_t *outp = (uint64_t *) dst; - const uint64_t *inp = (const uint64_t *) src; - size_t i; - - for (i = 0; i < num; i++) - outp[i] = mbswap8(inp[i]); -} - -__forceinline -void sha1_init_digest(void *d, unsigned lane) -{ - uint32_t *digest = (uint32_t *)d; - - digest[lane + 0*16] = H0; - digest[lane + 1*16] = H1; - digest[lane + 2*16] = H2; - digest[lane + 3*16] = H3; - digest[lane + 4*16] = H4; -} - -__forceinline -void -sha_generic_init(void *digest, const int sha_type, unsigned lane) -{ - if (sha_type == 1) - sha1_init_digest(digest, lane); -} - -__forceinline -void sha1_sc_digest(void *p, void *d, unsigned lane) -{ - uint32_t *p_digest = (uint32_t *)p; - uint32_t *d_digest = (uint32_t *)d; - - p_digest[0] = d_digest[lane + 0*16]; - p_digest[1] = d_digest[lane + 1*16]; - p_digest[2] = d_digest[lane + 2*16]; - p_digest[3] = d_digest[lane + 3*16]; - p_digest[4] = d_digest[lane + 4*16]; -} - -__forceinline -void -sha_generic_sc(void *digest, void *d, unsigned lane, const int sha_type) -{ - if (sha_type == 1) - sha1_sc_digest(digest, d, lane); -} - -__forceinline -void sha_generic_write_digest(void *dst, const void *src, const int sha_type) -{ - if (sha_type == 1) - copy_bswap4_array(dst, src, NUM_SHA_DIGEST_WORDS); -} - -__forceinline -void store8_be(void *outp, const uint64_t val) -{ - *((uint64_t *)outp) = mbswap8(val); -} - -__forceinline -void var_memcpy(void *dst, const void *src, const uint64_t len) -{ - uint64_t i; - const uint8_t *src8 = (const uint8_t *)src; - uint8_t *dst8 = (uint8_t *)dst; - - for (i = 0; i < len; i++) - dst8[i] = src8[i]; -} - -__forceinline -IMB_JOB * -submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, - const unsigned max_jobs, const int is_submit, - const int is_avx, const int sha_type, - const uint64_t blk_size, const uint64_t pad_size) -{ - unsigned lane, min_len, min_idx; - IMB_JOB *ret_job = NULL; - unsigned i; - uint8_t cb[IMB_SHA_512_BLOCK_SIZE]; - union { - uint32_t digest1[NUM_SHA_256_DIGEST_WORDS]; - uint64_t digest2[NUM_SHA_512_DIGEST_WORDS]; - } local_digest; - void *ld = (void *) &local_digest; - uint64_t r; - - if (is_submit) { - /* - * SUBMIT - * - get a free lane id - */ - - lane = state->unused_lanes & 15; - state->unused_lanes >>= 4; - state->args.data_ptr[lane] = - job->src + job->hash_start_src_offset_in_bytes; - - sha_generic_init(state->args.digest, sha_type, lane); - - /* copy job data in and set up initial blocks */ - state->ldata[lane].job_in_lane = job; - state->lens[lane] = (uint16_t)job->msg_len_to_hash_in_bytes; - - /* enough jobs to start processing? */ - if (state->unused_lanes != 0xf) - return NULL; - } else { - /* - * FLUSH - * - find 1st non null job - */ - for (lane = 0; lane < max_jobs; lane++) - if (state->ldata[lane].job_in_lane != NULL) - break; - if (lane >= max_jobs) - return NULL; /* no not null job */ - } - - if (is_submit) { - /* - * SUBMIT - * - find min common length to process - */ - min_idx = 0; - min_len = state->lens[0]; - - for (i = 1; i < max_jobs; i++) { - if (min_len > state->lens[i]) { - min_idx = i; - min_len = state->lens[i]; - } - } - } else { - /* - * FLUSH - * - copy good (not null) lane onto empty lanes - * - find min common length to process across not null lanes - */ - min_idx = lane; - min_len = state->lens[lane]; - - for (i = 0; i < max_jobs; i++) { - if (i == lane) - continue; - - if (state->ldata[i].job_in_lane != NULL) { - if (min_len > state->lens[i]) { - min_idx = i; - min_len = state->lens[i]; - } - } else { - state->args.data_ptr[i] = - state->args.data_ptr[lane]; - state->lens[i] = UINT16_MAX; - } - } - } - - /* subtract min len from all lanes */ - for (i = 0; i < max_jobs; i++) - state->lens[i] -= (uint16_t)((min_len/blk_size)*blk_size); - - r = min_len % blk_size; - - /* run the algorithmic code on full selected blocks */ - if(min_len >= blk_size) - sha1_mult_sse(&state->args, (uint32_t)(min_len/blk_size)); - - /* process partial block */ - memset(cb, 0, sizeof(cb)); - var_memcpy(cb, state->args.data_ptr[min_idx], r); - cb[r] = 0x80; - - sha_generic_sc(ld, state->args.digest, min_idx, sha_type); - - if (r >= (blk_size - pad_size)) { - /* length will be encoded in the next block */ - sha_generic_one_block(cb, ld, is_avx, sha_type); - memset(cb, 0, sizeof(cb)); - } - - ret_job = state->ldata[min_idx].job_in_lane; - - /* encode bit length */ - store8_be(&cb[blk_size - 8], ret_job->msg_len_to_hash_in_bytes * 8); - sha_generic_one_block(cb, ld, is_avx, sha_type); - - /* put back processed packet into unused lanes, set job as complete */ - state->unused_lanes = (state->unused_lanes << 4) | min_idx; - sha_generic_write_digest(ret_job->auth_tag_output, ld, sha_type); - ret_job->status |= IMB_STATUS_COMPLETED_AUTH; - state->ldata[min_idx].job_in_lane = NULL; - -#ifdef SAFE_DATA - clear_mem(cb, sizeof(cb)); - clear_mem(&local_digest, sizeof(local_digest)); - clear_scratch_gps(); - if (is_avx) - clear_scratch_xmms_avx(); - else - clear_scratch_xmms_sse(); -#endif - - return ret_job; -} - /* ========================================================================= */ /* Lower level "out of order" schedulers */ /* ========================================================================= */ @@ -1269,7 +1006,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#ifndef AVX2 +#ifdef SSE MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1329,16 +1066,14 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: - #ifdef AVX2 + #ifdef SSE + return SUBMIT_JOB_SHA1(sha_1_ooo, job); + #else IMB_SHA1(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; - #else - return submit_flush_job_sha_1(sha_1_ooo, job, 4, 1, 0, 1, - IMB_SHA1_BLOCK_SIZE, - SHA1_PAD_SIZE); #endif case IMB_AUTH_SHA_224: IMB_SHA224(state, @@ -1483,7 +1218,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#ifndef AVX2 +#ifdef SSE MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1515,11 +1250,9 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); - #ifndef AVX2 case IMB_AUTH_SHA_1: - return submit_flush_job_sha_1(sha_1_ooo, job, 4, 0, 0, 1, - IMB_SHA1_BLOCK_SIZE, - SHA1_PAD_SIZE); + #ifdef SSE + return FLUSH_JOB_SHA1(sha_1_ooo, job); #endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h new file mode 100644 index 00000000..4160c58d --- /dev/null +++ b/lib/include/sha_mb_mgr.h @@ -0,0 +1,216 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_generic.h" +#include "ipsec_ooo_mgr.h" +#include "constants.h" + +extern void call_sha1_mult_sse_from_c(SHA1_ARGS *args, + uint32_t size_in_blocks); + +__forceinline +void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, + const size_t offset, const unsigned lane) +{ + uint32_t *outp = (uint32_t *) dst; + const uint32_t *inp = (const uint32_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[lane + i*offset]); +} + +__forceinline +void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[lane + 0*16] = H0; + digest[lane + 1*16] = H1; + digest[lane + 2*16] = H2; + digest[lane + 3*16] = H3; + digest[lane + 4*16] = H4; +} + +__forceinline +void +sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) +{ + if (sha_type == 1) + sha1_mb_init_digest(digest, lane); +} + +__forceinline +void sha_mb_generic_write_digest(void *dst, const void *src, + const int sha_type, const size_t offset, + const unsigned lane) +{ + if (sha_type == 1) + copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, offset, + lane); +} + +__forceinline +void create_extra_blocks(MB_MGR_HMAC_SHA_1_OOO *state, + const uint64_t blk_size, const uint64_t r, + const unsigned min_idx) +{ + HMAC_SHA1_LANE_DATA *ld = &state->ldata[min_idx]; + const uint64_t xblk_size = blk_size*state->ldata[min_idx].extra_blocks; + + memset(ld->extra_block, 0, sizeof(ld->extra_block)); + + var_memcpy(ld->extra_block, state->args.data_ptr[min_idx], r); + ld->extra_block[r] = 0x80; + + store8_be(&ld->extra_block[xblk_size - 8], + ld->job_in_lane->msg_len_to_hash_in_bytes * 8); + + state->args.data_ptr[min_idx] = &ld->extra_block[0]; + + state->lens[min_idx] = (uint16_t)xblk_size; + + state->ldata[min_idx].extra_blocks = 0; +} + +__forceinline +IMB_JOB * +submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, + const unsigned max_jobs, const int is_submit, + const int sha_type, const uint64_t blk_size, + const uint64_t pad_size, + void (*fn)(SHA1_ARGS *, uint32_t)) +{ + unsigned lane, min_len, min_idx; + IMB_JOB *ret_job = NULL; + unsigned i; + + if (is_submit) { + /* + * SUBMIT + * - get a free lane id + */ + + lane = state->unused_lanes & 15; + state->unused_lanes >>= 4; + state->args.data_ptr[lane] = + job->src + job->hash_start_src_offset_in_bytes; + + sha_mb_generic_init(state->args.digest, sha_type, lane); + + /* copy job data in and set up initial blocks */ + state->ldata[lane].job_in_lane = job; + state->lens[lane] = (uint16_t)job->msg_len_to_hash_in_bytes; + state->ldata[lane].extra_blocks = 1; + + /* enough jobs to start processing? */ + if (state->unused_lanes != 0xf) + return NULL; + } else { + /* + * FLUSH + * - find 1st non null job + */ + for (lane = 0; lane < max_jobs; lane++) + if (state->ldata[lane].job_in_lane != NULL) + break; + if (lane >= max_jobs) + return NULL; /* no not null job */ + } + + do { + if (is_submit) { + /* + * SUBMIT + * - find min common length to process + */ + min_idx = 0; + min_len = state->lens[0]; + + for (i = 1; i < max_jobs; i++) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } + } else { + /* + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ + min_idx = lane; + min_len = state->lens[lane]; + + for (i = 0; i < max_jobs; i++) { + if (i == lane) + continue; + + if (state->ldata[i].job_in_lane != NULL) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } else { + state->args.data_ptr[i] = + state->args.data_ptr[lane]; + state->lens[i] = UINT16_MAX; + } + } + } + + /* subtract min len from all lanes */ + for (i = 0; i < max_jobs; i++) + state->lens[i] -= + (uint16_t)((min_len/blk_size)*blk_size); + + const uint64_t r = min_len % blk_size; + + if (r >= (blk_size - pad_size)) + state->ldata[min_idx].extra_blocks = 2; + + /* run the algorithmic code on full selected blocks */ + if(min_len >= blk_size) + (*fn)(&state->args, + (uint32_t)(min_len/blk_size)); + + /* create extra blocks */ + if (state->ldata[min_idx].extra_blocks != 0) + create_extra_blocks(state, blk_size, r, min_idx); + + } while(state->lens[min_idx] != 0); + + ret_job = state->ldata[min_idx].job_in_lane; + + /* put back processed packet into unused lanes, set job as complete */ + state->unused_lanes = (state->unused_lanes << 4) | min_idx; + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 16, min_idx); + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; + state->ldata[min_idx].job_in_lane = NULL; + + return ret_job; +} diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 1f29deba..8c2e6eff 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -29,6 +29,7 @@ #include #include +#define SSE #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse #include "intel-ipsec-mb.h" @@ -192,6 +193,11 @@ IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); +IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_sse(const void *key_exp, void *key1, void *key2); uint32_t hec_32_sse(const uint8_t *in); @@ -246,6 +252,8 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse +#define SUBMIT_JOB_SHA1 submit_job_sha1_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse @@ -1180,8 +1188,6 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->ldata[j].job_in_lane = NULL; } - - IMB_DLL_LOCAL void init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) { diff --git a/lib/sse/sha1_x4_sse.asm b/lib/sse/sha1_x4_sse.asm index 8ffe44e9..f8e400de 100644 --- a/lib/sse/sha1_x4_sse.asm +++ b/lib/sse/sha1_x4_sse.asm @@ -79,7 +79,7 @@ mksection .text ; r3 = {d3 c3 b3 a3} ; %define XMM_STORAGE 16*10 -%define GP_STORAGE 8*3 +%define GP_STORAGE 8 %define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE %define GP_OFFSET XMM_STORAGE @@ -229,8 +229,7 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; FRAMESZ must be an odd multiple of 8 -%define FRAMESZ 16*16 +%define FRAMESZ 16*16 + 8 %define MOVPS movdqu @@ -295,10 +294,10 @@ mksection .text %endm %macro FUNC_SAVE 0 +%ifndef LINUX mov r11, rsp sub rsp, VARIABLE_OFFSET and rsp, ~15 ; align rsp to 16 bytes - movdqa [rsp + 0*16], xmm6 movdqa [rsp + 1*16], xmm7 movdqa [rsp + 2*16], xmm8 @@ -309,14 +308,7 @@ mksection .text movdqa [rsp + 7*16], xmm13 movdqa [rsp + 8*16], xmm14 movdqa [rsp + 9*16], xmm15 -%endif -%ifidn __OUTPUT_FORMAT__, win64 - ; rdi and rdx need to be maintained for Windows - mov [rsp + GP_OFFSET], rdi - mov [rsp + GP_OFFSET + 8], r11 ;; rsp pointer -%else - mov [rsp + GP_OFFSET], rsi - mov [rsp + GP_OFFSET + 8], r11 ;; rsp pointer + mov [rsp + GP_OFFSET], r11 ;; rsp pointer %endif %endmacro @@ -332,8 +324,8 @@ mksection .text movdqa xmm13, [rsp + 7*16] movdqa xmm14, [rsp + 8*16] movdqa xmm15, [rsp + 9*16] -%endif +%ifdef SAFE_DATA pxor xmm5, xmm5 movdqa [rsp + 0*16], xmm5 movdqa [rsp + 1*16], xmm5 @@ -345,15 +337,9 @@ mksection .text movdqa [rsp + 7*16], xmm5 movdqa [rsp + 8*16], xmm5 movdqa [rsp + 9*16], xmm5 +%endif -%ifidn __OUTPUT_FORMAT__, win64 - ; rdi and rdx need to be maintained for Windows - mov rdi, [rsp + GP_OFFSET] - mov rsp, [rsp + GP_OFFSET + 8] ;; rsp pointer -%else - ; rsi needs to be maintained for Linux - mov rsi, [rsp + GP_OFFSET] - mov rsp, [rsp + GP_OFFSET + 8] ;; rsp pointer + mov rsp, [rsp + GP_OFFSET] ;; rsp pointer %endif %endmacro @@ -366,8 +352,10 @@ align 32 ; arg 2 : rdx : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_mult_sse,function,internal) sha1_mult_sse: - FUNC_SAVE + mov r11, rsp sub rsp, FRAMESZ + and rsp, ~15 ; align rsp to 16 bytes + mov [rsp + 16*16], r11 ;; Initialize digests movdqa A, [arg1 + 0*SHA1_DIGEST_ROW_SIZE] @@ -497,7 +485,14 @@ lloop: clear_all_xmms_sse_asm %endif - add rsp, FRAMESZ + mov rsp, [rsp + 16*16] + ret + +; void call_sha1_mult_sse_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha1_mult_sse_from_c,function,internal) +call_sha1_mult_sse_from_c: + FUNC_SAVE + call sha1_mult_sse FUNC_RESTORE ret diff --git a/lib/sse/sha_mb_sse.c b/lib/sse/sha_mb_sse.c new file mode 100644 index 00000000..e7fce1c3 --- /dev/null +++ b/lib/sse/sha_mb_sse.c @@ -0,0 +1,52 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_mb_mgr.h" + +IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); + +/* ========================================================================== */ +/* + * SHA1 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 4, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_sse_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 4, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_sse_from_c); +} diff --git a/lib/win_x64.mak b/lib/win_x64.mak index ca8cb180..6850b6dd 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -278,6 +278,7 @@ lib_objs1 = \ $(OBJ_DIR)\md5_one_block.obj \ $(OBJ_DIR)\sha_sse.obj \ $(OBJ_DIR)\sha_avx.obj \ + $(OBJ_DIR)\sha_mb_sse.obj \ $(OBJ_DIR)\des_key.obj \ $(OBJ_DIR)\des_basic.obj \ $(OBJ_DIR)\chacha20_sse.obj \ -- GitLab From b180931ca8a24e8e87162d03119c9eb2ef4e53f0 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 12 May 2022 16:28:54 +0000 Subject: [PATCH 099/369] lib: Only archive object files in static library build --- lib/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile b/lib/Makefile index 195245df..53c29600 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -717,7 +717,7 @@ else ln -f -s $(LIB).so.$(SO_VERSION) $(LIB_DIR)/$(LIB).so endif else - $(AR) -qcs $@ $^ + $(AR) -qcs $@ $(target_obj_files) endif ifeq ($(SAFE_PARAM), n) @echo "NOTE:" $(SAFE_PARAM_MSG1) $(SAFE_PARAM_MSG2) -- GitLab From a6a837293d8acad130415dac5ff922eee5fadc65 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 15:30:15 +0000 Subject: [PATCH 100/369] avx512: [ZUC-EIA3-256] Add functions to skip first 16 KS bytes For old buffers, 16 bytes of keystream are saved from previous iteration of ZUC-EIA3-256 (for 16-byte tags), so to avoid extra memcpy's, skip the first 16 bytes of KS and write the rest for these buffers. --- lib/avx512/zuc_x16_avx512.asm | 86 ++++++++++++++++++++--- lib/avx512/zuc_x16_vaes_avx512.asm | 2 + lib/include/zuc_internal.h | 105 +++++++++++++++++++++++++---- 3 files changed, 173 insertions(+), 20 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index e1909c96..0d1fef5e 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -45,6 +45,8 @@ %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_avx512 %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_avx512 +%define ZUC_KEYGEN64B_SKIP16_16 asm_ZucGenKeystream64B_16_skip16_avx512 +%define ZUC_KEYGEN_SKIP16_16 asm_ZucGenKeystream_16_skip16_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_avx512 %define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_avx512 @@ -377,6 +379,12 @@ dw 0x0000, 0xffff, 0x0000, 0x0000 dw 0xffff, 0x0000, 0x0000, 0x0000 dw 0x0000, 0x0000, 0x0000, 0x0000 +mov_16B_mask: +dw 0000000000000000b, 0000000000001111b, 0000000011110000b, 0000000011111111b +dw 0000111100000000b, 0000111100001111b, 0000111111110000b, 0000111111111111b +dw 1111000000000000b, 1111000000001111b, 1111000011110000b, 1111000011111111b +dw 1111111100000000b, 1111111100001111b, 1111111111110000b, 1111111111111111b + mov_8B_mask: dw 1100110011001100b, 1100110011001111b, 1100110011111100b, 1100110011111111b dw 1100111111001100b, 1100111111001111b, 1100111111111100b, 1100111111111111b @@ -838,7 +846,7 @@ align 64 %define %%TMP %22 ; [in] Temporary GP register %define %%KMASK1 %23 ; [clobbered] Temporary K mask %define %%KMASK2 %24 ; [clobbered] Temporary K mask -%define %%SKIP_ROUNDS %25 ; [constant] Number of rounds to skip +%define %%SKIP_ROUNDS %25 ; [constant] Number of rounds to skip (1, 2 or 4) %if (%0 == 18) vmovdqu64 [%%KS + %%KEY_OFF*4], %%DATA64B_L0 @@ -864,11 +872,17 @@ align 64 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_0_4_8_12] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] - ; Shifting left 8 bytes of KS for lanes which first 8 bytes are skipped + ; Shifting left 4/8/16 bytes of KS for lanes which first 4/8/16 bytes are skipped +%if %%SKIP_ROUNDS == 4 + vmovdqu8 %%DATA64B_L3{%%KMASK1}, %%DATA64B_L2 + vmovdqu8 %%DATA64B_L2{%%KMASK1}, %%DATA64B_L1 + vmovdqu8 %%DATA64B_L1{%%KMASK1}, %%DATA64B_L0 +%else vpalignr %%DATA64B_L3{%%KMASK1}, %%DATA64B_L3, %%DATA64B_L2, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L2{%%KMASK1}, %%DATA64B_L2, %%DATA64B_L1, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L1{%%KMASK1}, %%DATA64B_L1, %%DATA64B_L0, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L0{%%KMASK1}, %%DATA64B_L0, %%DATA64B_L3, (16 - %%SKIP_ROUNDS * 4) +%endif vmovdqu32 [%%KS + %%KEY_OFF*4]{%%KMASK2}, %%DATA64B_L0 vmovdqu32 [%%KS + %%KEY_OFF*4 + 64], %%DATA64B_L1 vmovdqu32 [%%KS + %%KEY_OFF*4 + 2*64], %%DATA64B_L2 @@ -877,10 +891,16 @@ align 64 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_1_5_9_13] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] +%if %%SKIP_ROUNDS == 4 + vmovdqu8 %%DATA64B_L7{%%KMASK1}, %%DATA64B_L6 + vmovdqu8 %%DATA64B_L6{%%KMASK1}, %%DATA64B_L5 + vmovdqu8 %%DATA64B_L5{%%KMASK1}, %%DATA64B_L4 +%else vpalignr %%DATA64B_L7{%%KMASK1}, %%DATA64B_L7, %%DATA64B_L6, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L6{%%KMASK1}, %%DATA64B_L6, %%DATA64B_L5, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L5{%%KMASK1}, %%DATA64B_L5, %%DATA64B_L4, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L4{%%KMASK1}, %%DATA64B_L4, %%DATA64B_L7, (16 - %%SKIP_ROUNDS * 4) +%endif vmovdqu32 [%%KS + %%KEY_OFF*4 + 512]{%%KMASK2}, %%DATA64B_L4 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512 + 64], %%DATA64B_L5 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512 + 64*2], %%DATA64B_L6 @@ -889,10 +909,16 @@ align 64 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_2_6_10_14] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] +%if %%SKIP_ROUNDS == 4 + vmovdqu8 %%DATA64B_L11{%%KMASK1}, %%DATA64B_L10 + vmovdqu8 %%DATA64B_L10{%%KMASK1}, %%DATA64B_L9 + vmovdqu8 %%DATA64B_L9{%%KMASK1}, %%DATA64B_L8 +%else vpalignr %%DATA64B_L11{%%KMASK1}, %%DATA64B_L11, %%DATA64B_L10, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L10{%%KMASK1}, %%DATA64B_L10, %%DATA64B_L9, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L9{%%KMASK1}, %%DATA64B_L9, %%DATA64B_L8, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L8{%%KMASK1}, %%DATA64B_L8, %%DATA64B_L11, (16 - %%SKIP_ROUNDS * 4) +%endif vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2]{%%KMASK2}, %%DATA64B_L8 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2 + 64], %%DATA64B_L9 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*2 + 64*2], %%DATA64B_L10 @@ -901,10 +927,16 @@ align 64 pext DWORD(%%TMP), DWORD(%%LANE_MASK), [rel extr_bits_3_7_11_15] kmovq %%KMASK1, [%%ALIGN_MASK + 8*%%TMP] kmovw %%KMASK2, [%%MOV_MASK + 2*%%TMP] +%if %%SKIP_ROUNDS == 4 + vmovdqu8 %%DATA64B_L15{%%KMASK1}, %%DATA64B_L14 + vmovdqu8 %%DATA64B_L14{%%KMASK1}, %%DATA64B_L13 + vmovdqu8 %%DATA64B_L13{%%KMASK1}, %%DATA64B_L12 +%else vpalignr %%DATA64B_L15{%%KMASK1}, %%DATA64B_L15, %%DATA64B_L14, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L14{%%KMASK1}, %%DATA64B_L14, %%DATA64B_L13, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L13{%%KMASK1}, %%DATA64B_L13, %%DATA64B_L12, (16 - %%SKIP_ROUNDS * 4) vpalignr %%DATA64B_L12{%%KMASK1}, %%DATA64B_L12, %%DATA64B_L15, (16 - %%SKIP_ROUNDS * 4) +%endif vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3]{%%KMASK2}, %%DATA64B_L12 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3 + 64], %%DATA64B_L13 vmovdqu32 [%%KS + %%KEY_OFF*4 + 512*3 + 64*2], %%DATA64B_L14 @@ -1363,7 +1395,7 @@ init_for_auth_tag_8B: %macro KEYGEN_16_AVX512 3-4 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds %define %%KEY_OFF %2 ; [in] Offset to start writing Keystream -%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip +%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip (1, 2 or 4) %define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate all keystream words %define pState arg1 @@ -1443,7 +1475,7 @@ init_for_auth_tag_8B: vmovdqa32 %%R2, [pState + OFS_R2] ; Store all 4 bytes of keystream in a single 64-byte buffer -%if (%%NUM_ROUNDS == 1) +%if (%%NUM_ROUNDS <= %%SKIP_ROUNDS) BITS_REORG16 pState, 1, %%FULL_LANE_KMASK, %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ %%ZTMP7, %%ZTMP8, %%ZTMP9, %%BLEND_KMASK, %%X0, %%X1, %%X2, %%KSTR1 NONLIN_FUN16 pState, %%FULL_LANE_KMASK, %%X0, %%X1, %%X2, %%R1, %%R2, \ @@ -1454,7 +1486,7 @@ init_for_auth_tag_8B: %%ZTMP6, %%MASK31, %%ZTMP7, work vmovdqa32 [pState + OFS_R1]{%%FULL_LANE_KMASK}, %%R1 vmovdqa32 [pState + OFS_R2]{%%FULL_LANE_KMASK}, %%R2 -%else ;; %%NUM_ROUNDS != 1 +%else ;; %%NUM_ROUNDS > %%SKIP_ROUNDS ; Generate N*4B of keystream in N rounds ; Generate first bytes of KS for all lanes %assign %%N 1 @@ -1504,8 +1536,10 @@ init_for_auth_tag_8B: lea %%TMP1, [rel alignr_mask] %if %%SKIP_ROUNDS == 1 lea %%TMP2, [rel mov_4B_mask] -%else ; %%SKIP_ROUNDS == 2 +%elif %%SKIP_ROUNDS == 2 lea %%TMP2, [rel mov_8B_mask] +%else ; %%SKIP_ROUNDS == 4 + lea %%TMP2, [rel mov_16B_mask] %endif STORE_KSTR16 pKS, %%ZTMP7, %%ZTMP5, %%KSTR13, %%KSTR1, %%ZTMP8, %%ZTMP6, %%KSTR14, %%KSTR2, \ %%ZTMP3, %%ZTMP1, %%KSTR15, %%KSTR3, %%ZTMP4, %%ZTMP2, %%KSTR16, %%KSTR4, %%KEY_OFF, \ @@ -1519,7 +1553,7 @@ init_for_auth_tag_8B: %if (%0 == 4) REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK %if (%%NUM_ROUNDS >= %%SKIP_ROUNDS) - REORDER_LFSR pState, (%%NUM_ROUNDS - %%SKIP_ROUNDS), %%SKIP_LANE_KMASK ; 2 less rounds for "old" buffers + REORDER_LFSR pState, (%%NUM_ROUNDS - %%SKIP_ROUNDS), %%SKIP_LANE_KMASK ; 1/2/4 less rounds for "old" buffers %endif %else REORDER_LFSR pState, %%NUM_ROUNDS, %%FULL_LANE_KMASK @@ -2042,6 +2076,23 @@ Eia3_N64B_tag_8B: ret +; +;; void asm_ZucGenKeystream64B_16_skip16_avx512(state16_t *pSta, u32* pKeyStr[16], +;; const u32 key_off, +;; const u16 lane_mask) +;; +MKGLOBAL(ZUC_KEYGEN64B_SKIP16_16,function,internal) +ZUC_KEYGEN64B_SKIP16_16: + endbranch64 + + FUNC_SAVE + + KEYGEN_16_AVX512 16, arg3, 4, arg4 + + FUNC_RESTORE + + ret + ; ;; void asm_ZucGenKeystream64B_16_skip8_avx512(state16_t *pSta, u32* pKeyStr[16], ;; const u32 key_off, @@ -2094,7 +2145,7 @@ ZUC_KEYGEN8B_16: %macro KEYGEN_VAR_16_AVX512 3-4 %define %%NUM_ROUNDS %1 ; [in] Number of 4-byte rounds (GP dowrd register) %define %%KEY_OFF %2 ; [in] Offset to start writing Keystream -%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip +%define %%SKIP_ROUNDS %3 ; [constant] Number of rounds to skip (1, 2 or 4) %define %%LANE_MASK %4 ; [in] Lane mask with lanes to generate full keystream (rest 1-2 words less) cmp %%NUM_ROUNDS, 16 @@ -2171,6 +2222,25 @@ ZUC_KEYGEN_16: ret ;; +;; void asm_ZucGenKeystream_16_skip16_avx512(state16_t *pSta, u32* pKeyStr[16], +;; const u32 key_off, +;; const u16 lane_mask, +;; u32 numRounds) +;; +MKGLOBAL(ZUC_KEYGEN_SKIP16_16,function,internal) +ZUC_KEYGEN_SKIP16_16: + endbranch64 + + mov r10, arg5 + + FUNC_SAVE + + KEYGEN_VAR_16_AVX512 r10d, arg3, 4, arg4 + + FUNC_RESTORE + + ret +;; ;; void asm_ZucGenKeystream_16_skip8_avx512(state16_t *pSta, u32* pKeyStr[16], ;; const u32 key_off, ;; const u16 lane_mask, diff --git a/lib/avx512/zuc_x16_vaes_avx512.asm b/lib/avx512/zuc_x16_vaes_avx512.asm index 65f40001..551fe808 100644 --- a/lib/avx512/zuc_x16_vaes_avx512.asm +++ b/lib/avx512/zuc_x16_vaes_avx512.asm @@ -36,8 +36,10 @@ %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_gfni_avx512 %define ZUC_KEYGEN8B_16 asm_ZucGenKeystream8B_16_gfni_avx512 %define ZUC_KEYGEN_16 asm_ZucGenKeystream_16_gfni_avx512 +%define ZUC_KEYGEN64B_SKIP16_16 asm_ZucGenKeystream64B_16_skip16_gfni_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_gfni_avx512 %define ZUC_KEYGEN64B_SKIP4_16 asm_ZucGenKeystream64B_16_skip4_gfni_avx512 +%define ZUC_KEYGEN_SKIP16_16 asm_ZucGenKeystream_16_skip16_gfni_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_gfni_avx512 %define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_gfni_avx512 %define ZUC_ROUND64B_16 asm_Eia3Round64B_16_VPCLMUL diff --git a/lib/include/zuc_internal.h b/lib/include/zuc_internal.h index 39e3b0f3..dee5bd2e 100755 --- a/lib/include/zuc_internal.h +++ b/lib/include/zuc_internal.h @@ -663,6 +663,44 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_avx512(ZucState16_t *pState, IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_gfni_avx512(ZucState16_t *pState, uint32_t *pKeyStr, const unsigned key_off); +/** + ****************************************************************************** + * + * @description + * Definition of the external function that implements the working + * stage of the ZUC algorithm. The function will generate 64 bytes of + * keystream for 16 packets in parallel, except for selected lanes, + * which will have 48 bytes of keystream generated instead. + * + * @param[in] pState Pointer to a ZUC state structure of type + * @ref ZucState16_t + * + * @param[in,out] pKeyStr Array of pointers to 16 input buffers + * that will contain the generated keystream + * for these 16 packets. + * + * @param[in] key_off Starting offset for writing KS. + * + * @param[in] lane_mask Mask containing lanes which will have 64 + * bytes of KS generated (16 bytes less for + * the rest) + * + * @pre + * A successful call to @ref asm_ZucInitialization_16_avx512 to initialize + * the ZUC state. + * + *****************************************************************************/ +IMB_DLL_LOCAL void +asm_ZucGenKeystream64B_16_skip16_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); + +IMB_DLL_LOCAL void +asm_ZucGenKeystream64B_16_skip16_gfni_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); /** ****************************************************************************** * @@ -690,10 +728,11 @@ IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_gfni_avx512(ZucState16_t *pState, * the ZUC state. * *****************************************************************************/ -IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_skip8_avx512(ZucState16_t *pState, - uint32_t *pKeyStr, - const unsigned key_off, - const uint16_t lane_mask); +IMB_DLL_LOCAL void +asm_ZucGenKeystream64B_16_skip8_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_skip8_gfni_avx512(ZucState16_t *pState, @@ -727,10 +766,11 @@ asm_ZucGenKeystream64B_16_skip8_gfni_avx512(ZucState16_t *pState, * the ZUC state. * *****************************************************************************/ -IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_skip4_avx512(ZucState16_t *pState, - uint32_t *pKeyStr, - const unsigned key_off, - const uint16_t lane_mask); +IMB_DLL_LOCAL void +asm_ZucGenKeystream64B_16_skip4_avx512(ZucState16_t *pState, + uint32_t *pKeyStr, + const unsigned key_off, + const uint16_t lane_mask); IMB_DLL_LOCAL void asm_ZucGenKeystream64B_16_skip4_gfni_avx512(ZucState16_t *pState, @@ -935,6 +975,47 @@ asm_ZucGenKeystream_16_gfni_avx512(ZucState16_t *pState, uint32_t *pKstr, const unsigned key_off, const uint32_t numRounds); +/** + ****************************************************************************** + * + * @description + * Definition of the external function that implements the working + * stage of the ZUC algorithm. The function will generate N*4 bytes of + * keystream for sixteen packets in parallel. + * + * @param[in] pState Pointer to a ZUC state structure of type + * @ref ZucState16_t + * + * @param[in,out] pKeyStr Array of pointers to 16 input buffers + * that will contain the generated keystream + * for these 16 packets. + * + * @param[in] key_off Starting offset for writing KS. + * + * @param[in] lane_mask Mask containing lanes which will have N*4 + * bytes of KS generated (16 bytes less for + * the rest) + * + * @param[in] numRounds Number of 4-byte rounds (1 to 16 rounds) + * + * @pre + * A successful call to @ref asm_ZucInitialization_16_avx512 to initialize + * ZUC state. + * + *****************************************************************************/ +IMB_DLL_LOCAL void +asm_ZucGenKeystream_16_skip16_avx512(ZucState16_t *pState, + uint32_t *pKstr, + const unsigned key_off, + const uint16_t lane_mask, + const uint32_t numRounds); + +IMB_DLL_LOCAL void +asm_ZucGenKeystream_16_skip16_gfni_avx512(ZucState16_t *pState, + uint32_t *pKstr, + const unsigned key_off, + const uint16_t lane_mask, + const uint32_t numRounds); /** ****************************************************************************** @@ -960,8 +1041,8 @@ asm_ZucGenKeystream_16_gfni_avx512(ZucState16_t *pState, * @param[in] numRounds Number of 4-byte rounds (1 to 16 rounds) * * @pre - * A successful call to @ref asm_ZucInitialization to initialize the ZUC - * state. + * A successful call to @ref asm_ZucInitialization_16_avx512 to initialize + * ZUC state. * *****************************************************************************/ IMB_DLL_LOCAL void @@ -1002,8 +1083,8 @@ asm_ZucGenKeystream_16_skip8_gfni_avx512(ZucState16_t *pState, * @param[in] numRounds Number of 4-byte rounds (1 to 16 rounds) * * @pre - * A successful call to @ref asm_ZucInitialization to initialize the ZUC - * state. + * A successful call to @ref asm_ZucInitialization_16_avx512 to initialize + * ZUC state. * *****************************************************************************/ IMB_DLL_LOCAL void -- GitLab From e3e558896df43db4205ed64a1f33e209ace653c0 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:38:50 +0000 Subject: [PATCH 101/369] avx512: [ZUC-EIA3-256] Initialize 16-byte digests --- lib/avx512/zuc_x16_avx512.asm | 97 ++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 0d1fef5e..2764462b 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -415,6 +415,10 @@ align 64 shuf_mask_0_0_0_dw1: times 4 db 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +expand_mask: +db 0x00, 0x03, 0x0c, 0x0f, 0x30, 0x33, 0x3c, 0x3f +db 0xc0, 0xc3, 0xcc, 0xcf, 0xf0, 0xf3, 0xfc, 0xff + ;; Calculate address for next bytes of keystream (KS) ;; Memory for KS is laid out in the following way: ;; - There are 128 bytes of KS for each buffer spread in chunks of 16 bytes, @@ -664,6 +668,60 @@ align 64 vpunpckhqdq %%IN01, %%K0, %%T0 vpunpcklqdq %%IN02, %%K1, %%T1 vpunpckhqdq %%IN03, %%K1, %%T1 +%endmacro + +; +; Performs a 4x16 32-bit transpose +; +; Input (each item is a 32-bit word): +; A0 A1 .. A15 +; B0 B1 .. B15 +; C0 C1 .. C15 +; D0 D1 .. D15 +; +; Output (each item is a 32-bit word): +; A0 B0 C0 D0 A1 B1 .. C3 D3 +; A4 B4 C4 D4 A5 B5 .. C7 D7 +; A8 B8 C8 D8 A9 B9 .. C11 D11 +; A12 B12 C12 D12 A13 B13 .. C15 D15 +; +%macro TRANSPOSE4_U32 16 +%define %%IN00 %1 ; [in/out] Input row 0 / Output column 0 +%define %%IN01 %2 ; [in/out] Input row 1 / Output column 1 +%define %%IN02 %3 ; [in/out] Input row 2 / Output column 2 +%define %%IN03 %4 ; [in/out] Input row 3 / Output column 3 +%define %%T0 %5 ; [clobbered] Temporary ZMM register +%define %%T1 %6 ; [clobbered] Temporary ZMM register +%define %%T2 %7 ; [clobbered] Temporary ZMM register +%define %%T3 %8 ; [clobbered] Temporary ZMM register +%define %%K0 %9 ; [clobbered] Temporary ZMM register +%define %%K1 %10 ; [clobbered] Temporary ZMM register +%define %%K2 %11 ; [clobbered] Temporary ZMM register +%define %%K3 %12 ; [clobbered] Temporary ZMM register +%define %%H0 %13 ; [clobbered] Temporary ZMM register +%define %%H1 %14 ; [clobbered] Temporary ZMM register +%define %%H2 %15 ; [clobbered] Temporary ZMM register +%define %%H3 %16 ; [clobbered] Temporary ZMM register + + vpunpckldq %%K0, %%IN00, %%IN01 + vpunpckhdq %%K1, %%IN00, %%IN01 + vpunpckldq %%T0, %%IN02, %%IN03 + vpunpckhdq %%T1, %%IN02, %%IN03 + + vpunpcklqdq %%K2, %%K0, %%T0 + vpunpckhqdq %%T2, %%K0, %%T0 + vpunpcklqdq %%K3, %%K1, %%T1 + vpunpckhqdq %%T3, %%K1, %%T1 + + vshufi64x2 %%H0, %%K2, %%T2, 0x44 + vshufi64x2 %%H1, %%K2, %%T2, 0xee + vshufi64x2 %%H2, %%K3, %%T3, 0x44 + vshufi64x2 %%H3, %%K3, %%T3, 0xee + + vshufi64x2 %%IN00, %%H0, %%H2, 0x88 + vshufi64x2 %%IN01, %%H0, %%H2, 0xdd + vshufi64x2 %%IN02, %%H1, %%H3, 0x88 + vshufi64x2 %%IN03, %%H1, %%H3, 0xdd %endmacro @@ -1188,6 +1246,8 @@ align 64 %define %%TMP_KMASK1 k3 %define %%TMP_KMASK2 k4 +%define %%TMP_KMASK3 k5 +%define %%TMP_KMASK4 k6 kmovw %%INIT_LANE_KMASK, DWORD(%%LANE_MASK) @@ -1307,7 +1367,33 @@ align 64 vmovdqa64 [%%TAGS]{%%TMP_KMASK1}, %%ZTMP1 vmovdqa64 [%%TAGS + 64]{%%TMP_KMASK2}, %%ZTMP2 REORDER_LFSR %%STATE, 2, %%INIT_LANE_KMASK -%elif %%TAG_SIZE == 16 ;; TODO +%elif %%TAG_SIZE == 16 + lea %%TMP, [rel expand_mask] + kmovd DWORD(%%TMP2), %%INIT_LANE_KMASK + and DWORD(%%TMP2), 0xf + kmovb %%TMP_KMASK1, [%%TMP + %%TMP2] ; First 4 lanes + kmovd DWORD(%%TMP2), %%INIT_LANE_KMASK + shr DWORD(%%TMP2), 4 + and DWORD(%%TMP2), 0xf + kmovb %%TMP_KMASK2, [%%TMP + %%TMP2] ; Second 4 lanes + + kmovd DWORD(%%TMP2), %%INIT_LANE_KMASK + shr DWORD(%%TMP2), 8 + and DWORD(%%TMP2), 0xf + kmovb %%TMP_KMASK3, [%%TMP + %%TMP2] ; Third 4 lanes + kmovd DWORD(%%TMP2), %%INIT_LANE_KMASK + shr DWORD(%%TMP2), 12 + kmovb %%TMP_KMASK4, [%%TMP + %%TMP2] ; Fourth 4 lanes + + TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, \ + %%ZTMP5, %%ZTMP6, %%ZTMP7, %%ZTMP8, \ + %%ZTMP9, %%ZTMP10, %%ZTMP11, %%ZTMP12 + vmovdqa64 [%%TAGS]{%%TMP_KMASK1}, %%KSTR1 + vmovdqa64 [%%TAGS + 64]{%%TMP_KMASK2}, %%KSTR2 + vmovdqa64 [%%TAGS + 64*2]{%%TMP_KMASK3}, %%KSTR3 + vmovdqa64 [%%TAGS + 64*3]{%%TMP_KMASK4}, %%KSTR4 + REORDER_LFSR %%STATE, 4, %%INIT_LANE_KMASK %endif %endmacro ; INIT_16_AVX512 @@ -1355,11 +1441,18 @@ ZUC256_INIT: or tag_sz, tag_sz jz init_for_cipher - ;; TODO: Check for 16B tags cmp tag_sz, 8 je init_for_auth_tag_8B jb init_for_auth_tag_4B +init_for_auth_tag_16B: + FUNC_SAVE + + INIT_16_AVX512 pKe, pIv, pState, lane_mask, r12, r13, 256, 16, tags + + FUNC_RESTORE + + ret init_for_cipher: FUNC_SAVE -- GitLab From c7fa0125b961f744d031bb4870d52bf6702858af Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 4 May 2022 08:48:34 +0000 Subject: [PATCH 102/369] avx512: [ZUC-EIA3-256] Extend DIGEST_DATA to produce 16-byte tags --- lib/avx512/zuc_x16_avx512.asm | 65 +++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 2764462b..51e72496 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -419,6 +419,14 @@ expand_mask: db 0x00, 0x03, 0x0c, 0x0f, 0x30, 0x33, 0x3c, 0x3f db 0xc0, 0xc3, 0xcc, 0xcf, 0xf0, 0xf3, 0xfc, 0xff +align 64 +shuf_mask_0_dw1_0_0: +times 4 db 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff + +align 64 +shuf_mask_dw1_0_0_0: +times 4 db 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x04, 0x05, 0x06, 0x07 + ;; Calculate address for next bytes of keystream (KS) ;; Memory for KS is laid out in the following way: ;; - There are 128 bytes of KS for each buffer spread in chunks of 16 bytes, @@ -1687,8 +1695,8 @@ init_for_auth_tag_8B: ;; %macro DIGEST_DATA 14 %define %%DATA %1 ; [in] Input data (16 bytes) per buffer -%define %%KS_L %2 ; [in] Lower 16 bytes of KS per buffer -%define %%KS_H %3 ; [in] Higher 16 bytes of KS per buffer +%define %%KS_L %2 ; [in/clobbered] Lower 16 bytes of KS per buffer +%define %%KS_H %3 ; [in/clobbered] Higher 16 bytes of KS per buffer %define %%KS_M1 %4 ; [clobbered] Temporary XMM/YMM/ZMM register %define %%KS_M2 %5 ; [cloberred] Temporary XMM/YMM/ZMM register %define %%IN_OUT %6 ; [in/out] Accumulated digest @@ -1699,7 +1707,7 @@ init_for_auth_tag_8B: %define %%TMP4 %11 ; [clobbered] Temporary XMM/YMM/ZMM register %define %%TMP5 %12 ; [clobbered] Temporary XMM/YMM/ZMM register %define %%TMP6 %13 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TAG_SIZE %14 ; [in] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %14 ; [in] Tag size (4, 8 or 16 bytes) ;; Set up KS ;; @@ -1755,6 +1763,36 @@ init_for_auth_tag_8B: ; XOR with bits 32-63 of previous digest vpxorq %%IN_OUT, %%TMP5 +%if %%TAG_SIZE == 16 + ; Prepare data and calculate bits 95-64 of tag + vpclmulqdq %%TMP3, %%TMP1, %%KS_M1, 0x00 + vpclmulqdq %%TMP4, %%TMP1, %%KS_M1, 0x11 + vpclmulqdq %%TMP5, %%TMP2, %%KS_M2, 0x00 + vpclmulqdq %%TMP6, %%TMP2, %%KS_M2, 0x11 + + ; XOR all the products and move bits 63-32 to bits 95-64 + vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 + vpxorq %%TMP5, %%TMP5, %%TMP6 + vpshufb %%TMP5, %%TMP5, [rel shuf_mask_0_dw1_0_0] + + ; XOR with previous bits 64-95 of previous digest + vpxorq %%IN_OUT, %%TMP5 + + ; Prepare data and calculate bits 127-96 of tag + vpclmulqdq %%TMP3, %%TMP1, %%KS_M1, 0x10 + vpclmulqdq %%TMP4, %%TMP1, %%KS_M2, 0x01 + vpclmulqdq %%TMP5, %%TMP2, %%KS_M2, 0x10 + vpclmulqdq %%TMP6, %%TMP2, %%KS_H, 0x01 + + ; XOR all the products and move bits 63-32 to bits 127-96 + vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 + vpxorq %%TMP5, %%TMP5, %%TMP6 + vpshufb %%TMP5, %%TMP5, [rel shuf_mask_dw1_0_0_0] + + ; XOR with lower 96 bits, to construct 128 bits of tag + vpxorq %%IN_OUT, %%TMP5 + +%endif ; %%TAG_SIZE == 16 %endif ; %%TAG_SIZE >= 8 %endmacro @@ -1891,12 +1929,12 @@ init_for_auth_tag_8B: %define %%XTMP9 xmm9 %define %%XTMP10 xmm0 %define %%KS_L %%XTMP9 -%define %%KS_H xmm21 +%define %%KS_H xmm15 %define %%XDIGEST_0 xmm13 %define %%XDIGEST_1 xmm14 %define %%XDIGEST_2 xmm19 %define %%XDIGEST_3 xmm20 -%define %%Z_TEMP_DIGEST zmm15 +%define %%Z_TEMP_DIGEST zmm21 %define %%REV_TABLE_L xmm16 %define %%REV_TABLE_H xmm17 %define %%REV_AND_TABLE xmm18 @@ -2718,8 +2756,6 @@ _no_final_rounds: mov %%DATA_ADDR2, [%%DATA + %%IDX*8 + 2*32] mov %%DATA_ADDR3, [%%DATA + %%IDX*8 + 3*32] - vmovdqu64 %%KS_TRANS0, [%%KS + %%IDX*64*2*4] - %assign %%I 0 %assign %%J 1 %rep 4 @@ -2728,6 +2764,7 @@ _no_final_rounds: vinserti32x4 APPEND(%%DATA_TRANS, %%I), [%%DATA_ADDR2 + 16*%%I], 2 vinserti32x4 APPEND(%%DATA_TRANS, %%I), [%%DATA_ADDR3 + 16*%%I], 3 + vmovdqu64 APPEND(%%KS_TRANS, %%I), [%%KS + %%IDX*64*2*4 + 64*%%I] vmovdqu64 APPEND(%%KS_TRANS, %%J), [%%KS + %%IDX*64*2*4 + 64*%%J] ;; Reverse bits of next 16 bytes from all 4 buffers @@ -2863,13 +2900,8 @@ _no_final_rounds: vpor %%XDATA, %%XDATA, %%XTMP4 ; %%DATA - bit reversed data bytes ; Read the next 2 blocks of 16 bytes of KS -%if %%K != 0 - vmovdqa %%KS_L, %%KS_H + vmovdqu %%KS_L, [%%KS + (16*%%J + %%I*512) + %%K*(16*4)] vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (%%K + 1)*(16*4)] -%else - vmovdqu %%KS_L, [%%KS + (16*%%J + %%I*512)] - vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (16*4)] -%endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE @@ -3118,13 +3150,8 @@ round_4B: ; %%XDATA - bit reversed data bytes ; Read the next 2 blocks of 16 bytes of %%KS -%if K != 0 - vmovdqa %%KS_L, %%KS_H + vmovdqu %%KS_L, [%%KS + (16*I + J*512) + %%OFFSET*4] vmovdqu %%KS_H, [%%KS + (16*I + J*512) + %%OFFSET*4 + (16*4)] -%else - vmovdqu %%KS_L, [%%KS + (16*I + J*512) + (0*4)] - vmovdqu %%KS_H, [%%KS + (16*I + J*512) + (16*4)] -%endif ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE -- GitLab From f0bd80a1d65458b231103f848006c3e748ff7f2d Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 16:04:39 +0000 Subject: [PATCH 103/369] avx512: [ZUC-EIA3-256] Update 16-byte tags Add code to permute data to update 16-byte tags, with new data being digested. --- lib/avx512/zuc_x16_avx512.asm | 68 +++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 51e72496..d640f5ee 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -1796,9 +1796,9 @@ init_for_auth_tag_8B: %endif ; %%TAG_SIZE >= 8 %endmacro -%macro UPDATE_TAGS 13-16 +%macro UPDATE_TAGS 13-14 %define %%T %1 ; [in] Pointer to digests -%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16 bytes) %define %%ORDER_TAGS %3 ; [constant] Order of tags (order_0_4_8_12 or order_0_1_2_3) %define %%TMP %4 ; [clobbered] Temporary GP register %define %%PERM_DIGEST_KMASK1 %5 ; [clobbered] Permutation mask for digests @@ -1811,8 +1811,6 @@ init_for_auth_tag_8B: %define %%ZTMP2 %12 ; [clobbered] Temporary ZMM register %define %%ZTMP3 %13 ; [clobbered] Temporary ZMM register %define %%ZTMP4 %14 ; [clobbered] Temporary ZMM register -%define %%ZTMP5 %15 ; [clobbered] Temporary ZMM register -%define %%ZTMP6 %16 ; [clobbered] Temporary ZMM register %if %%TAG_SIZE == 4 %ifidn %%ORDER_TAGS, order_0_4_8_12 @@ -1828,56 +1826,70 @@ init_for_auth_tag_8B: vmovdqa64 %%ZTMP2, [rel shuf_mask_4B_tags_0_1_2_3] vmovdqa64 %%ZTMP3, [rel shuf_mask_4B_tags_0_1_2_3 + 64] %endif - vmovdqu64 %%ZTMP1, [%%T] ; Input tags ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. vpermt2d %%DIGEST_0{%%PERM_DIGEST_KMASK1}{z}, %%ZTMP2, %%DIGEST_1 vpermt2d %%DIGEST_2{%%PERM_DIGEST_KMASK2}{z}, %%ZTMP3, %%DIGEST_3 - vpternlogq %%ZTMP1, %%DIGEST_0, %%DIGEST_2, 0x96 ; A XOR B XOR C - vmovdqu64 [%%T], %%ZTMP1 + vpternlogq %%DIGEST_0, %%DIGEST_2, [%%T], 0x96 ; A XOR B XOR C + vmovdqu64 [%%T], %%DIGEST_0 %elif %%TAG_SIZE == 8 - ; Input tags - vmovdqu64 %%ZTMP1, [%%T] - vmovdqu64 %%ZTMP2, [%%T + 64] %ifidn %%ORDER_TAGS, order_0_4_8_12 mov DWORD(%%TMP), 0x33 kmovd %%PERM_DIGEST_KMASK1, DWORD(%%TMP) kshiftld %%PERM_DIGEST_KMASK2, %%PERM_DIGEST_KMASK1, 2 - vmovdqa64 %%ZTMP3, [rel shuf_mask_8B_tags_0_1_4_5] - vmovdqa64 %%ZTMP4, [rel shuf_mask_8B_tags_2_3_6_7] - vmovdqa64 %%ZTMP5, [rel shuf_mask_8B_tags_8_9_12_13] - vmovdqa64 %%ZTMP6, [rel shuf_mask_8B_tags_10_11_14_15] + vmovdqa64 %%ZTMP1, [rel shuf_mask_8B_tags_0_1_4_5] + vmovdqa64 %%ZTMP2, [rel shuf_mask_8B_tags_2_3_6_7] + vmovdqa64 %%ZTMP3, [rel shuf_mask_8B_tags_8_9_12_13] + vmovdqa64 %%ZTMP4, [rel shuf_mask_8B_tags_10_11_14_15] ; Get result tags for 16 buffers in different positions in each lane ; and blend these tags into two ZMM registers ; Then, XOR the results with the previous tags and write out the result. + vpermi2q %%ZTMP1{%%PERM_DIGEST_KMASK1}{z}, %%DIGEST_0, %%DIGEST_1 + vpermi2q %%ZTMP2{%%PERM_DIGEST_KMASK2}{z}, %%DIGEST_2, %%DIGEST_3 vpermi2q %%ZTMP3{%%PERM_DIGEST_KMASK1}{z}, %%DIGEST_0, %%DIGEST_1 vpermi2q %%ZTMP4{%%PERM_DIGEST_KMASK2}{z}, %%DIGEST_2, %%DIGEST_3 - vpermi2q %%ZTMP5{%%PERM_DIGEST_KMASK1}{z}, %%DIGEST_0, %%DIGEST_1 - vpermi2q %%ZTMP6{%%PERM_DIGEST_KMASK2}{z}, %%DIGEST_2, %%DIGEST_3 - vpternlogq %%ZTMP1, %%ZTMP3, %%ZTMP4, 0x96 ; A XOR B XOR C - vpternlogq %%ZTMP2, %%ZTMP5, %%ZTMP6, 0x96 ; A XOR B XOR C + vpternlogq %%ZTMP1, %%ZTMP2, [%%T], 0x96 ; A XOR B XOR C + vpternlogq %%ZTMP3, %%ZTMP4, [%%T + 64], 0x96 ; A XOR B XOR C %else ; %%ORDER_TAGS == order_0_1_2_3 - ; Input tags vmovdqa64 %%ZTMP3, [rel shuf_mask_8B_tags] ; Get result tags for 16 buffers in different position in each lane ; and blend these tags into an ZMM register. ; Then, XOR the results with the previous tags and write out the result. vpermt2q %%DIGEST_0, %%ZTMP3, %%DIGEST_1 vpermt2q %%DIGEST_2, %%ZTMP3, %%DIGEST_3 - vpxorq %%ZTMP1, %%ZTMP1, %%DIGEST_0 - vpxorq %%ZTMP2, %%ZTMP2, %%DIGEST_2 + vpxorq %%ZTMP1, %%DIGEST_0, [%%T] + vpxorq %%ZTMP3, %%DIGEST_2, [%%T + 64] %endif vmovdqu64 [%%T], %%ZTMP1 - vmovdqu64 [%%T + 64], %%ZTMP2 + vmovdqu64 [%%T + 64], %%ZTMP3 %else ;; %%TAG_SIZE == 16 +%ifidn %%ORDER_TAGS, order_0_4_8_12 + ; Get result tags for 16 buffers in different positions in each lane + ; from 0,4,8,12 to 0,1,2,3 + ; Then, XOR the results with the previous tags and write out the result. + + TRANSPOSE4_U128_INPLACE %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 + %endif + + ; XOR with previous tags and store + vpxorq %%DIGEST_0, [%%T] + vpxorq %%DIGEST_1, [%%T + 64] + vpxorq %%DIGEST_2, [%%T + 64*2] + vpxorq %%DIGEST_3, [%%T + 64*3] + vmovdqa64 [%%T], %%DIGEST_0 + vmovdqa64 [%%T + 64], %%DIGEST_1 + vmovdqa64 [%%T + 64*2], %%DIGEST_2 + vmovdqa64 [%%T + 64*3], %%DIGEST_3 +%endif ; %%TAG_SIZE %endmacro ; ; Generate 64 bytes of keystream @@ -2114,7 +2126,7 @@ init_for_auth_tag_8B: UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 ; Update R1/R2 vmovdqa64 [%%STATE + OFS_R1], %%R1 @@ -2795,7 +2807,7 @@ _no_final_rounds: UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 ; Update data pointers vmovdqu64 %%ZTMP1, [%%DATA] @@ -2854,8 +2866,6 @@ _no_final_rounds: %define %%ZTMP2 zmm23 %define %%ZTMP3 zmm24 %define %%ZTMP4 zmm25 -%define %%ZTMP5 zmm26 -%define %%ZTMP6 zmm27 %define %%DIGEST_0 zmm28 %define %%DIGEST_1 zmm29 %define %%DIGEST_2 zmm30 @@ -2926,7 +2936,7 @@ _no_final_rounds: UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 ; Update data pointers vmovdqu64 %%ZTMP2, [%%DATA] @@ -3067,8 +3077,6 @@ round_4B: %define %%ZTMP2 zmm8 %define %%ZTMP3 zmm9 %define %%ZTMP4 zmm10 -%define %%ZTMP5 zmm11 -%define %%ZTMP6 zmm12 %define %%VALID_KMASK k1 ; Mask with valid lanes %define %%SHUF_DATA_KMASK k2 ; Mask to shuffle data @@ -3225,7 +3233,7 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): UPDATE_TAGS %%T, %%TAG_SIZE, order_0_1_2_3, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6 + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 ; These last steps should be done only for the buffers that ; have no more data to authenticate -- GitLab From ad5cfc57461a865e75cf7dbe7f93dc6babd56577 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 16:55:30 +0000 Subject: [PATCH 104/369] avx512: [ZUC-EIA3-256] Copy last 16 KS bytes to start Copy last 16 bytes of keystream if there is only one round to copy, when generating 16-byte tags with ZUC-EIA3-256. --- lib/avx512/zuc_x16_avx512.asm | 73 ++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index d640f5ee..7069c4b2 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -3398,6 +3398,8 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): %define %%KS_WORDS_TO_COPY 1 %elif %%TAG_SIZE == 8 %define %%KS_WORDS_TO_COPY 2 +%else ;; %%TAG_SIZE == 16 +%define %%KS_WORDS_TO_COPY 4 %endif %endif ;; %%KEY_SIZE @@ -3406,7 +3408,76 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): shl DWORD(%%MIN_LEN), (4+2) and DWORD(%%TMP1), 0xf add DWORD(%%MIN_LEN), DWORD(%%TMP1) -%if %%KS_WORDS_TO_COPY == 2 +%if %%KS_WORDS_TO_COPY == 4 + ; Memcpy last 16 bytes of KS into start + or DWORD(%%TMP1), DWORD(%%TMP1) + jz %%_copy_16bytes + + cmp DWORD(%%TMP1), 8 + je %%_copy_8bytes_8bytes + ja %%_copy_4bytes_12bytes + + ; Fall-through if 16 bytes to copy are 12 contiguous bytes and 4 separated bytes +%%_copy_12bytes_4bytes: +%assign %%i 0 +%rep 4 +%assign %%j 0 +%rep 4 + mov %%TMP1, [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], %%TMP1 + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + %%MIN_LEN + 8] + mov [%%KS + 512*%%i + 16*%%j + 8], DWORD(%%TMP1) + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + (48+12) + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j + 12], DWORD(%%TMP1) +%assign %%j (%%j + 1) +%endrep +%assign %%i (%%i + 1) +%endrep + jmp %%_ks_copied + +%%_copy_8bytes_8bytes: +%assign %%i 0 +%rep 4 +%assign %%j 0 +%rep 4 + mov %%TMP1, [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], %%TMP1 + mov %%TMP1, [%%KS + 512*%%i + 16*%%j + (48+8) + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j + 8], %%TMP1 +%assign %%j (%%j + 1) +%endrep +%assign %%i (%%i + 1) +%endrep + jmp %%_ks_copied +%%_copy_4bytes_12bytes: +%assign %%i 0 +%rep 4 +%assign %%j 0 +%rep 4 + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j], DWORD(%%TMP1) + mov %%TMP1, [%%KS + 512*%%i + 16*%%j + (48+4) + %%MIN_LEN] + mov [%%KS + 512*%%i + 16*%%j + 4], %%TMP1 + mov DWORD(%%TMP1), [%%KS + 512*%%i + 16*%%j + (48+4) + %%MIN_LEN + 8] + mov [%%KS + 512*%%i + 16*%%j + 12], DWORD(%%TMP1) +%assign %%j (%%j + 1) +%endrep +%assign %%i (%%i + 1) +%endrep + jmp %%_ks_copied +%%_copy_16bytes: +%assign %%i 0 +%rep 4 +%assign %%j 0 +%rep 4 + vmovdqa64 %%XTMP1, [%%KS + 512*%%i + 16*%%j + %%MIN_LEN] + vmovdqa64 [%%KS + 512*%%i + 16*%%j], %%XTMP1 +%assign %%j (%%j + 1) +%endrep +%assign %%i (%%i + 1) +%endrep + +%elif %%KS_WORDS_TO_COPY == 2 ; Memcpy last 8 bytes of KS into start cmp DWORD(%%TMP1), 12 je %%_copy_2dwords -- GitLab From 528e6d80cf322380f879c6a7061f1eb19a56ee2b Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 18:15:13 +0000 Subject: [PATCH 105/369] avx512: [ZUC-EIA3-256] Support 16-byte tags in Round64B function --- lib/avx512/zuc_x16_avx512.asm | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 7069c4b2..e530d507 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2704,7 +2704,7 @@ _no_final_rounds: %define %%TMP4 %8 ; [clobbered] Temporary GP register %define %%TMP5 %9 ; [clobbered] Temporary GP register %define %%TMP6 %10 ; [clobbered] Temporary GP register -%define %%TAG_SIZE %11 ; [constant] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data %define %%TMP_KMASK1 k2 @@ -2840,7 +2840,7 @@ _no_final_rounds: %define %%TMP4 %8 ; [clobbered] Temporary GP register %define %%TMP5 %9 ; [clobbered] Temporary GP register %define %%TMP6 %10 ; [clobbered] Temporary GP register -%define %%TAG_SIZE %11 ; [constant] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %11 ; [constant] Tag size (4, 8 or 16 bytes) %define %%SHUF_DATA_KMASK k1 ; Mask to shuffle data %define %%TMP_KMASK1 k2 @@ -2990,6 +2990,21 @@ ZUC_ROUND64B_16: je round_8B jb round_4B + ;; Fall-through for 16-byte tag +round_16B: + + FUNC_SAVE + +%if USE_GFNI_VAES_VPCLMUL == 1 + ROUND64B_16_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 16 +%else + ROUND64B_16_NO_GFNI T, KS, DATA, LEN, rbx, r10, r11, r12, r13, r14, 16 +%endif + + FUNC_RESTORE + + ret + round_8B: FUNC_SAVE -- GitLab From 81ca2737f5cfd89ca83e2e7d517184eb342644c1 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 9 May 2022 16:15:48 +0000 Subject: [PATCH 106/369] avx512: [ZUC-EIA3-256] Support 16-byte tags in EIA3_N64_16 function --- lib/avx512/zuc_x16_avx512.asm | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index e530d507..d514e9e7 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -2199,6 +2199,7 @@ ZUC_EIA3_N64B: cmp TAG_SIZE, 8 je Eia3_N64B_tag_8B + ja Eia3_N64B_tag_16B ; Fall-through for 4 bytes Eia3_N64B_tag_4B: @@ -2219,6 +2220,15 @@ Eia3_N64B_tag_8B: ret +Eia3_N64B_tag_16B: + FUNC_SAVE + + ZUC_EIA3_16_64B_AVX512 STATE, KS, T, DATA, LEN, NROUNDS, 16 + + FUNC_RESTORE + + ret + ; ;; void asm_ZucGenKeystream64B_16_skip16_avx512(state16_t *pSta, u32* pKeyStr[16], ;; const u32 key_off, -- GitLab From 570ef7e7682e6f2b01568a74606c1d88b2f215e8 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 6 May 2022 18:23:20 +0000 Subject: [PATCH 107/369] avx512: [ZUC-EIA3-256] Support 16-byte tags in Remainder function --- lib/avx512/zuc_x16_avx512.asm | 220 +++++++++++++++++++++++----------- 1 file changed, 151 insertions(+), 69 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index d514e9e7..04822742 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -3042,6 +3042,101 @@ round_4B: ret + +; +; Reads a qword of KS, rotates it by LEN % 32, and store the results as a single dword +; +%macro READ_AND_ROTATE_KS_DWORD 4 +%define %%KS_ADDR %1 ; [in] Base address of KS to read +%define %%LEN_BUF %2 ; [in] Remaining bytes of data +%define %%IN_OFFSET_OUT_KS %3 ; [in/out] Offset to read qwords of KS +%define %%TMP1 %4 ; [clobbered] Temporary GP register + + mov %%TMP1, %%IN_OFFSET_OUT_KS + and %%TMP1, 0xf + ; Read last two dwords of KS, which can be scattered or contiguous + ; (First dword can be at the end of a 16-byte chunk) + cmp %%TMP1, 12 + je %%_read_2dwords + mov %%IN_OFFSET_OUT_KS, [%%KS_ADDR + %%IN_OFFSET_OUT_KS] + jmp %%_ks_qword_read + + ;; The 8 bytes of %%KS are separated +%%_read_2dwords: + mov DWORD(%%TMP1), [%%KS_ADDR + %%IN_OFFSET_OUT_KS] + mov DWORD(%%IN_OFFSET_OUT_KS), [%%KS_ADDR + %%IN_OFFSET_OUT_KS + (4+48)] + shl %%IN_OFFSET_OUT_KS, 32 + or %%IN_OFFSET_OUT_KS, %%TMP1 +%%_ks_qword_read: + ; Rotate left by MIN_LEN % 32 + mov %%TMP1, rcx + mov rcx, %%LEN_BUF + and rcx, 0x1F + rol %%IN_OFFSET_OUT_KS, cl + mov rcx, %%TMP1 +%endmacro +; +; Reads two qwords of KS, overlapped by 4 bytes (e.g. KS[0-7] and KS[4-11]), +; rotates both qwords by LEN % 32, and store the results as a single qword, +; where lower dword is the result of rotation on first qword, and upper dword +; is the rotation on second dword. +; +%macro READ_AND_ROTATE_KS_QWORD 5 +%define %%KS_ADDR %1 ; [in] Base address of KS to read +%define %%LEN_BUF %2 ; [in] Remaining bytes of data +%define %%IN_OFFSET_OUT_KS %3 ; [in/out] Offset to read qwords of KS +%define %%TMP1 %4 ; [clobbered] Temporary GP register +%define %%TMP2 %5 ; [clobbered] Temporary GP register + + mov %%TMP2, %%IN_OFFSET_OUT_KS + and %%TMP2, 0xf + ; Read last three dwords of KS, which can be scattered or contiguous + ; (First dword can be at the end of a 16-byte chunk and the other + ; two dwords in the next chunk; first two dwords can be at the end of + ; a 16-byte chunk and the other dword in the next chunk; or all three + ; dwords can be in the same 16-byte chunk) + cmp %%TMP2, 8 + je %%_read_8B_4B + cmp %%TMP2, 12 + je %%_read_4B_8B + + ;; All 12 bytes of KS are contiguous +%%_read_12B: + mov %%TMP1, [%%KS_ADDR + %%IN_OFFSET_OUT_KS] + mov %%IN_OFFSET_OUT_KS, [%%KS_ADDR + %%IN_OFFSET_OUT_KS + 4] + jmp %%_ks_qwords_read + + ;; The first 8 bytes of KS are contiguous, the other 4 are separated +%%_read_8B_4B: + mov %%TMP1, [%%KS_ADDR + %%IN_OFFSET_OUT_KS] + ; Read last 4 bytes of first segment and first 4 bytes of second segment + mov DWORD(%%TMP2), [%%KS_ADDR + %%IN_OFFSET_OUT_KS + 4] + mov DWORD(%%IN_OFFSET_OUT_KS), [%%KS_ADDR + %%IN_OFFSET_OUT_KS + (8+48)] + shl %%IN_OFFSET_OUT_KS, 32 + or %%IN_OFFSET_OUT_KS, %%TMP2 + + jmp %%_ks_qwords_read + ;; The first 8 bytes of KS are separated, the other 8 are contiguous +%%_read_4B_8B: + mov DWORD(%%TMP1), [%%KS_ADDR + %%IN_OFFSET_OUT_KS] + mov DWORD(%%TMP2), [%%KS_ADDR + %%IN_OFFSET_OUT_KS + (4+48)] + shl %%TMP2, 32 + or %%TMP1, %%TMP2 + mov %%IN_OFFSET_OUT_KS, [%%KS_ADDR + %%IN_OFFSET_OUT_KS + (4+48)] +%%_ks_qwords_read: + ; Rotate left by LEN_BUF % 32 + mov %%TMP2, rcx + mov rcx, %%LEN_BUF + and rcx, 0x1F + rol %%TMP1, cl + rol %%IN_OFFSET_OUT_KS, cl + mov rcx, %%TMP2 + + shl %%IN_OFFSET_OUT_KS, 32 + mov DWORD(%%TMP1), DWORD(%%TMP1) ; Clear top 32 bits + or %%IN_OFFSET_OUT_KS, %%TMP1 +%endmacro + %macro REMAINDER_16 14 %define %%T %1 ; [in] Pointer to digests %define %%KS %2 ; [in] Pointer to keystream (128x16 bytes) @@ -3056,7 +3151,7 @@ round_4B: %define %%TMP6 %11 ; [clobbered] Temporary GP register %define %%TMP7 %12 ; [clobbered] Temporary GP register %define %%KEY_SIZE %13 ; [constant] Key size (128 or 256) -%define %%TAG_SIZE %14 ; [constant] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %14 ; [constant] Tag size (4, 8 or 16 bytes) %define %%DIGEST_0 zmm28 %define %%DIGEST_1 zmm29 @@ -3297,34 +3392,13 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): and %%TMP3, 0x3 shl %%TMP3, 2 add %%TMP2, %%TMP3 ;; Offset to last dwords of KS, from base address - mov %%TMP3, %%TMP2 - and %%TMP3, 0xf %if %%TAG_SIZE == 4 ; Read 4-byte digest mov DWORD(%%DIGEST), [%%T + 4*%%IDX] - ; Read last two dwords of KS, which can be scattered or contiguous - ; (First dword can be at the end of a 16-byte chunk) - cmp %%TMP3, 12 - je %%_read_2dwords - mov %%TMP1, [%%KS_ADDR + %%TMP2] - jmp %%_ks_qword_read - - ;; The 8 bytes of %%KS are separated -%%_read_2dwords: - mov DWORD(%%TMP1), [%%KS_ADDR + %%TMP2] - mov DWORD(%%TMP2), [%%KS_ADDR + %%TMP2 + (4+48)] - shl %%TMP2, 32 - or %%TMP1, %%TMP2 -%%_ks_qword_read: - ; Rotate left by MIN_LEN % 32 - mov %%TMP2, rcx - mov rcx, %%LEN_BUF - and rcx, 0x1F - rol %%TMP1, cl - mov rcx, %%TMP2 + READ_AND_ROTATE_KS_DWORD %%KS_ADDR, %%LEN_BUF, %%TMP2, %%TMP1 ; XOR with current digest - xor DWORD(%%DIGEST), DWORD(%%TMP1) + xor DWORD(%%DIGEST), DWORD(%%TMP2) %if %%KEY_SIZE == 128 ; Read keystr[L - 1] (last dword of keyStr) @@ -3349,51 +3423,7 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): ; Read 8-byte digest mov %%DIGEST, [%%T + 8*%%IDX] - ; Read last three dwords of KS, which can be scattered or contiguous - ; (First dword can be at the end of a 16-byte chunk and the other - ; two dwords in the next chunk; first two dwords can be at the end of - ; a 16-byte chunk and the other dword in the next chunk; or all three - ; dwords can be in the same 16-byte chunk) - cmp %%TMP3, 8 - je %%_read_8B_4B - cmp %%TMP3, 12 - je %%_read_4B_8B - - ;; All 12 bytes of KS are contiguous -%%_read_12B: - mov %%TMP1, [%%KS_ADDR + %%TMP2] - mov %%TMP2, [%%KS_ADDR + %%TMP2 + 4] - jmp %%_ks_qwords_read - - ;; The first 8 bytes of KS are contiguous, the other 4 are separated -%%_read_8B_4B: - mov %%TMP1, [%%KS_ADDR + %%TMP2] - ; Read last 4 bytes of first segment and first 4 bytes of second segment - mov DWORD(%%TMP3), [%%KS_ADDR + %%TMP2 + 4] - mov DWORD(%%TMP2), [%%KS_ADDR + %%TMP2 + (8+48)] - shl %%TMP2, 32 - or %%TMP2, %%TMP3 - - jmp %%_ks_qwords_read - ;; The first 8 bytes of KS are separated, the other 8 are contiguous -%%_read_4B_8B: - mov DWORD(%%TMP1), [%%KS_ADDR + %%TMP2] - mov DWORD(%%TMP3), [%%KS_ADDR + %%TMP2 + (4+48)] - shl %%TMP3, 32 - or %%TMP1, %%TMP3 - mov %%TMP2, [%%KS_ADDR + %%TMP2 + (4+48)] -%%_ks_qwords_read: - ; Rotate left by MIN_LEN % 32 - mov %%TMP3, rcx - mov rcx, %%LEN_BUF - and rcx, 0x1F - rol %%TMP1, cl - rol %%TMP2, cl - mov rcx, %%TMP3 - - shl %%TMP2, 32 - mov DWORD(%%TMP1), DWORD(%%TMP1) ; Clear top 32 bits - or %%TMP2, %%TMP1 + READ_AND_ROTATE_KS_QWORD %%KS_ADDR, %%LEN_BUF, %%TMP2, %%TMP1, %%TMP3 ; XOR with current digest xor %%DIGEST, %%TMP2 @@ -3403,6 +3433,49 @@ APPEND3(%%Eia3RoundsAVX_end,I,J): ror %%DIGEST, 32 mov [%%T + 8*%%IDX], %%DIGEST %else ; %%TAG_SIZE == 16 + ;; Update digest in two steps: + ;; - First, read the first 12 bytes of KS[MIN_LEN/32], + ;; rotate them and XOR the qword with first qword of digest + ;; - Last, skip 8 bytes of KS[MIN_LEN/32] and read another 12 bytes, + ;; rotate them and XOR the qword with second qword of digest + shl %%IDX, 4 + ; Read first 8 bytes of digest + mov %%DIGEST, [%%T + %%IDX] + + READ_AND_ROTATE_KS_QWORD %%KS_ADDR, %%LEN_BUF, %%TMP2, %%TMP1, %%TMP3 + + ; XOR with current first half of digest + xor %%DIGEST, %%TMP2 + + ; byte swap and write first half of digest out + bswap %%DIGEST + ror %%DIGEST, 32 + mov [%%T + %%IDX], %%DIGEST + + ; Read next 8 bytes after keyStr[MIN_LEN / 32] + mov %%TMP2, %%LEN_BUF + shr %%TMP2, 5 + add %%TMP2, 2 ; Add 2 dwords to offset + mov %%TMP3, %%TMP2 + shr %%TMP2, 2 + shl %%TMP2, (4+2) + and %%TMP3, 0x3 + shl %%TMP3, 2 + add %%TMP2, %%TMP3 ;; Offset to last dwords of KS, from base address + + ; Read second 8 bytes of digest + mov %%DIGEST, [%%T + %%IDX + 8] + + READ_AND_ROTATE_KS_QWORD %%KS_ADDR, %%LEN_BUF, %%TMP2, %%TMP1, %%TMP3 + + ; XOR with current second half of digest + xor %%DIGEST, %%TMP2 + + ; byte swap and write second half of digest out + bswap %%DIGEST + ror %%DIGEST, 32 + mov [%%T + %%IDX + 8], %%DIGEST + shr %%IDX, 4 %endif %%skip_comput: @@ -3615,6 +3688,15 @@ ZUC256_REMAINDER_16: je remainder_8B jb remainder_4B + ; Fall-through for 16-byte tag +remainder_16B: + FUNC_SAVE + + REMAINDER_16 T, KS, DATA, LEN, N_BITS, rax, rbx, r11, r12, r13, r14, r15, 256, 16 + + FUNC_RESTORE + + ret remainder_8B: FUNC_SAVE -- GitLab From fc12115287e7f17f1780ddfa4d26c6a91289cc14 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 9 May 2022 20:54:43 +0000 Subject: [PATCH 108/369] avx512: [ZUC-EIA3-256] Support 16-byte tags in submit/flush --- ReleaseNotes.txt | 2 +- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 95 ++++++++++++------- .../mb_mgr_zuc_submit_flush_gfni_avx512.asm | 2 + lib/include/ipsec_ooo_mgr.h | 2 +- lib/include/mb_mgr_datastruct.asm | 2 +- test/zuc_test.c | 4 +- 6 files changed, 66 insertions(+), 41 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 17c8ce6b..b9b2ae61 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -5,7 +5,7 @@ Unreleased ======================================================================== Library -- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE, AVX and AVX2 +- ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE, AVX, AVX2 and AVX512 - AES-ECB AVX512-VAES implementation added - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index 74602159..dedda8b4 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -48,6 +48,8 @@ %define ZUC_CIPHER asm_ZucCipher_16_avx512 %define ZUC_REMAINDER_16 asm_Eia3RemainderAVX512_16 %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16 +%define ZUC_KEYGEN_SKIP16_16 asm_ZucGenKeystream_16_skip16_avx512 +%define ZUC_KEYGEN64B_SKIP16_16 asm_ZucGenKeystream64B_16_skip16_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_avx512 %define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_avx512 @@ -75,6 +77,10 @@ extern asm_Eia3RemainderAVX512_16 extern asm_Eia3RemainderAVX512_16_VPCLMUL extern asm_Eia3_256_RemainderAVX512_16 extern asm_Eia3_256_RemainderAVX512_16_VPCLMUL +extern asm_ZucGenKeystream_16_skip16_avx512 +extern asm_ZucGenKeystream_16_skip16_gfni_avx512 +extern asm_ZucGenKeystream64B_16_skip16_avx512 +extern asm_ZucGenKeystream64B_16_skip16_gfni_avx512 extern asm_ZucGenKeystream_16_skip8_avx512 extern asm_ZucGenKeystream_16_skip8_gfni_avx512 extern asm_ZucGenKeystream64B_16_skip8_avx512 @@ -236,8 +242,9 @@ mksection .text ;; Find min length for lanes 0-7 vphminposuw xmm2, xmm0 + xor job_rax, job_rax cmp qword [state + _zuc_lanes_in_use], 16 - jne %%return_null_submit_eea3 + jne %%return_submit_eea3 ; Find min length for lanes 8-15 vpextrw DWORD(min_len), xmm2, 0 ; min value @@ -336,12 +343,6 @@ mksection .text mov rdi, [rsp + _gpr_save + 8*7] %endif mov rsp, [rsp + _rsp_save] ; original SP - - ret - -%%return_null_submit_eea3: - xor job_rax, job_rax - jmp %%return_submit_eea3 %endmacro %macro FLUSH_JOB_ZUC_EEA3 1 @@ -380,8 +381,9 @@ mksection .text mov [rsp + _rsp_save], rax ; original SP ; check for empty + xor job_rax, job_rax cmp qword [state + _zuc_lanes_in_use], 0 - jz %%return_null_flush_eea3 + jz %%return_flush_eea3 ; Find lanes with NULL jobs vpxorq zmm0, zmm0 @@ -563,12 +565,6 @@ mksection .text mov rdi, [rsp + _gpr_save + 8*7] %endif mov rsp, [rsp + _rsp_save] ; original SP - - ret - -%%return_null_flush_eea3: - xor job_rax, job_rax - jmp %%return_flush_eea3 %endmacro ; JOB* SUBMIT_JOB_ZUC128_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -578,6 +574,7 @@ MKGLOBAL(SUBMIT_JOB_ZUC128_EEA3,function,internal) SUBMIT_JOB_ZUC128_EEA3: endbranch64 SUBMIT_JOB_ZUC_EEA3 128 + ret ; JOB* SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) ; arg 1 : state @@ -586,6 +583,7 @@ MKGLOBAL(SUBMIT_JOB_ZUC256_EEA3,function,internal) SUBMIT_JOB_ZUC256_EEA3: endbranch64 SUBMIT_JOB_ZUC_EEA3 256 + ret ; JOB* FLUSH_JOB_ZUC128_EEA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state @@ -593,6 +591,7 @@ MKGLOBAL(FLUSH_JOB_ZUC128_EEA3,function,internal) FLUSH_JOB_ZUC128_EEA3: endbranch64 FLUSH_JOB_ZUC_EEA3 128 + ret ; JOB* FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state @@ -600,11 +599,12 @@ MKGLOBAL(FLUSH_JOB_ZUC256_EEA3,function,internal) FLUSH_JOB_ZUC256_EEA3: endbranch64 FLUSH_JOB_ZUC_EEA3 256 + ret %macro ZUC_EIA3_16_BUFFER 6 %define %%OOO %1 ; [in] Pointer to ZUC OOO manager %define %%KEY_SIZE %2 ; [constant] Key size (16 or 32) -%define %%TAG_SIZE %3 ; [constant] Tag size (4 or 8 bytes) +%define %%TAG_SIZE %3 ; [constant] Tag size (4, 8 or 16 bytes) %define %%L %4 ; [clobbered] Temporary GP register (dword) %define %%REMAIN_BITS %5 ; [clobbered] Temporary GP register (dword) %define %%TMP %6 ; [clobbered] Temporary GP register @@ -651,8 +651,10 @@ FLUSH_JOB_ZUC256_EEA3: %else ; %%KEY_SIZE == 256 %if %%TAG_SIZE == 4 call ZUC_KEYGEN_SKIP4_16 -%else ;; %%TAG_SIZE = 8 +%elif %%TAG_SIZE == 8 call ZUC_KEYGEN_SKIP8_16 +%else ;; %%TAG_SIZE == 16 + call ZUC_KEYGEN_SKIP16_16 %endif %endif ; %%KEY_SIZE @@ -673,8 +675,10 @@ FLUSH_JOB_ZUC256_EEA3: %else ; %%KEY_SIZE == 256 %if %%TAG_SIZE == 4 call ZUC_KEYGEN64B_SKIP4_16 -%else ;; %%TAG_SIZE = 8 +%elif %%TAG_SIZE == 8 call ZUC_KEYGEN64B_SKIP8_16 +%else ;; %%TAG_SIZE == 16 + call ZUC_KEYGEN64B_SKIP16_16 %endif %endif sub %%L, 16 @@ -779,7 +783,7 @@ FLUSH_JOB_ZUC256_EEA3: %macro SUBMIT_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) -%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16 bytes) ; idx needs to be in rbp %define len rbp @@ -869,8 +873,13 @@ FLUSH_JOB_ZUC256_EEA3: ; Reset temporary digest for the lane %if %%TAG_SIZE == 4 mov dword [state + _zuc_args_digest + lane*4], 0 -%else +%elif %%TAG_SIZE == 8 mov qword [state + _zuc_args_digest + lane*8], 0 +%else ; %%TAG_SIZE == 16 + vpxor xmm0, xmm0 + shl lane, 4 + vmovdqa [state + _zuc_args_digest + lane], xmm0 + shr lane, 4 %endif mov tmp, [job + _src] @@ -888,8 +897,9 @@ FLUSH_JOB_ZUC256_EEA3: vmovdqu16 ymm0{k1}, ymm1 vmovdqa64 [state + _zuc_lens], ymm0 + xor job_rax, job_rax cmp qword [state + _zuc_lanes_in_use], 16 - jne %%return_null_submit_eia3 + jne %%return_submit_eia3 ;; Find min length for lanes 0-7 vphminposuw xmm2, xmm0 @@ -954,6 +964,11 @@ FLUSH_JOB_ZUC256_EEA3: %elif %%TAG_SIZE == 8 mov r10, [state + _zuc_args_digest + idx*8] mov [r11], r10 +%else ; %%TAG_SIZE == 16 + shl idx, 4 + vmovdqa xmm0, [state + _zuc_args_digest + idx] + vmovdqu [r11], xmm0 + shr idx, 4 %endif shl unused_lanes, 4 or unused_lanes, idx @@ -987,17 +1002,11 @@ FLUSH_JOB_ZUC256_EEA3: mov rdi, [rsp + _gpr_save + 8*7] %endif mov rsp, [rsp + _rsp_save] ; original SP - - ret - -%%return_null_submit_eia3: - xor job_rax, job_rax - jmp %%return_submit_eia3 %endmacro %macro FLUSH_JOB_ZUC_EIA3 2 %define %%KEY_SIZE %1 ; [constant] Key size (128 or 256) -%define %%TAG_SIZE %2 ; [constant] Tag size (4 or 8) +%define %%TAG_SIZE %2 ; [constant] Tag size (4, 8 or 16 bytes) %define unused_lanes rbx %define tmp1 rbx @@ -1029,8 +1038,9 @@ FLUSH_JOB_ZUC256_EEA3: mov [rsp + _rsp_save], rax ; original SP ; check for empty + xor job_rax, job_rax cmp qword [state + _zuc_lanes_in_use], 0 - jz %%return_null_flush_eia3 + jz %%return_flush_eia3 ; find a lane with a null job vpxorq zmm0, zmm0 @@ -1153,6 +1163,11 @@ FLUSH_JOB_ZUC256_EEA3: %elif %%TAG_SIZE == 8 mov r10, [state + _zuc_args_digest + idx*8] mov [r11], r10 +%else ; %%TAG_SIZE == 16 + shl idx, 4 + vmovdqa xmm0, [state + _zuc_args_digest + idx] + vmovdqu [r11], xmm0 + shr idx, 4 %endif shl unused_lanes, 4 or unused_lanes, idx @@ -1186,12 +1201,6 @@ FLUSH_JOB_ZUC256_EEA3: mov rdi, [rsp + _gpr_save + 8*7] %endif mov rsp, [rsp + _rsp_save] ; original SP - - ret - -%%return_null_flush_eia3: - xor job_rax, job_rax - jmp %%return_flush_eia3 %endmacro ; JOB* SUBMIT_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -1201,12 +1210,13 @@ MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: endbranch64 SUBMIT_JOB_ZUC_EIA3 128, 4 + ret ; JOB* SUBMIT_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job, ; const uint64_t tag_sz) ; arg 1 : state ; arg 2 : job -; arg 3 : tag size (4 or 8 bytes) +; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: endbranch64 @@ -1214,11 +1224,17 @@ SUBMIT_JOB_ZUC256_EIA3: je submit_tag_8B jb submit_tag_4B +submit_tag_16B: + SUBMIT_JOB_ZUC_EIA3 256, 16 + ret + submit_tag_8B: SUBMIT_JOB_ZUC_EIA3 256, 8 + ret submit_tag_4B: SUBMIT_JOB_ZUC_EIA3 256, 4 + ret ; JOB* FLUSH_JOB_ZUC128_EIA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state @@ -1226,11 +1242,12 @@ MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: endbranch64 FLUSH_JOB_ZUC_EIA3 128, 4 + ret ; JOB* FLUSH_JOB_ZUC256_EIA3(MB_MGR_ZUC_OOO *state, ; const uint64_t tag_sz) ; arg 1 : state -; arg 2 : tag size (4 or 8 bytes) +; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: endbranch64 @@ -1238,10 +1255,16 @@ FLUSH_JOB_ZUC256_EIA3: je flush_tag_8B jb flush_tag_4B +flush_tag_16B: + FLUSH_JOB_ZUC_EIA3 256, 16 + ret + flush_tag_8B: FLUSH_JOB_ZUC_EIA3 256, 8 + ret flush_tag_4B: FLUSH_JOB_ZUC_EIA3 256, 4 + ret mksection stack-noexec diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm index cddf54e2..50a6ccad 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm @@ -37,6 +37,8 @@ %define ZUC256_INIT_16 asm_Zuc256Initialization_16_gfni_avx512 %define ZUC_REMAINDER_16 asm_Eia3RemainderAVX512_16_VPCLMUL %define ZUC256_REMAINDER_16 asm_Eia3_256_RemainderAVX512_16_VPCLMUL +%define ZUC_KEYGEN_SKIP16_16 asm_ZucGenKeystream_16_skip16_gfni_avx512 +%define ZUC_KEYGEN64B_SKIP16_16 asm_ZucGenKeystream64B_16_skip16_gfni_avx512 %define ZUC_KEYGEN_SKIP8_16 asm_ZucGenKeystream_16_skip8_gfni_avx512 %define ZUC_KEYGEN64B_SKIP8_16 asm_ZucGenKeystream64B_16_skip8_gfni_avx512 %define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_gfni_avx512 diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 4fb683f9..94baba03 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -155,7 +155,7 @@ typedef struct { DECLARE_ALIGNED(uint8_t *out[16], 64); const uint8_t *keys[16]; DECLARE_ALIGNED(uint8_t iv[16*32], 32); - DECLARE_ALIGNED(uint32_t digest[16*2], 64); + DECLARE_ALIGNED(uint32_t digest[16*4], 64); /* Memory for 128 bytes of KS for 16 buffers */ DECLARE_ALIGNED(uint32_t ks[16 * 2 * 16], 64); } ZUC_ARGS_x16; diff --git a/lib/include/mb_mgr_datastruct.asm b/lib/include/mb_mgr_datastruct.asm index eadf0668..16282046 100644 --- a/lib/include/mb_mgr_datastruct.asm +++ b/lib/include/mb_mgr_datastruct.asm @@ -239,7 +239,7 @@ FIELD _zucarg_in, 16*8, 64 ; array of 16 pointers to in text FIELD _zucarg_out, 16*8, 64 ; array of 16 pointers to out text FIELD _zucarg_keys, 16*8, 8 ; array of 16 pointers to keys FIELD _zucarg_IV, 16*32, 32 ; array of IVs (up to 25 bytes each) -FIELD _zucarg_digest, 16*8, 64 ; array of 16 digests +FIELD _zucarg_digest, 16*16, 64 ; array of 16 digests FIELD _zucarg_KS, 16*128, 64 ; array of 128-byte keystream of 16 buffers END_FIELDS %assign _ZUC_ARGS_X16_size _FIELD_OFFSET diff --git a/test/zuc_test.c b/test/zuc_test.c index 306a32bb..1b150da6 100644 --- a/test/zuc_test.c +++ b/test/zuc_test.c @@ -1088,7 +1088,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, memcpy(pSrcData[j], vector->message, byteLength); iv_lens[j] = vector->iv_length; } - for (tag_sz = 4; tag_sz <= 8; tag_sz *= 2) { + for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, @@ -1117,7 +1117,7 @@ int validate_zuc256_EIA3(struct IMB_MGR *mb_mgr, uint8_t **pSrcData, iv_lens[i] = vector->iv_length; } - for (tag_sz = 4; tag_sz <= 8; tag_sz *= 2) { + for (tag_sz = 4; tag_sz <= 16; tag_sz *= 2) { submit_eia3_jobs(mb_mgr, pKeys, pIV, pSrcData, pDstData, bitLength, numBuffs, -- GitLab From a782005879c466254dc9f8d3f242b14af8772579 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Thu, 11 Nov 2021 16:14:20 +0000 Subject: [PATCH 109/369] SUSE: add new spec file for packaging --- rpm/SUSE/intel-ipsec-mb.spec | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 rpm/SUSE/intel-ipsec-mb.spec diff --git a/rpm/SUSE/intel-ipsec-mb.spec b/rpm/SUSE/intel-ipsec-mb.spec new file mode 100644 index 00000000..0ee0733c --- /dev/null +++ b/rpm/SUSE/intel-ipsec-mb.spec @@ -0,0 +1,100 @@ +# +# spec file for package intel-ipsec-mb +# +# Copyright (c) 2022 SUSE LLC +# Copyright (c) 2022, Intel Corporation +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via https://bugs.opensuse.org/ +# + +# Versions numbers +%global major 1 +%global minor 2 +%global patch 0 + +# GitHub properties +%global githubver %{major}.%{minor} +%global githubfull %{name}-%{githubver} + +%global rpm_name libIPSec_MB + +Name: intel-ipsec-mb +Summary: IPSec cryptography library optimized for Intel Architecture +Release: 0 +Version: %{major}.%{minor}.%{patch} +License: BSD-3-Clause +Group: Development/Libraries/C and C++ +URL: https://github.com/intel/%{name} +Source0: https://github.com/intel/%{name}/archive/v%{githubver}.tar.gz#/%{githubfull}.tar.gz +ExclusiveArch: x86_64 +BuildRequires: make +BuildRequires: gcc >= 4.8.3 +BuildRequires: nasm >= 2.14 + +%description +An IPSec cryptography library optimized for Intel Architecture +and primarily targeted at packet processing applications. + +%package devel +Summary: Headers for the Intel IPSec cryptographic library +Requires: %{rpm_name}%{major}%{?_isa} = %{version}-%{release} +Group: Development/Libraries/C and C++ + +%description devel +An IPSec cryptography library optimized for Intel Architecture +and primarily targeted at packet processing applications. + +This package contains the headers for building programs with the library. + +%package -n %{rpm_name}%{major} +Summary: IPSec cryptography library optimized for Intel Architecture +Group: System/Libraries + +%description -n %{rpm_name}%{major} +An IPSec cryptography library optimized for Intel Architecture +and primarily targeted at packet processing applications. + +%prep +%autosetup -n %{name}-%{githubver} + +%build +cd lib +%make_build EXTRA_CFLAGS='%{optflags}' + +%install +install -d %{buildroot}/%{_includedir} +install -m 0644 %{_builddir}/%{githubfull}/lib/intel-ipsec-mb.h %{buildroot}/%{_includedir} +install -d %{buildroot}/%{_libdir} +install -s -m 0755 %{_builddir}/%{githubfull}/lib/libIPSec_MB.so.%{version} %{buildroot}/%{_libdir} +install -d %{buildroot}/%{_mandir}/man7 +install -m 0444 lib/libipsec-mb.7 %{buildroot}/%{_mandir}/man7 +install -m 0444 lib/libipsec-mb-dev.7 %{buildroot}/%{_mandir}/man7 +cd %{buildroot}/%{_libdir} +ln -s libIPSec_MB.so.%{version} libIPSec_MB.so.%{major} +ln -s libIPSec_MB.so.%{version} libIPSec_MB.so + +%files -n %{rpm_name}%{major} +%license LICENSE +%{_libdir}/libIPSec_MB.so.%{version} +%{_libdir}/libIPSec_MB.so.%{major} + +%files devel +%doc README ReleaseNotes.txt +%{_includedir}/intel-ipsec-mb.h +%{_mandir}/man7/libipsec-mb.7.gz +%{_mandir}/man7/libipsec-mb-dev.7.gz +%{_libdir}/libIPSec_MB.so + +%post -n %{rpm_name}%{major} -p /sbin/ldconfig +%postun -n %{rpm_name}%{major} -p /sbin/ldconfig + +%changelog -- GitLab From 8552d1c3c60881ee24f63a30f66ad48abb32367c Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 13 May 2022 13:34:22 +0000 Subject: [PATCH 110/369] lib: Improve burst API parameter checking --- lib/include/error.inc | 3 ++- lib/include/mb_mgr_code.h | 13 +++++++++++++ lib/intel-ipsec-mb.h | 3 +-- lib/x86_64/error.c | 5 ++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/include/error.inc b/lib/include/error.inc index b290ece5..e70167e2 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -92,7 +92,8 @@ SET_ERRNO_TYPES \ IMB_ERR_JOB_NULL_XCBC_K3, \ IMB_ERR_JOB_CIPH_DIR, \ IMB_ERR_JOB_NULL_GHASH_INIT_TAG, \ - IMB_ERR_MISSING_CPUFLAGS_INIT_MGR + IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, \ + IMB_ERR_NULL_JOB ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 28d6f52d..db554d7b 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3083,6 +3083,11 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + /* validate job */ if (is_job_invalid(state, job)) { job->status = IMB_STATUS_INVALID_ARGS; @@ -3140,6 +3145,10 @@ SUBMIT_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); return 0; } + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } #endif return submit_burst_and_check(state, jobs, n_jobs, 1); @@ -3156,6 +3165,10 @@ SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); return 0; } + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } #endif return submit_burst_and_check(state, jobs, n_jobs, 0); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 9fc02ec6..13bab042 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -177,7 +177,6 @@ typedef enum { */ typedef enum { IMB_ERR_MIN = 2000, - /* job api */ IMB_ERR_NULL_MBMGR, IMB_ERR_JOB_NULL_SRC, IMB_ERR_JOB_NULL_DST, @@ -199,7 +198,6 @@ typedef enum { IMB_ERR_JOB_NULL_SGL_CTX, IMB_ERR_JOB_NULL_NEXT_IV, IMB_ERR_JOB_PON_PLI, - /* direct api */ IMB_ERR_NULL_SRC, IMB_ERR_NULL_DST, IMB_ERR_NULL_KEY, @@ -225,6 +223,7 @@ typedef enum { IMB_ERR_JOB_CIPH_DIR, IMB_ERR_JOB_NULL_GHASH_INIT_TAG, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, + IMB_ERR_NULL_JOB, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index bcdbe9df..d4fe80df 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -86,7 +86,8 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_JOB_NULL_XCBC_K3, IMB_ERR_JOB_CIPH_DIR, IMB_ERR_JOB_NULL_GHASH_INIT_TAG, - IMB_ERR_MISSING_CPUFLAGS_INIT_MGR + IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, + IMB_ERR_NULL_JOB }; #ifdef DEBUG @@ -208,6 +209,8 @@ imb_get_strerror(int errnum) case IMB_ERR_MISSING_CPUFLAGS_INIT_MGR: return "Failed to initialize IMB_MGR due to missing " "required CPU flags"; + case IMB_ERR_NULL_JOB: + return "NULL job pointer"; default: return strerror(errnum); } -- GitLab From 0d359d4be82fcd72f34ba7a603b6acaadf61b003 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 13 May 2022 13:34:56 +0000 Subject: [PATCH 111/369] test: [API] Add burst API parameters check tests --- test/api_test.c | 317 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/test/api_test.c b/test/api_test.c index 7baf5b55..4599117c 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -38,11 +38,14 @@ #define __func__ __FUNCTION__ #endif +#define MAX_BURST_JOBS 32 + int api_test(struct IMB_MGR *mb_mgr); enum { TEST_UNEXPECTED_JOB = 1, TEST_INVALID_JOB, + TEST_INVALID_BURST, TEST_AUTH_SRC_NULL = 100, TEST_AUTH_AUTH_TAG_OUTPUT_NULL, TEST_AUTH_TAG_OUTPUT_LEN_ZERO, @@ -632,6 +635,150 @@ is_submit_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, return 1; } +/* + * @brief Submits \a job using the burst API and verifies it failed with + * invalid arguments status and error value + */ +static int +is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, + const int test_num, int expected_errnum) +{ + IMB_JOB jobs[MAX_BURST_JOBS] = {0}; + uint32_t i, completed_jobs; + int err; + + /* duplicate job to test */ + for (i = 0; i < MAX_BURST_JOBS; i++) + jobs[i] = *job; + + /* submit the job for processing */ + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, MAX_BURST_JOBS); + if (completed_jobs != 0) { + printf("%s : test %d, hash_alg %d, chain_order %d, " + "cipher_dir %d, cipher_mode %d : " + "unexpected number of completed jobs: %u\n", + __func__, test_num, (int) job->hash_alg, + (int) job->chain_order, (int) job->cipher_direction, + (int) job->cipher_mode, completed_jobs); + } + + err = imb_get_errno(mb_mgr); + if (err != expected_errnum) { + printf("%s : test %d, hash_alg %d, chain_order %d, " + "cipher_dir %d, cipher_mode %d : " + "unexpected error: %s\n", + __func__, test_num, (int) job->hash_alg, + (int) job->chain_order, (int) job->cipher_direction, + (int) job->cipher_mode, imb_get_strerror(err)); + return 0; + } + + if (jobs[0].status != IMB_STATUS_INVALID_ARGS) { + printf("%s : test %d, hash_alg %d, chain_order %d, " + "cipher_dir %d, cipher_mode %d : " + "unexpected job->status %d != IMB_STATUS_INVALID_ARGS\n", + __func__, test_num, (int) job->hash_alg, + (int) job->chain_order, + (int) job->cipher_direction, + (int) job->cipher_mode, (int) job->status); + return 0; + } + + return 1; +} + +/* + * @brief Performs BURST API behavior tests + */ +static int +test_burst_api(struct IMB_MGR *mb_mgr) +{ + struct IMB_JOB *job = NULL, jobs[MAX_BURST_JOBS] = {0}; + uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + int err; + + printf("BURST API behavior test:\n"); + + /* ======== test 1 : NULL MB_MGR */ + + /** + * Using sse API directly since IMB_SUBMIT_BURST + * macro cannot be used to test for NULL MB_MGR. + */ + + /* test only when SAFE_PARAM is enabled */ + if ((mb_mgr->features & IMB_FEATURE_SAFE_PARAM) != 0) { + + completed_jobs = submit_burst_sse(NULL, jobs, n_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of " + "completed jobs: %u\n", __func__, + TEST_INVALID_BURST, completed_jobs); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_MBMGR) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf("."); + } + + /* ======== test 2 : NULL jobs array */ + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, job, n_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_JOB) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, imb_get_strerror(err)); + return 1; + } + printf("."); + + /* ======== test 3 : invalid job */ + + /* fill in valid jobs */ + for (i = 0; i < n_jobs; i++) { + job = &jobs[i]; + fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); + } + + /* set a single invalid field */ + jobs[n_jobs - 1].enc_keys = NULL; + + /* no jobs should complete if any job is invalid */ + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, n_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_JOB_NULL_KEY) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, imb_get_strerror(err)); + return 1; + } + printf("."); + + printf("\n"); + return 0; +} + /* * @brief Checks for AEAD algorithms */ @@ -707,6 +854,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_SRC_NULL, IMB_ERR_JOB_NULL_SRC)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_AUTH_SRC_NULL, + IMB_ERR_JOB_NULL_SRC)) + return 1; printf("."); } @@ -739,6 +892,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_AUTH_TAG_OUTPUT_NULL, IMB_ERR_JOB_NULL_AUTH)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_AUTH_AUTH_TAG_OUTPUT_NULL, + IMB_ERR_JOB_NULL_AUTH)) + return 1; printf("."); } @@ -771,6 +930,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_TAG_OUTPUT_LEN_ZERO, IMB_ERR_JOB_AUTH_TAG_LEN)) return 1; + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_AUTH_TAG_OUTPUT_LEN_ZERO, + IMB_ERR_JOB_AUTH_TAG_LEN)) + return 1; printf("."); } @@ -849,6 +1013,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_MSG_LEN_GT_MAX, IMB_ERR_JOB_AUTH_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_AUTH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_AUTH_LEN)) + return 1; printf("."); } @@ -898,6 +1068,12 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_MSG_LEN_ZERO, IMB_ERR_JOB_AUTH_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_AUTH_MSG_LEN_ZERO, + IMB_ERR_JOB_AUTH_LEN)) + return 1; printf("."); } @@ -935,6 +1111,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_IV_LEN, IMB_ERR_JOB_IV_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_IV_LEN, + IMB_ERR_JOB_IV_LEN)) + return 1; printf("."); } @@ -976,6 +1157,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_HMAC_IPAD, err_ipad)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_HMAC_IPAD, + err_ipad)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -989,6 +1175,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_HMAC_OPAD, err_opad)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_HMAC_OPAD, + err_opad)) + return 1; printf("."); } @@ -1011,6 +1202,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_XCBC_K1_EXP, IMB_ERR_JOB_NULL_XCBC_K1_EXP)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_XCBC_K1_EXP, + IMB_ERR_JOB_NULL_XCBC_K1_EXP)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -1021,6 +1217,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_XCBC_K2, IMB_ERR_JOB_NULL_XCBC_K2)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_XCBC_K2, + IMB_ERR_JOB_NULL_XCBC_K2)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -1031,6 +1232,10 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_XCBC_K3, IMB_ERR_JOB_NULL_XCBC_K3)) return 1; + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_XCBC_K3, + IMB_ERR_JOB_NULL_XCBC_K3)) + return 1; printf("."); } @@ -1052,6 +1257,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_GHASH_KEY, IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_GHASH_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -1062,6 +1272,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_GHASH_INIT_TAG, IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_GHASH_INIT_TAG, + IMB_ERR_JOB_NULL_GHASH_INIT_TAG)) + return 1; printf("."); } @@ -1084,6 +1299,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_GMAC_KEY, IMB_ERR_JOB_NULL_AUTH_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_GMAC_KEY, + IMB_ERR_JOB_NULL_AUTH_KEY)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -1094,6 +1314,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_NULL_GMAC_IV, IMB_ERR_JOB_NULL_IV)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_NULL_GMAC_IV, + IMB_ERR_JOB_NULL_IV)) + return 1; printf("."); fill_in_job(job, cipher, dir, @@ -1104,6 +1329,11 @@ test_job_invalid_mac_args(struct IMB_MGR *mb_mgr) TEST_AUTH_GMAC_IV_LEN, IMB_ERR_JOB_IV_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_AUTH_GMAC_IV_LEN, + IMB_ERR_JOB_IV_LEN)) + return 1; printf("."); } } @@ -1162,6 +1392,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_SRC_NULL, IMB_ERR_JOB_NULL_SRC)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_SRC_NULL, + IMB_ERR_JOB_NULL_SRC)) + return 1; printf("."); } @@ -1191,6 +1427,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DST_NULL, IMB_ERR_JOB_NULL_DST)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_DST_NULL, + IMB_ERR_JOB_NULL_DST)) + return 1; printf("."); } @@ -1224,6 +1466,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_IV_NULL, IMB_ERR_JOB_NULL_IV)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_IV_NULL, + IMB_ERR_JOB_NULL_IV)) + return 1; printf("."); } /* @@ -1257,6 +1505,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DIR, IMB_ERR_JOB_CIPH_DIR)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, &template_job, + TEST_CIPH_DIR, + IMB_ERR_JOB_CIPH_DIR)) + return 1; printf("."); } } @@ -1289,6 +1542,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_ENC_KEY_NULL, IMB_ERR_JOB_NULL_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_ENC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) + return 1; break; } printf("."); @@ -1324,6 +1583,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DEC_KEY_NULL, IMB_ERR_JOB_NULL_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) + return 1; break; case IMB_CIPHER_CNTR: case IMB_CIPHER_CNTR_BITLEN: @@ -1338,6 +1603,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DEC_KEY_NULL, IMB_ERR_JOB_NULL_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) + return 1; break; case IMB_CIPHER_DOCSIS_SEC_BPI: template_job.enc_keys = NULL; @@ -1345,6 +1616,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DEC_KEY_NULL, IMB_ERR_JOB_NULL_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) + return 1; template_job.enc_keys = template_job.dec_keys; template_job.dec_keys = NULL; @@ -1352,6 +1629,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_DEC_KEY_NULL, IMB_ERR_JOB_NULL_KEY)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + &template_job, + TEST_CIPH_DEC_KEY_NULL, + IMB_ERR_JOB_NULL_KEY)) + return 1; break; case IMB_CIPHER_NULL: case IMB_CIPHER_CUSTOM: @@ -1404,6 +1687,12 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_MSG_LEN_ZERO, IMB_ERR_JOB_CIPH_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, + job, + TEST_CIPH_MSG_LEN_ZERO, + IMB_ERR_JOB_CIPH_LEN)) + return 1; } printf("."); } @@ -1483,6 +1772,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_CIPH_LEN)) return 1; + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_CIPH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_CIPH_LEN)) + return 1; + printf("."); } @@ -1615,6 +1909,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_IV_LEN, IMB_ERR_JOB_IV_LEN)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_CIPH_IV_LEN, + IMB_ERR_JOB_IV_LEN)) + return 1; printf("."); } } @@ -1648,6 +1947,11 @@ test_job_invalid_cipher_args(struct IMB_MGR *mb_mgr) TEST_CIPH_NEXT_IV_NULL, IMB_ERR_JOB_NULL_NEXT_IV)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_CIPH_NEXT_IV_NULL, + IMB_ERR_JOB_NULL_NEXT_IV)) + return 1; printf("."); } @@ -1703,6 +2007,11 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) TEST_INVALID_PON_PLI, IMB_ERR_JOB_PON_PLI)) return 1; + + if (!is_submit_burst_invalid(mb_mgr, &template_job, + TEST_INVALID_PON_PLI, + IMB_ERR_JOB_PON_PLI)) + return 1; printf("."); } @@ -1749,6 +2058,11 @@ test_job_invalid_misc_args(struct IMB_MGR *mb_mgr) IMB_ERR_JOB_CIPH_LEN)) return 1; + if (!is_submit_burst_invalid(mb_mgr, job, + TEST_CIPH_MSG_LEN_GT_MAX, + IMB_ERR_JOB_CIPH_LEN)) + return 1; + printf("."); } @@ -1955,6 +2269,9 @@ api_test(struct IMB_MGR *mb_mgr) errors += test_job_api(mb_mgr); run++; + errors += test_burst_api(mb_mgr); + run++; + errors += test_job_invalid_mac_args(mb_mgr); run++; -- GitLab From 73b4adea3fef81d8f94cef1acfc5ff726de128f7 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 11 May 2022 11:03:05 +0000 Subject: [PATCH 112/369] perf: 100ms time box added for a single algorithm@packet-size test duration - this is for Linux and FreeBSD only --- perf/ipsec_perf.c | 95 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index ff2bd321..56b4b70c 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -32,6 +32,10 @@ #include #include #include +#ifdef LINUX +#include +#include +#endif #ifdef _WIN32 #include /* memalign() or _aligned_malloc()/aligned_free() */ @@ -915,6 +919,17 @@ static int plot_output_option = 0; static uint32_t burst_api = 0; /* burst API enable/disable flag */ static uint32_t burst_size = 0; /* num jobs to pass to burst API */ +#ifdef LINUX +static volatile int timebox_on = 1; /* flag to stop the test loop */ +static int use_timebox = 1; /* time-box feature on/off flag */ + +static void timebox_callback(int sig) +{ + (void) sig; + timebox_on = 0; +} +#endif + /* Return rdtsc to core cycle scale factor */ static double get_tsc_to_core_scale(const int turbo) { @@ -1878,6 +1893,27 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, params->aad_size; } +#ifdef LINUX +#define TIMEOUT_MS 100 /*< max time for one packet size to be tested for */ + + uint32_t jobs_done = 0; /*< to track how many jobs done over time */ + + if (use_timebox) { + struct itimerval it_next; + + /* set up one shot timer */ + it_next.it_interval.tv_sec = 0; + it_next.it_interval.tv_usec = 0; + it_next.it_value.tv_sec = TIMEOUT_MS / 1000; + it_next.it_value.tv_usec = (TIMEOUT_MS % 1000) * 1000; + if (setitimer(ITIMER_REAL, &it_next, NULL)) { + perror("setitimer(one-shot)"); + exit(EXIT_FAILURE); + } + timebox_on = 1; + } +#endif + #ifndef _WIN32 if (use_unhalted_cycles) time = read_cycles(params->core); @@ -1889,7 +1925,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, IMB_JOB jobs[MAX_BURST_SIZE]; uint32_t num_jobs = num_iter; - while (num_jobs) { + while (num_jobs && timebox_on) { uint32_t n_jobs = (num_jobs / burst_size) ? burst_size : num_jobs; @@ -1920,8 +1956,11 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif num_jobs -= n_jobs; } +#ifdef LINUX + jobs_done = num_iter - num_jobs; +#endif } else { - for (i = 0; i < num_iter; i++) { + for (i = 0; (i < num_iter) && timebox_on; i++) { job = IMB_GET_NEXT_JOB(mb_mgr); *job = job_template; @@ -1945,7 +1984,9 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job = IMB_GET_COMPLETED_JOB(mb_mgr); } } - +#ifdef LINUX + jobs_done = i; +#endif while ((job = IMB_FLUSH_JOB(mb_mgr))) { #ifdef DEBUG if (job->status != IMB_STATUS_COMPLETED) { @@ -1971,6 +2012,25 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif time = __rdtscp(&aux) - time; +#ifdef LINUX + if (use_timebox) { + /* disarm the timer */ + struct itimerval it_disarm; + + memset(&it_disarm, 0, sizeof(it_disarm)); + + if (setitimer(ITIMER_REAL, &it_disarm, NULL)) { + perror("setitimer(disarm)"); + exit(EXIT_FAILURE); + } + + /* calculate return value */ + if (jobs_done == 0) + return 0; + + return time / jobs_done; + } +#endif if (!num_iter) return time; @@ -2728,6 +2788,10 @@ static void usage(void) "--no-tsc-detect: don't check TSC to core scaling\n" "--tag-size: modify tag size\n" "--plot: Adjust text output for direct use with plot output\n" +#ifdef LINUX + "--no-time-box: disables 100ms watchdog timer on " + "an algorithm@packet-size performance test\n" +#endif "--burst-api: use burst API for perf tests\n" "--burst-size: number of jobs to submit per burst\n", MAX_NUM_THREADS + 1); @@ -3312,6 +3376,10 @@ int main(int argc, char *argv[]) "more than %d\n", MAX_BURST_SIZE); return EXIT_FAILURE; } +#ifdef LINUX + } else if (strcmp(argv[i], "--no-time-box") == 0) { + use_timebox = 0; +#endif } else { usage(); return EXIT_FAILURE; @@ -3436,6 +3504,17 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } +#ifdef LINUX + /* Check timebox option vs number of threads bigger than 1 */ + if (use_timebox && num_t > 1) { + fprintf(stderr, + "Time-box feature, enabled by default, doesn't work " + "safely with number of threads bigger than one! Please " + "use '--no-time-box' option to disable\n"); + return EXIT_FAILURE; + } +#endif + /* if cycles selected then init MSR module */ if (use_unhalted_cycles) { if (core_mask == 0) { @@ -3508,6 +3587,16 @@ int main(int argc, char *argv[]) srand(ITER_SCALE_LONG + ITER_SCALE_SHORT + ITER_SCALE_SMOKE); +#ifdef LINUX + if (use_timebox) { + /* set up timebox callback function */ + if (signal(SIGALRM, timebox_callback) == SIG_ERR) { + perror("signal(SIGALRM)"); + return EXIT_FAILURE; + } + } +#endif + if (num_t > 1) { uint32_t n; -- GitLab From 85f0142b42d43b107809ad30f940dca650d04719 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 11 May 2022 12:48:54 +0000 Subject: [PATCH 113/369] perf: added support for --no-time-box option in the Python performance tool --- perf/ipsec_perf_tool.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/perf/ipsec_perf_tool.py b/perf/ipsec_perf_tool.py index 52a18349..4ef9bc0b 100755 --- a/perf/ipsec_perf_tool.py +++ b/perf/ipsec_perf_tool.py @@ -62,7 +62,7 @@ class Variant: hash_alg=None, aead_alg=None, sizes=None, offset=None, cold_cache=False, shani_off=False, gcm_job_api=False, unhalted_cycles=False, quick_test=False, smoke_test=False, - imix=None, aad_size=None, job_iter=None): + imix=None, aad_size=None, job_iter=None, no_time_box=False): """Build perf app command line""" global PERF_APP @@ -87,6 +87,7 @@ class Variant: self.imix = imix self.aad_size = aad_size self.job_iter = job_iter + self.no_time_box = no_time_box if self.arch is not None: self.cmd += ' --arch {}'.format(self.arch) @@ -139,6 +140,9 @@ class Variant: if self.smoke_test is True: self.cmd += ' --smoke' + if self.no_time_box is True: + self.cmd += ' --no-time-box' + if self.imix is not None: self.cmd += ' --imix {}'.format(self.imix) @@ -415,7 +419,8 @@ def parse_args(): help="size of AAD for AEAD algorithms") parser.add_argument("--job-iter", default=None, type=int, help="number of tests iterations for each job size") - + parser.add_argument("--no-time-box", default=False, action='store_true', + help="disables time box feature for single packet size test duration (100ms)") args = parser.parse_args() @@ -454,7 +459,7 @@ def parse_args(): alg_types, args.job_size, args.cold_cache, args.arch_best, \ args.shani_off, args.gcm_job_api, args.unhalted_cycles, \ args.quick, args.smoke, args.imix, \ - args.aad_size, args.job_iter + args.aad_size, args.job_iter, args.no_time_box def run_test(core=None): @@ -524,7 +529,7 @@ def main(): # parse command line args archs, cores, directions, offset, alg_types, sizes, cold_cache, arch_best, \ shani_off, gcm_job_api, unhalted_cycles, quick_test, smoke_test, \ - imix, aad_size, job_iter = parse_args() + imix, aad_size, job_iter, no_time_box = parse_args() # validate requested archs are supported if arch_best is True: @@ -575,7 +580,7 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 if 'hash-only' in alg_types: @@ -586,7 +591,7 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 if 'aead-only' in alg_types: @@ -597,7 +602,7 @@ def main(): cold_cache=cold_cache, shani_off=shani_off, gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, - aad_size=aad_size, job_iter=job_iter)) + aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 if 'cipher-hash-all' in alg_types: @@ -611,7 +616,7 @@ def main(): shani_off=shani_off, gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, - job_iter=job_iter)) + job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 # take starting timestamp -- GitLab From 6b25b3cbbc85f24e32f052de0309f31cab4a3653 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 11 May 2022 15:20:01 +0100 Subject: [PATCH 114/369] perf: added Windows support for time-box feature --- perf/ipsec_perf.c | 59 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 56b4b70c..f7b2da43 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -919,10 +919,10 @@ static int plot_output_option = 0; static uint32_t burst_api = 0; /* burst API enable/disable flag */ static uint32_t burst_size = 0; /* num jobs to pass to burst API */ -#ifdef LINUX static volatile int timebox_on = 1; /* flag to stop the test loop */ static int use_timebox = 1; /* time-box feature on/off flag */ +#ifdef LINUX static void timebox_callback(int sig) { (void) sig; @@ -930,6 +930,15 @@ static void timebox_callback(int sig) } #endif +#ifdef _WIN32 +static void CALLBACK timebox_callback(PVOID lpParam, BOOLEAN TimerFired) +{ + (void) lpParam; + (void) TimerFired; + timebox_on = 0; +} +#endif + /* Return rdtsc to core cycle scale factor */ static double get_tsc_to_core_scale(const int turbo) { @@ -1893,12 +1902,16 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, params->aad_size; } -#ifdef LINUX #define TIMEOUT_MS 100 /*< max time for one packet size to be tested for */ uint32_t jobs_done = 0; /*< to track how many jobs done over time */ +#ifdef _WIN32 + HANDLE hTimebox = NULL; + HANDLE hTimeboxQueue = NULL; +#endif if (use_timebox) { +#ifdef LINUX struct itimerval it_next; /* set up one shot timer */ @@ -1910,9 +1923,27 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, perror("setitimer(one-shot)"); exit(EXIT_FAILURE); } +#else /* _WIN32 */ + /* create the timer queue */ + hTimeboxQueue = CreateTimerQueue(); + if (NULL == hTimeboxQueue) { + fprintf(stderr, "CreateTimerQueue() error %u\n", + (unsigned) GetLastError()); + exit(EXIT_FAILURE); + } + + /* set a timer to call the timebox */ + if (!CreateTimerQueueTimer(&hTimebox, hTimeboxQueue, + (WAITORTIMERCALLBACK) + timebox_callback, + NULL, TIMEOUT_MS, 0, 0)) { + fprintf(stderr, "CreateTimerQueueTimer() error %u\n", + (unsigned) GetLastError()); + exit(EXIT_FAILURE); + } +#endif timebox_on = 1; } -#endif #ifndef _WIN32 if (use_unhalted_cycles) @@ -1956,9 +1987,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif num_jobs -= n_jobs; } -#ifdef LINUX jobs_done = num_iter - num_jobs; -#endif + } else { for (i = 0; (i < num_iter) && timebox_on; i++) { job = IMB_GET_NEXT_JOB(mb_mgr); @@ -1984,9 +2014,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job = IMB_GET_COMPLETED_JOB(mb_mgr); } } -#ifdef LINUX jobs_done = i; -#endif + while ((job = IMB_FLUSH_JOB(mb_mgr))) { #ifdef DEBUG if (job->status != IMB_STATUS_COMPLETED) { @@ -2012,8 +2041,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif time = __rdtscp(&aux) - time; -#ifdef LINUX if (use_timebox) { +#ifdef LINUX /* disarm the timer */ struct itimerval it_disarm; @@ -2023,6 +2052,12 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, perror("setitimer(disarm)"); exit(EXIT_FAILURE); } +#else /* _WIN32 */ + /* delete all timeboxes in the timer queue */ + if (!DeleteTimerQueue(hTimeboxQueue)) + fprintf(stderr, "DeleteTimerQueue() error %u\n", + (unsigned) GetLastError()); +#endif /* calculate return value */ if (jobs_done == 0) @@ -2030,7 +2065,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, return time / jobs_done; } -#endif + if (!num_iter) return time; @@ -2788,10 +2823,8 @@ static void usage(void) "--no-tsc-detect: don't check TSC to core scaling\n" "--tag-size: modify tag size\n" "--plot: Adjust text output for direct use with plot output\n" -#ifdef LINUX "--no-time-box: disables 100ms watchdog timer on " "an algorithm@packet-size performance test\n" -#endif "--burst-api: use burst API for perf tests\n" "--burst-size: number of jobs to submit per burst\n", MAX_NUM_THREADS + 1); @@ -3376,10 +3409,8 @@ int main(int argc, char *argv[]) "more than %d\n", MAX_BURST_SIZE); return EXIT_FAILURE; } -#ifdef LINUX } else if (strcmp(argv[i], "--no-time-box") == 0) { use_timebox = 0; -#endif } else { usage(); return EXIT_FAILURE; @@ -3504,7 +3535,6 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } -#ifdef LINUX /* Check timebox option vs number of threads bigger than 1 */ if (use_timebox && num_t > 1) { fprintf(stderr, @@ -3513,7 +3543,6 @@ int main(int argc, char *argv[]) "use '--no-time-box' option to disable\n"); return EXIT_FAILURE; } -#endif /* if cycles selected then init MSR module */ if (use_unhalted_cycles) { -- GitLab From 120d81c938cc78a9eab5f82a0d7114207ad76ac6 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 11 May 2022 14:50:57 +0000 Subject: [PATCH 115/369] build: [perf] disables volatile and comparison check rules in the style check --- perf/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perf/Makefile b/perf/Makefile index 97eb29e2..f204e575 100644 --- a/perf/Makefile +++ b/perf/Makefile @@ -126,4 +126,5 @@ style: $(CHECKPATCH) --no-tree --no-signoff --emacs --no-color \ --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,\ UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,\ -COMPLEX_MACRO,SPACING,STORAGE_CLASS,USE_FUNC,NEW_TYPEDEFS $(SOURCES_STYLE) +COMPLEX_MACRO,SPACING,STORAGE_CLASS,USE_FUNC,NEW_TYPEDEFS,VOLATILE,\ +CONSTANT_COMPARISON $(SOURCES_STYLE) -- GitLab From 48ab03c9e65065c8dc99312db3c329123e85ca0d Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 11 May 2022 13:17:02 +0000 Subject: [PATCH 116/369] test: [xvalid] increase max jobs for IMIX testing To improve code coverage, increase number of maximum jobs for IMIX testing, from 16 to 17, to cover multi-buffer implementations that have 16 lanes (so an extra job can replace a completed job). --- test/ipsec_xvalid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 18f6a2cc..2b0fc8bc 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -670,7 +670,7 @@ uint32_t job_sizes[NUM_RANGE] = {DEFAULT_JOB_SIZE_MIN, DEFAULT_JOB_SIZE_STEP, DEFAULT_JOB_SIZE_MAX}; /* Max number of jobs to submit in IMIX testing */ -uint32_t max_num_jobs = 16; +uint32_t max_num_jobs = 17; /* IMIX disabled by default */ unsigned int imix_enabled = 0; /* cipher and authentication IV sizes */ -- GitLab From c3d420480db88f309d3cb98388685b1421cb8393 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 11 May 2022 14:19:38 +0000 Subject: [PATCH 117/369] test: [xvalid] test multiple tag sizes for ZUC/CCM ZUC-EIA3-256 supports 3 different tag sizes (4, 8 and 16 bytes), which uses different code paths. AES-CCM supports 7 different tag sizes (4, 6, 8, 10, 12, 14 and 16 bytes), affecting the first block to authenticate.. This commit tests all these tag sizes for these two algorithms by default. --- test/ipsec_xvalid.c | 104 +++++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 34 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 2b0fc8bc..f6858754 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -70,6 +70,7 @@ #define MAX_GCM_AAD_SIZE 1024 #define MAX_CCM_AAD_SIZE 46 #define MAX_AAD_SIZE 1024 +#define NUM_TAG_SIZES 7 #define MAX_IV_SIZE 25 /* IV size for ZUC-256 */ #define MAX_TAG_SIZE 16 /* Max tag size for ZUC-256 */ @@ -136,6 +137,7 @@ struct data { uint8_t auth_key[MAX_KEY_SIZE]; struct cipher_auth_keys enc_keys; struct cipher_auth_keys dec_keys; + uint8_t tag_size; }; struct custom_job_params { @@ -580,7 +582,7 @@ struct variant_s { struct params_s params; }; -const uint8_t auth_tag_length_bytes[] = { +const uint8_t auth_tag_len_bytes[] = { 12, /* IMB_AUTH_HMAC_SHA_1 */ 14, /* IMB_AUTH_HMAC_SHA_224 */ 16, /* IMB_AUTH_HMAC_SHA_256 */ @@ -1742,12 +1744,7 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, unsigned int num_processed_jobs = 0; uint8_t next_iv[IMB_AES_BLOCK_SIZE]; uint16_t pli = 0; - uint8_t tag_size; - - if (auth_tag_size == 0) - tag_size = auth_tag_length_bytes[params->hash_alg - 1]; - else - tag_size = auth_tag_size; + uint8_t tag_size = data->tag_size; if (num_jobs == 0) return ret; @@ -2221,30 +2218,19 @@ exit: return ret; } -/* Runs test for each buffer size */ static void -process_variant(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, - IMB_MGR *dec_mgr, const IMB_ARCH dec_arch, - struct params_s *params, struct data *variant_data, - const unsigned int safe_check) +test_single(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, + IMB_MGR *dec_mgr, const IMB_ARCH dec_arch, + struct params_s *params, struct data *variant_data, + const uint32_t buf_size, + const unsigned int safe_check) { -#ifdef PIN_BASED_CEC - const uint32_t sizes = job_sizes[RANGE_MAX]; -#else - const uint32_t sizes = params->num_sizes; -#endif - uint32_t sz; + unsigned int i; + unsigned int num_tag_sizes = 0; + uint8_t tag_sizes[NUM_TAG_SIZES]; uint64_t min_aad_sz = 0; uint64_t max_aad_sz, aad_sz; - if (verbose) { - printf("[INFO] "); - print_algo_info(params); - } - - /* Reset the variant data */ - clear_data(variant_data); - if (params->cipher_mode == IMB_CIPHER_GCM) max_aad_sz = MAX_GCM_AAD_SIZE; else if (params->cipher_mode == IMB_CIPHER_CCM) @@ -2252,13 +2238,28 @@ process_variant(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, else max_aad_sz = 0; - for (sz = 0; sz < sizes; sz++) { -#ifdef PIN_BASED_CEC - const uint32_t buf_size = job_sizes[RANGE_MIN]; -#else - const uint32_t buf_size = job_sizes[RANGE_MIN] + - (sz * job_sizes[RANGE_STEP]); -#endif + /* If tag size is defined by user, only test this size */ + if (auth_tag_size != 0) { + tag_sizes[0] = auth_tag_size; + num_tag_sizes = 1; + } else { + /* If CCM, test all tag sizes supported (4,6,8,10,12,14,16) */ + if (params->hash_alg == IMB_AUTH_AES_CCM) { + for (i = 4; i <= 16; i += 2) + tag_sizes[num_tag_sizes++] = i; + /* If ZUC-EIA3-256, test all tag sizes supported (4,8,16) */ + } else if (params->hash_alg == IMB_AUTH_ZUC256_EIA3_BITLEN) { + for (i = 4; i <= 16; i *= 2) + tag_sizes[num_tag_sizes++] = i; + } else { + tag_sizes[0] = auth_tag_len_bytes[params->hash_alg - 1]; + num_tag_sizes = 1; + } + } + + for (i = 0; i < num_tag_sizes; i++) { + variant_data->tag_size = tag_sizes[i]; + for (aad_sz = min_aad_sz; aad_sz <= max_aad_sz; aad_sz++) { params->aad_size = aad_sz; params->buf_size = buf_size; @@ -2326,14 +2327,49 @@ process_variant(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, params, variant_data, 0, 0, 1) < 0) exit(EXIT_FAILURE); } + } +} + +/* Runs test for each buffer size */ +static void +process_variant(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, + IMB_MGR *dec_mgr, const IMB_ARCH dec_arch, + struct params_s *params, struct data *variant_data, + const unsigned int safe_check) +{ +#ifdef PIN_BASED_CEC + const uint32_t sizes = job_sizes[RANGE_MAX]; +#else + const uint32_t sizes = params->num_sizes; +#endif + uint32_t sz; + + if (verbose) { + printf("[INFO] "); + print_algo_info(params); + } + + /* Reset the variant data */ + clear_data(variant_data); + + for (sz = 0; sz < sizes; sz++) { +#ifdef PIN_BASED_CEC + const uint32_t buf_size = job_sizes[RANGE_MIN]; +#else + const uint32_t buf_size = job_sizes[RANGE_MIN] + + (sz * job_sizes[RANGE_STEP]); +#endif + test_single(enc_mgr, enc_arch, dec_mgr, + dec_arch, params, variant_data, + buf_size, safe_check); } /* Perform IMIX tests */ if (imix_enabled) { unsigned int i, j; - params->aad_size = min_aad_sz; + params->aad_size = 0; for (i = 2; i <= max_num_jobs; i++) { for (j = 0; j < IMIX_ITER; j++) { -- GitLab From 0231cceaeae3d501bf0427d473d90223a72db2fc Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Mon, 16 May 2022 08:36:18 +0000 Subject: [PATCH 118/369] avx: [SHA1] expose multi-buffer API --- lib/Makefile | 1 + lib/avx/mb_mgr_avx.c | 20 ++++++++++++ lib/avx/sha1_x4_avx.asm | 64 ++++++++++++++++++++++++++++++++++++++- lib/avx/sha_mb_avx.c | 52 +++++++++++++++++++++++++++++++ lib/include/mb_mgr_code.h | 8 ++--- lib/include/sha_mb_mgr.h | 3 ++ lib/win_x64.mak | 1 + 7 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 lib/avx/sha_mb_avx.c diff --git a/lib/Makefile b/lib/Makefile index 53c29600..a3c5aa1a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -258,6 +258,7 @@ c_lib_objs := \ sha_sse.o \ sha_mb_sse.o \ sha_avx.o \ + sha_mb_avx.o \ des_key.o \ des_basic.o \ version.o \ diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index d2c74be7..6d1e75aa 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -29,6 +29,7 @@ #include #include +#define AVX #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_avx #include "intel-ipsec-mb.h" @@ -84,6 +85,12 @@ IMB_JOB *submit_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); IMB_JOB *flush_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); + +IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); + uint32_t hec_32_avx(const uint8_t *in); uint64_t hec_64_avx(const uint8_t *in); @@ -172,6 +179,9 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx +#define FLUSH_JOB_SHA1 flush_job_sha1_avx + #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx @@ -505,6 +515,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -862,6 +873,15 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(aes128_cbcs_ooo->job_in_lane)); aes128_cbcs_ooo->unused_lanes = 0xF76543210; aes128_cbcs_ooo->num_lanes_inuse = 0; + + /* Init SHA1 out-of-order fields */ + sha_1_ooo->lens[0] = 0; + sha_1_ooo->lens[1] = 0; + sha_1_ooo->lens[2] = 0; + sha_1_ooo->lens[3] = 0; + sha_1_ooo->unused_lanes = 0xF3210; + for (j = 0; j < AVX_NUM_SHA1_LANES; j++) + sha_1_ooo->ldata[j].job_in_lane = NULL; } IMB_DLL_LOCAL void diff --git a/lib/avx/sha1_x4_avx.asm b/lib/avx/sha1_x4_avx.asm index 70e6d8d0..79e68f3e 100644 --- a/lib/avx/sha1_x4_avx.asm +++ b/lib/avx/sha1_x4_avx.asm @@ -75,6 +75,12 @@ mksection .text ; r0 = {d2 c2 b2 a2} ; r3 = {d3 c3 b3 a3} ; +%define XMM_STORAGE 16*10 +%define GP_STORAGE 8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + %macro TRANSPOSE 6 %define %%r0 %1 %define %%r1 %2 @@ -288,6 +294,55 @@ mksection .text %xdefine W14 TMP_ %endm +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes +%ifndef LINUX + vmovdqa [rsp + 0*16], xmm6 + vmovdqa [rsp + 1*16], xmm7 + vmovdqa [rsp + 2*16], xmm8 + vmovdqa [rsp + 3*16], xmm9 + vmovdqa [rsp + 4*16], xmm10 + vmovdqa [rsp + 5*16], xmm11 + vmovdqa [rsp + 6*16], xmm12 + vmovdqa [rsp + 7*16], xmm13 + vmovdqa [rsp + 8*16], xmm14 + vmovdqa [rsp + 9*16], xmm15 +%endif + mov [rsp + GP_OFFSET], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 +%ifndef LINUX + vmovdqa xmm6, [rsp + 0*16] + vmovdqa xmm7, [rsp + 1*16] + vmovdqa xmm8, [rsp + 2*16] + vmovdqa xmm9, [rsp + 3*16] + vmovdqa xmm10, [rsp + 4*16] + vmovdqa xmm11, [rsp + 5*16] + vmovdqa xmm12, [rsp + 6*16] + vmovdqa xmm13, [rsp + 7*16] + vmovdqa xmm14, [rsp + 8*16] + vmovdqa xmm15, [rsp + 9*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa [rsp + 0*16], xmm5 + vmovdqa [rsp + 1*16], xmm5 + vmovdqa [rsp + 2*16], xmm5 + vmovdqa [rsp + 3*16], xmm5 + vmovdqa [rsp + 4*16], xmm5 + vmovdqa [rsp + 5*16], xmm5 + vmovdqa [rsp + 6*16], xmm5 + vmovdqa [rsp + 7*16], xmm5 + vmovdqa [rsp + 8*16], xmm5 + vmovdqa [rsp + 9*16], xmm5 +%endif +%endif + mov rsp, [rsp + GP_OFFSET] ;; rsp pointer +%endmacro + align 32 ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -426,8 +481,15 @@ lloop: %endrep %endif - add rsp, FRAMESZ + add rsp, FRAMESZ + ret +; void call_sha1_mult_avx_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha1_mult_avx_from_c,function,internal) +call_sha1_mult_avx_from_c: + FUNC_SAVE + call sha1_mult_avx + FUNC_RESTORE ret mksection stack-noexec diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c new file mode 100644 index 00000000..02966d68 --- /dev/null +++ b/lib/avx/sha_mb_avx.c @@ -0,0 +1,52 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_mb_mgr.h" + +IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); + +/* ========================================================================== */ +/* + * SHA1 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 4, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_avx_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 4, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_avx_from_c); +} diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index db554d7b..0db409a6 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1006,7 +1006,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1066,7 +1066,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: - #ifdef SSE + #if defined(SSE) || defined(AVX) return SUBMIT_JOB_SHA1(sha_1_ooo, job); #else IMB_SHA1(state, @@ -1218,7 +1218,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1251,7 +1251,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: - #ifdef SSE + #if defined(SSE) || defined(AVX) return FLUSH_JOB_SHA1(sha_1_ooo, job); #endif case IMB_AUTH_AES_XCBC: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 4160c58d..26a41de1 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -32,6 +32,9 @@ extern void call_sha1_mult_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void call_sha1_mult_avx_from_c(SHA1_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 6850b6dd..7b58d9c6 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -279,6 +279,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha_sse.obj \ $(OBJ_DIR)\sha_avx.obj \ $(OBJ_DIR)\sha_mb_sse.obj \ + $(OBJ_DIR)\sha_mb_avx.obj \ $(OBJ_DIR)\des_key.obj \ $(OBJ_DIR)\des_basic.obj \ $(OBJ_DIR)\chacha20_sse.obj \ -- GitLab From 617d6b2209dd51bb6cb0ed8726f9f6b8eeffdf06 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 17 May 2022 12:31:00 +0000 Subject: [PATCH 119/369] avx2: [SHA1] expose multi-buffer API --- lib/Makefile | 1 + lib/avx2/mb_mgr_avx2.c | 16 +++++++++ lib/avx2/sha1_x8_avx2.asm | 73 ++++++++++++++++++++++++++++++++++++++- lib/avx2/sha_mb_avx2.c | 52 ++++++++++++++++++++++++++++ lib/include/mb_mgr_code.h | 8 ++--- lib/include/sha_mb_mgr.h | 3 ++ lib/win_x64.mak | 1 + 7 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 lib/avx2/sha_mb_avx2.c diff --git a/lib/Makefile b/lib/Makefile index a3c5aa1a..39115406 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -259,6 +259,7 @@ c_lib_objs := \ sha_mb_sse.o \ sha_avx.o \ sha_mb_avx.o \ + sha_mb_avx2.o \ des_key.o \ des_basic.o \ version.o \ diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 4ecaf022..616beba4 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -87,6 +87,11 @@ IMB_JOB *submit_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); +IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_avx2(const void *key_exp, void *key1, void *key2); uint32_t hec_32_avx(const uint8_t *in); @@ -180,6 +185,9 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx2 + #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx @@ -482,6 +490,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -839,6 +848,13 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(aes128_cbcs_ooo->job_in_lane)); aes128_cbcs_ooo->unused_lanes = 0xF76543210; aes128_cbcs_ooo->num_lanes_inuse = 0; + + /* Init SHA1 out-of-order fields */ + sha_1_ooo->unused_lanes = 0xF76543210; + for (j = 0; j < AVX2_NUM_SHA1_LANES; j++){ + sha_1_ooo->lens[j] = 0; + sha_1_ooo->ldata[j].job_in_lane = NULL; + } } IMB_DLL_LOCAL void diff --git a/lib/avx2/sha1_x8_avx2.asm b/lib/avx2/sha1_x8_avx2.asm index f25dc6a0..cf5b86aa 100644 --- a/lib/avx2/sha1_x8_avx2.asm +++ b/lib/avx2/sha1_x8_avx2.asm @@ -67,6 +67,12 @@ K60_79: ;ddq 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 mksection .text +%define XMM_STORAGE 16*10 +%define GP_STORAGE 8*5 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + %ifdef LINUX %define arg1 rdi %define arg2 rsi @@ -296,6 +302,63 @@ mksection .text %xdefine W14 TMP_ %endm +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes +%ifndef LINUX + vmovdqa [rsp + 0*16], xmm6 + vmovdqa [rsp + 1*16], xmm7 + vmovdqa [rsp + 2*16], xmm8 + vmovdqa [rsp + 3*16], xmm9 + vmovdqa [rsp + 4*16], xmm10 + vmovdqa [rsp + 5*16], xmm11 + vmovdqa [rsp + 6*16], xmm12 + vmovdqa [rsp + 7*16], xmm13 + vmovdqa [rsp + 8*16], xmm14 + vmovdqa [rsp + 9*16], xmm15 +%endif + mov [rsp + GP_OFFSET], r12 + mov [rsp + GP_OFFSET + 8], r13 + mov [rsp + GP_OFFSET + 2*8], r14 + mov [rsp + GP_OFFSET + 3*8], r15 + mov [rsp + GP_OFFSET + 4*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 +%ifndef LINUX + vmovdqa xmm6, [rsp + 0*16] + vmovdqa xmm7, [rsp + 1*16] + vmovdqa xmm8, [rsp + 2*16] + vmovdqa xmm9, [rsp + 3*16] + vmovdqa xmm10, [rsp + 4*16] + vmovdqa xmm11, [rsp + 5*16] + vmovdqa xmm12, [rsp + 6*16] + vmovdqa xmm13, [rsp + 7*16] + vmovdqa xmm14, [rsp + 8*16] + vmovdqa xmm15, [rsp + 9*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa [rsp + 0*16], xmm5 + vmovdqa [rsp + 1*16], xmm5 + vmovdqa [rsp + 2*16], xmm5 + vmovdqa [rsp + 3*16], xmm5 + vmovdqa [rsp + 4*16], xmm5 + vmovdqa [rsp + 5*16], xmm5 + vmovdqa [rsp + 6*16], xmm5 + vmovdqa [rsp + 7*16], xmm5 + vmovdqa [rsp + 8*16], xmm5 + vmovdqa [rsp + 9*16], xmm5 +%endif +%endif + mov r12, [rsp + GP_OFFSET] + mov r13, [rsp + GP_OFFSET + 8] + mov r14, [rsp + GP_OFFSET + 2*8] + mov r15, [rsp + GP_OFFSET + 3*8] + mov rsp, [rsp + GP_OFFSET + 4*8] ;; rsp pointer +%endmacro + align 32 ; void sha1_x8_avx2(void *state, int num_blks) @@ -454,8 +517,16 @@ lloop: %endrep %endif - add rsp, FRAMESZ + add rsp, FRAMESZ + + ret +; void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha1_x8_avx2_from_c,function,internal) +call_sha1_x8_avx2_from_c: + FUNC_SAVE + call sha1_x8_avx2 + FUNC_RESTORE ret mksection stack-noexec diff --git a/lib/avx2/sha_mb_avx2.c b/lib/avx2/sha_mb_avx2.c new file mode 100644 index 00000000..e496bfa8 --- /dev/null +++ b/lib/avx2/sha_mb_avx2.c @@ -0,0 +1,52 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_mb_mgr.h" + +IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); + +/* ========================================================================== */ +/* + * SHA1 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 8, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_x8_avx2_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 8, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_x8_avx2_from_c); +} diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 0db409a6..e4ecd833 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1006,7 +1006,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1066,7 +1066,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: - #if defined(SSE) || defined(AVX) + #if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA1(sha_1_ooo, job); #else IMB_SHA1(state, @@ -1218,7 +1218,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #endif #ifdef AVX512 @@ -1251,7 +1251,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: - #if defined(SSE) || defined(AVX) + #if defined(SSE) || defined(AVX) || defined(AVX2) return FLUSH_JOB_SHA1(sha_1_ooo, job); #endif case IMB_AUTH_AES_XCBC: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 26a41de1..5f9f6db1 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -35,6 +35,9 @@ extern void call_sha1_mult_sse_from_c(SHA1_ARGS *args, extern void call_sha1_mult_avx_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 7b58d9c6..a63c83cb 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -280,6 +280,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha_avx.obj \ $(OBJ_DIR)\sha_mb_sse.obj \ $(OBJ_DIR)\sha_mb_avx.obj \ + $(OBJ_DIR)\sha_mb_avx2.obj \ $(OBJ_DIR)\des_key.obj \ $(OBJ_DIR)\des_basic.obj \ $(OBJ_DIR)\chacha20_sse.obj \ -- GitLab From 1a0210ef93e086256d549ab6c5f7c0560c9d8a6f Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 17 May 2022 14:23:17 +0000 Subject: [PATCH 120/369] avx512: [SHA1] expose multi-buffer API --- lib/Makefile | 1 + lib/avx512/mb_mgr_avx512.c | 17 ++++++++ lib/avx512/sha1_x16_avx512.asm | 70 ++++++++++++++++++++++++++++++ lib/avx512/sha_mb_avx512.c | 52 ++++++++++++++++++++++ lib/include/mb_mgr_code.h | 14 ------ lib/include/sha_mb_mgr.h | 8 +++- lib/no-aesni/mb_mgr_sse_no_aesni.c | 16 +++++++ lib/win_x64.mak | 1 + 8 files changed, 163 insertions(+), 16 deletions(-) create mode 100644 lib/avx512/sha_mb_avx512.c diff --git a/lib/Makefile b/lib/Makefile index 39115406..ea5e1b1d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -260,6 +260,7 @@ c_lib_objs := \ sha_avx.o \ sha_mb_avx.o \ sha_mb_avx2.o \ + sha_mb_avx512.o \ des_key.o \ des_basic.o \ version.o \ diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 3c070cf4..5b5d044d 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -153,6 +153,11 @@ IMB_JOB *submit_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); +IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_avx512(const void *key_exp, void *key1, void *key2); uint32_t hec_32_avx(const uint8_t *in); @@ -250,6 +255,9 @@ IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx512 #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx512 +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx512 + #define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 #define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 @@ -1170,6 +1178,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; + MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { @@ -1777,6 +1786,14 @@ reset_ooo_mgrs(IMB_MGR *state) snow3g_uia2_ooo->init_done = 0; memset(snow3g_uia2_ooo->lens, 0, sizeof(snow3g_uia2_ooo->lens)); + + /* Init SHA1 out-of-order fields */ + sha_1_ooo->unused_lanes = 0xFEDCBA9876543210; + sha_1_ooo->num_lanes_inuse = 0; + for (j = 0; j < AVX512_NUM_SHA1_LANES; j++){ + sha_1_ooo->lens[j] = 0; + sha_1_ooo->ldata[j].job_in_lane = NULL; + } } IMB_DLL_LOCAL void diff --git a/lib/avx512/sha1_x16_avx512.asm b/lib/avx512/sha1_x16_avx512.asm index e0a43b5f..2822535e 100644 --- a/lib/avx512/sha1_x16_avx512.asm +++ b/lib/avx512/sha1_x16_avx512.asm @@ -94,6 +94,12 @@ mksection .text %define APPEND(a,b) a %+ b +%define XMM_STORAGE 16*10 +%define GP_STORAGE 8*5 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + %ifdef LINUX %define arg1 rdi %define arg2 rsi @@ -229,6 +235,62 @@ mksection .text %endif %endmacro +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET +%ifndef LINUX + vmovdqa32 [rsp + 0*16], xmm6 + vmovdqa32 [rsp + 1*16], xmm7 + vmovdqa32 [rsp + 2*16], xmm8 + vmovdqa32 [rsp + 3*16], xmm9 + vmovdqa32 [rsp + 4*16], xmm10 + vmovdqa32 [rsp + 5*16], xmm11 + vmovdqa32 [rsp + 6*16], xmm12 + vmovdqa32 [rsp + 7*16], xmm13 + vmovdqa32 [rsp + 8*16], xmm14 + vmovdqa32 [rsp + 9*16], xmm15 +%endif + mov [rsp + GP_OFFSET], r12 + mov [rsp + GP_OFFSET + 8], r13 + mov [rsp + GP_OFFSET + 2*8], r14 + mov [rsp + GP_OFFSET + 3*8], r15 + mov [rsp + GP_OFFSET + 4*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 +%ifndef LINUX + vmovdqa32 xmm6, [rsp + 0*16] + vmovdqa32 xmm7, [rsp + 1*16] + vmovdqa32 xmm8, [rsp + 2*16] + vmovdqa32 xmm9, [rsp + 3*16] + vmovdqa32 xmm10, [rsp + 4*16] + vmovdqa32 xmm11, [rsp + 5*16] + vmovdqa32 xmm12, [rsp + 6*16] + vmovdqa32 xmm13, [rsp + 7*16] + vmovdqa32 xmm14, [rsp + 8*16] + vmovdqa32 xmm15, [rsp + 9*16] + +%ifdef SAFE_DATA + vpxord xmm5, xmm5, xmm5 + vmovdqa32 [rsp + 0*16], xmm5 + vmovdqa32 [rsp + 1*16], xmm5 + vmovdqa32 [rsp + 2*16], xmm5 + vmovdqa32 [rsp + 3*16], xmm5 + vmovdqa32 [rsp + 4*16], xmm5 + vmovdqa32 [rsp + 5*16], xmm5 + vmovdqa32 [rsp + 6*16], xmm5 + vmovdqa32 [rsp + 7*16], xmm5 + vmovdqa32 [rsp + 8*16], xmm5 + vmovdqa32 [rsp + 9*16], xmm5 +%endif +%endif + mov r12, [rsp + GP_OFFSET] + mov r13, [rsp + GP_OFFSET + 8] + mov r14, [rsp + GP_OFFSET + 2*8] + mov r15, [rsp + GP_OFFSET + 3*8] + mov rsp, [rsp + GP_OFFSET + 4*8] ;; rsp pointer +%endmacro + align 64 ; void sha1_mult_x16_avx3(void **input_data, UINT128 *digest, UINT32 size) ; arg 1 : pointer to SHA1 args structure @@ -443,4 +505,12 @@ lastLoop: ret +; void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha1_x16_avx512_from_c,function,internal) +call_sha1_x16_avx512_from_c: + FUNC_SAVE + call sha1_x16_avx512 + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx512/sha_mb_avx512.c b/lib/avx512/sha_mb_avx512.c new file mode 100644 index 00000000..188e9eea --- /dev/null +++ b/lib/avx512/sha_mb_avx512.c @@ -0,0 +1,52 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_mb_mgr.h" + +IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); + +/* ========================================================================== */ +/* + * SHA1 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 16, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_x16_avx512_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 16, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_x16_avx512_from_c); +} diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index e4ecd833..3fb04875 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1006,9 +1006,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -1066,15 +1064,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: - #if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA1(sha_1_ooo, job); - #else - IMB_SHA1(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - #endif case IMB_AUTH_SHA_224: IMB_SHA224(state, job->src + job->hash_start_src_offset_in_bytes, @@ -1218,9 +1208,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -1251,9 +1239,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: - #if defined(SSE) || defined(AVX) || defined(AVX2) return FLUSH_JOB_SHA1(sha_1_ooo, job); - #endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 5f9f6db1..0eff90e0 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -38,6 +38,9 @@ extern void call_sha1_mult_avx_from_c(SHA1_ARGS *args, extern void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) @@ -189,9 +192,10 @@ submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, } /* subtract min len from all lanes */ + const uint64_t min_len_blk = min_len & (~(blk_size - 1)); + for (i = 0; i < max_jobs; i++) - state->lens[i] -= - (uint16_t)((min_len/blk_size)*blk_size); + state->lens[i] -= min_len_blk; const uint64_t r = min_len % blk_size; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 53e53b18..f949cafc 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -127,6 +127,11 @@ IMB_JOB *submit_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); +IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); + uint32_t hec_32_sse_no_aesni(const uint8_t *in); uint64_t hec_64_sse_no_aesni(const uint8_t *in); @@ -179,6 +184,8 @@ IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse_no_aesni #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni +#define SUBMIT_JOB_SHA1 submit_job_sha1_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni @@ -863,6 +870,15 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(aes128_cbcs_ooo->job_in_lane)); aes128_cbcs_ooo->num_lanes_inuse = 0; aes128_cbcs_ooo->unused_lanes = 0xF3210; + + /* Init SHA1 out-of-order fields */ + sha_1_ooo->lens[0] = 0; + sha_1_ooo->lens[1] = 0; + sha_1_ooo->lens[2] = 0; + sha_1_ooo->lens[3] = 0; + sha_1_ooo->unused_lanes = 0xF3210; + for (j = 0; j < SSE_NUM_SHA1_LANES; j++) + sha_1_ooo->ldata[j].job_in_lane = NULL; } IMB_DLL_LOCAL void diff --git a/lib/win_x64.mak b/lib/win_x64.mak index a63c83cb..d76a3849 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -281,6 +281,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha_mb_sse.obj \ $(OBJ_DIR)\sha_mb_avx.obj \ $(OBJ_DIR)\sha_mb_avx2.obj \ + $(OBJ_DIR)\sha_mb_avx512.obj \ $(OBJ_DIR)\des_key.obj \ $(OBJ_DIR)\des_basic.obj \ $(OBJ_DIR)\chacha20_sse.obj \ -- GitLab From 09b7e829700fdfdc1fe7bcbdc0085c77d9efa909 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 17 May 2022 15:50:22 +0000 Subject: [PATCH 121/369] lib: [SHA1] add new OOO MGR --- lib/avx/mb_mgr_avx.c | 7 ++++--- lib/avx/sha_mb_avx.c | 8 ++++---- lib/avx2/mb_mgr_avx2.c | 7 ++++--- lib/avx2/sha_mb_avx2.c | 8 ++++---- lib/avx512/mb_mgr_avx512.c | 6 +++--- lib/avx512/sha_mb_avx512.c | 8 ++++---- lib/include/ipsec_ooo_mgr.h | 9 +++++++++ lib/include/mb_mgr_code.h | 4 ++-- lib/include/sha_mb_mgr.h | 15 +++++++++------ lib/no-aesni/mb_mgr_sse_no_aesni.c | 6 ++++-- lib/sse/mb_mgr_sse.c | 7 ++++--- lib/sse/sha_mb_sse.c | 8 ++++---- lib/x86_64/alloc.c | 2 +- 13 files changed, 56 insertions(+), 39 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 6d1e75aa..b84ce7de 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -86,9 +86,9 @@ IMB_JOB *submit_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); uint32_t hec_32_avx(const uint8_t *in); @@ -515,7 +515,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -880,6 +880,7 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->lens[2] = 0; sha_1_ooo->lens[3] = 0; sha_1_ooo->unused_lanes = 0xF3210; + sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < AVX_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; } diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index 02966d68..923eaade 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -27,8 +27,8 @@ #include "include/sha_mb_mgr.h" -IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); /* ========================================================================== */ /* @@ -36,7 +36,7 @@ IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); */ IMB_DLL_LOCAL -IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, @@ -44,7 +44,7 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) } IMB_DLL_LOCAL -IMB_JOB *flush_job_sha1_avx(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 616beba4..d122e5d0 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -87,9 +87,9 @@ IMB_JOB *submit_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); void aes_cmac_256_subkey_gen_avx2(const void *key_exp, @@ -490,7 +490,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -851,6 +851,7 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA1 out-of-order fields */ sha_1_ooo->unused_lanes = 0xF76543210; + sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < AVX2_NUM_SHA1_LANES; j++){ sha_1_ooo->lens[j] = 0; sha_1_ooo->ldata[j].job_in_lane = NULL; diff --git a/lib/avx2/sha_mb_avx2.c b/lib/avx2/sha_mb_avx2.c index e496bfa8..39854bc4 100644 --- a/lib/avx2/sha_mb_avx2.c +++ b/lib/avx2/sha_mb_avx2.c @@ -27,8 +27,8 @@ #include "include/sha_mb_mgr.h" -IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); /* ========================================================================== */ /* @@ -36,7 +36,7 @@ IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); */ IMB_DLL_LOCAL -IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 8, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, @@ -44,7 +44,7 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) } IMB_DLL_LOCAL -IMB_JOB *flush_job_sha1_avx2(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 8, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 5b5d044d..ce8157cd 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -153,9 +153,9 @@ IMB_JOB *submit_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); void aes_cmac_256_subkey_gen_avx512(const void *key_exp, @@ -1178,7 +1178,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { diff --git a/lib/avx512/sha_mb_avx512.c b/lib/avx512/sha_mb_avx512.c index 188e9eea..254ebb52 100644 --- a/lib/avx512/sha_mb_avx512.c +++ b/lib/avx512/sha_mb_avx512.c @@ -27,8 +27,8 @@ #include "include/sha_mb_mgr.h" -IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); /* ========================================================================== */ /* @@ -36,7 +36,7 @@ IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); */ IMB_DLL_LOCAL -IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 16, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, @@ -44,7 +44,7 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) } IMB_DLL_LOCAL -IMB_JOB *flush_job_sha1_avx512(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 16, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 94baba03..129f607e 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -338,6 +338,15 @@ typedef struct { uint64_t road_block; } MB_MGR_HMAC_SHA_1_OOO; +typedef struct { + SHA1_ARGS args; + DECLARE_ALIGNED(uint64_t lens[AVX512_NUM_SHA1_LANES], 32); + uint64_t unused_lanes; + HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA1_LANES]; + uint32_t num_lanes_inuse; + uint64_t road_block; +} MB_MGR_SHA_1_OOO; + typedef struct { SHA256_ARGS args; DECLARE_ALIGNED(uint16_t lens[16], 16); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 3fb04875..424e8084 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1006,7 +1006,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -1208,7 +1208,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 0eff90e0..a8ef5204 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -82,7 +82,7 @@ void sha_mb_generic_write_digest(void *dst, const void *src, } __forceinline -void create_extra_blocks(MB_MGR_HMAC_SHA_1_OOO *state, +void create_extra_blocks(MB_MGR_SHA_1_OOO *state, const uint64_t blk_size, const uint64_t r, const unsigned min_idx) { @@ -106,13 +106,14 @@ void create_extra_blocks(MB_MGR_HMAC_SHA_1_OOO *state, __forceinline IMB_JOB * -submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, +submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, const unsigned max_jobs, const int is_submit, const int sha_type, const uint64_t blk_size, const uint64_t pad_size, void (*fn)(SHA1_ARGS *, uint32_t)) { - unsigned lane, min_len, min_idx; + unsigned lane, min_idx; + uint64_t min_len; IMB_JOB *ret_job = NULL; unsigned i; @@ -124,6 +125,7 @@ submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, lane = state->unused_lanes & 15; state->unused_lanes >>= 4; + state->num_lanes_inuse++; state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; @@ -131,11 +133,11 @@ submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; - state->lens[lane] = (uint16_t)job->msg_len_to_hash_in_bytes; + state->lens[lane] = job->msg_len_to_hash_in_bytes; state->ldata[lane].extra_blocks = 1; /* enough jobs to start processing? */ - if (state->unused_lanes != 0xf) + if (state->num_lanes_inuse != max_jobs) return NULL; } else { /* @@ -186,7 +188,7 @@ submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, } else { state->args.data_ptr[i] = state->args.data_ptr[lane]; - state->lens[i] = UINT16_MAX; + state->lens[i] = UINT64_MAX; } } } @@ -217,6 +219,7 @@ submit_flush_job_sha_1(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job, /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; + state->num_lanes_inuse--; sha_mb_generic_write_digest(ret_job->auth_tag_output, state->args.digest, sha_type, 16, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index f949cafc..77b1ffe7 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -127,9 +127,9 @@ IMB_JOB *submit_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); uint32_t hec_32_sse_no_aesni(const uint8_t *in); @@ -511,6 +511,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -877,6 +878,7 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->lens[2] = 0; sha_1_ooo->lens[3] = 0; sha_1_ooo->unused_lanes = 0xF3210; + sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < SSE_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; } diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 8c2e6eff..0cbe92e3 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -193,9 +193,9 @@ IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, const uint64_t tag_sz); -IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, +IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); void aes_cmac_256_subkey_gen_sse(const void *key_exp, @@ -722,7 +722,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_HMAC_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, sizeof(aes128_ooo->lens)); @@ -1184,6 +1184,7 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->lens[2] = 0; sha_1_ooo->lens[3] = 0; sha_1_ooo->unused_lanes = 0xF3210; + sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < SSE_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; } diff --git a/lib/sse/sha_mb_sse.c b/lib/sse/sha_mb_sse.c index e7fce1c3..1dd8f467 100644 --- a/lib/sse/sha_mb_sse.c +++ b/lib/sse/sha_mb_sse.c @@ -27,8 +27,8 @@ #include "include/sha_mb_mgr.h" -IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); /* ========================================================================== */ /* @@ -36,7 +36,7 @@ IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job); */ IMB_DLL_LOCAL -IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, @@ -44,7 +44,7 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) } IMB_DLL_LOCAL -IMB_JOB *flush_job_sha1_sse(MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) +IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index c0f66ebf..33b83663 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -84,7 +84,7 @@ const struct { OOO_INFO(aes256_cmac_ooo, MB_MGR_CMAC_OOO), OOO_INFO(snow3g_uea2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), - OOO_INFO(sha_1_ooo, MB_MGR_HMAC_SHA_1_OOO) + OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO) }; /** -- GitLab From f1472cd604793694201f4a17a450ade9d553490c Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 19 May 2022 13:28:40 +0000 Subject: [PATCH 122/369] perf: [SHA1] add new SHA1 algorithm --- perf/ipsec_perf.c | 80 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index f7b2da43..677af60a 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -156,14 +156,19 @@ enum test_cipher_mode_e { /* This enum will be mostly translated to IMB_HASH_ALG * (make sure to update h_alg_names list in print_times function) */ enum test_hash_alg_e { - TEST_SHA1 = 1, + TEST_SHA1_HMAC = 1, + TEST_SHA_224_HMAC, + TEST_SHA_256_HMAC, + TEST_SHA_384_HMAC, + TEST_SHA_512_HMAC, + TEST_XCBC, + TEST_MD5, + TEST_HASH_CMAC, /* added here to be included in AES tests */ + TEST_SHA1, TEST_SHA_224, TEST_SHA_256, TEST_SHA_384, TEST_SHA_512, - TEST_XCBC, - TEST_MD5, - TEST_HASH_CMAC, /* added here to be included in AES tests */ TEST_HASH_CMAC_BITLEN, TEST_HASH_CMAC_256, TEST_NULL_HASH, @@ -479,29 +484,59 @@ const struct str_value_mapping hash_algo_str_map[] = { { .name = "sha1-hmac", .values.job_params = { - .hash_alg = TEST_SHA1 + .hash_alg = TEST_SHA1_HMAC } }, { .name = "sha224-hmac", .values.job_params = { - .hash_alg = TEST_SHA_224 + .hash_alg = TEST_SHA_224_HMAC } }, { .name = "sha256-hmac", .values.job_params = { - .hash_alg = TEST_SHA_256 + .hash_alg = TEST_SHA_256_HMAC } }, { .name = "sha384-hmac", .values.job_params = { - .hash_alg = TEST_SHA_384 + .hash_alg = TEST_SHA_384_HMAC } }, { .name = "sha512-hmac", + .values.job_params = { + .hash_alg = TEST_SHA_512_HMAC + } + }, + { + .name = "sha1", + .values.job_params = { + .hash_alg = TEST_SHA1 + } + }, + { + .name = "sha224", + .values.job_params = { + .hash_alg = TEST_SHA_224 + } + }, + { + .name = "sha256", + .values.job_params = { + .hash_alg = TEST_SHA_256 + } + }, + { + .name = "sha384", + .values.job_params = { + .hash_alg = TEST_SHA_384 + } + }, + { + .name = "sha512", .values.job_params = { .hash_alg = TEST_SHA_512 } @@ -804,11 +839,11 @@ enum cache_type_e { enum cache_type_e cache_type = WARM; const uint32_t auth_tag_length_bytes[] = { - 12, /* SHA1 */ - 14, /* SHA_224 */ - 16, /* SHA_256 */ - 24, /* SHA_384 */ - 32, /* SHA_512 */ + 12, /* SHA1_HMAC */ + 14, /* SHA_224_HMAC */ + 16, /* SHA_256_HMAC */ + 24, /* SHA_384_HMAC */ + 32, /* SHA_512_HMAC */ 12, /* AES_XCBC */ 12, /* MD5 */ 0, /* NULL_HASH */ @@ -1656,6 +1691,21 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.auth_tag_output = (uint8_t *) digest; switch (params->hash_alg) { + case TEST_SHA1: + job_template.hash_alg = IMB_AUTH_SHA_1; + break; + case TEST_SHA_224: + job_template.hash_alg = IMB_AUTH_SHA_224; + break; + case TEST_SHA_256: + job_template.hash_alg = IMB_AUTH_SHA_256; + break; + case TEST_SHA_384: + job_template.hash_alg = IMB_AUTH_SHA_384; + break; + case TEST_SHA_512: + job_template.hash_alg = IMB_AUTH_SHA_512; + break; case TEST_XCBC: job_template.u.XCBC._k1_expanded = k1_expanded; job_template.u.XCBC._k2 = k2; @@ -2503,6 +2553,8 @@ print_times(struct variant_s *variant_list, struct params_s *params, "ENCRYPT", "DECRYPT" }; const char *h_alg_names[TEST_NUM_HASH_TESTS - 1] = { + "SHA1_HMAC", "SHA_224_HMAC", "SHA_256_HMAC", + "SHA_384_HMAC", "SHA_512_HMAC", "SHA1", "SHA_224", "SHA_256", "SHA_384", "SHA_512", "XCBC", "MD5", "CMAC", "CMAC_BITLEN", "CMAC_256", "NULL_HASH", "CRC32", "GCM", "CUSTOM", "CCM", @@ -2543,7 +2595,7 @@ print_times(struct variant_s *variant_list, struct params_s *params, for (col = 0; col < total_variants; col++) { par = variant_list[col].params; - const uint8_t h_alg = par.hash_alg - TEST_SHA1; + const uint8_t h_alg = par.hash_alg - TEST_SHA1_HMAC; printf("\t%s", h_alg_names[h_alg]); } -- GitLab From d218e1712f24d2eaecd3478fb30c81aada769bdd Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 19 May 2022 13:35:26 +0000 Subject: [PATCH 123/369] lib: update release notes and readme --- README | 1 + README.md | 1 + ReleaseNotes.txt | 2 ++ 3 files changed, 4 insertions(+) diff --git a/README b/README index f737a0d5..96061bf7 100644 --- a/README +++ b/README @@ -135,6 +135,7 @@ implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | + | SHA1-96 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index d8a37612..88775b74 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ Table 2. List of supported integrity algorithms and their implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | +| SHA1-96 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index b9b2ae61..16b0ea04 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -10,6 +10,7 @@ Library - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added +- SHA1 multi-buffer implementation added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) @@ -19,6 +20,7 @@ Test Applications Performance Application - GHASH support added (through JOB and direct API) +- Support added for SHA1/224/256/384/512 Fixes - Fixed incorrect 8-buffer SNOW3G keystream generation -- GitLab From af6a3cbc54c3be017dc1b24f53533da0ee0b8db4 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 20 May 2022 12:37:38 +0000 Subject: [PATCH 124/369] sse: [SHA1] move stack alignment code --- lib/sse/sha1_x4_sse.asm | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/sse/sha1_x4_sse.asm b/lib/sse/sha1_x4_sse.asm index f8e400de..ac0b131b 100644 --- a/lib/sse/sha1_x4_sse.asm +++ b/lib/sse/sha1_x4_sse.asm @@ -294,10 +294,10 @@ mksection .text %endm %macro FUNC_SAVE 0 -%ifndef LINUX mov r11, rsp sub rsp, VARIABLE_OFFSET and rsp, ~15 ; align rsp to 16 bytes +%ifndef LINUX movdqa [rsp + 0*16], xmm6 movdqa [rsp + 1*16], xmm7 movdqa [rsp + 2*16], xmm8 @@ -308,8 +308,8 @@ mksection .text movdqa [rsp + 7*16], xmm13 movdqa [rsp + 8*16], xmm14 movdqa [rsp + 9*16], xmm15 - mov [rsp + GP_OFFSET], r11 ;; rsp pointer %endif + mov [rsp + GP_OFFSET], r11 ;; rsp pointer %endmacro %macro FUNC_RESTORE 0 @@ -338,9 +338,8 @@ mksection .text movdqa [rsp + 8*16], xmm5 movdqa [rsp + 9*16], xmm5 %endif - - mov rsp, [rsp + GP_OFFSET] ;; rsp pointer %endif + mov rsp, [rsp + GP_OFFSET] ;; rsp pointer %endmacro align 32 @@ -352,10 +351,8 @@ align 32 ; arg 2 : rdx : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_mult_sse,function,internal) sha1_mult_sse: - mov r11, rsp + sub rsp, FRAMESZ - and rsp, ~15 ; align rsp to 16 bytes - mov [rsp + 16*16], r11 ;; Initialize digests movdqa A, [arg1 + 0*SHA1_DIGEST_ROW_SIZE] @@ -485,7 +482,7 @@ lloop: clear_all_xmms_sse_asm %endif - mov rsp, [rsp + 16*16] + add rsp, FRAMESZ ret ; void call_sha1_mult_sse_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); -- GitLab From 646fcd45a1a275e5e9f00de18532a831d2e056b1 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 16 May 2022 14:42:55 +0000 Subject: [PATCH 125/369] lib: Add new cipher only burst API Initial support is for AES-CBC only --- lib/avx/mb_mgr_avx.c | 4 + lib/avx2/mb_mgr_avx2.c | 4 + lib/avx512/mb_mgr_avx512.c | 4 + lib/include/mb_mgr_code.h | 382 ++++++++++++++++++++++------- lib/include/noaesni.h | 12 + lib/intel-ipsec-mb.h | 70 +++++- lib/no-aesni/mb_mgr_sse_no_aesni.c | 5 + lib/sse/mb_mgr_sse.c | 4 + 8 files changed, 394 insertions(+), 91 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index b84ce7de..9b7efd16 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -270,6 +270,8 @@ void aes_cmac_256_subkey_gen_avx(const void *key_exp, #define GET_COMPLETED_JOB get_completed_job_avx #define SUBMIT_BURST submit_burst_avx #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx /* ====================================================================== */ @@ -928,6 +930,8 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->submit_job = submit_job_avx; state->submit_burst = submit_burst_avx; state->submit_burst_nocheck= submit_burst_nocheck_avx; + state->submit_cipher_burst = submit_cipher_burst_avx; + state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx; state->submit_job_nocheck = submit_job_nocheck_avx; state->get_completed_job = get_completed_job_avx; state->flush_job = flush_job_avx; diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index d122e5d0..fbbea8ff 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -273,6 +273,8 @@ IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); #define GET_COMPLETED_JOB get_completed_job_avx2 #define SUBMIT_BURST submit_burst_avx2 #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2 /* ====================================================================== */ @@ -911,6 +913,8 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_job = submit_job_avx2; state->submit_burst = submit_burst_avx2; state->submit_burst_nocheck= submit_burst_nocheck_avx2; + state->submit_cipher_burst = submit_cipher_burst_avx2; + state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx2; state->submit_job_nocheck = submit_job_nocheck_avx2; state->get_completed_job = get_completed_job_avx2; state->flush_job = flush_job_avx2; diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index ce8157cd..bca060f2 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -488,6 +488,8 @@ SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, #define GET_COMPLETED_JOB get_completed_job_avx512 #define SUBMIT_BURST submit_burst_avx512 #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512 /* ====================================================================== */ @@ -1925,6 +1927,8 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) state->submit_job = submit_job_avx512; state->submit_burst = submit_burst_avx512; state->submit_burst_nocheck= submit_burst_nocheck_avx512; + state->submit_cipher_burst = submit_cipher_burst_avx512; + state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx512; state->submit_job_nocheck = submit_job_nocheck_avx512; state->get_completed_job = get_completed_job_avx512; state->flush_job = flush_job_avx512; diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 424e8084..eec342f9 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -1276,7 +1276,6 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) } } - /* ========================================================================= */ /* Job submit & flush functions */ /* ========================================================================= */ @@ -1287,7 +1286,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #define MB_MAX_LEN16 ((1 << 16) - 2) __forceinline int -is_job_invalid(IMB_MGR *state, const IMB_JOB *job) +is_job_invalid(IMB_MGR *state, const IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, const IMB_HASH_ALG hash_alg, + const IMB_CIPHER_DIRECTION cipher_direction, + const IMB_KEY_SIZE_BYTES key_len_in_bytes) { const uint64_t auth_tag_len_fips[] = { 0, /* INVALID selection */ @@ -1393,13 +1395,13 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) * of XGEM header */ const uint64_t max_pon_len = (1 << 14) + 8; - if (job->cipher_direction != IMB_DIR_DECRYPT && - job->cipher_direction != IMB_DIR_ENCRYPT && - job->cipher_mode != IMB_CIPHER_NULL) { + if (cipher_direction != IMB_DIR_DECRYPT && + cipher_direction != IMB_DIR_ENCRYPT && + cipher_mode != IMB_CIPHER_NULL) { imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); return 1; } - switch (job->cipher_mode) { + switch (cipher_mode) { case IMB_CIPHER_CBC: case IMB_CIPHER_CBCS_1_9: if (job->src == NULL) { @@ -1414,19 +1416,19 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_IV); return 1; } - if (job->cipher_direction == IMB_DIR_ENCRYPT && + if (cipher_direction == IMB_DIR_ENCRYPT && job->enc_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(24) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1438,7 +1440,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); return 1; } - if (job->cipher_mode == IMB_CIPHER_CBCS_1_9) { + if (cipher_mode == IMB_CIPHER_CBCS_1_9) { if (job->msg_len_to_cipher_in_bytes > ((1ULL << (60)) - 1)) { imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); @@ -1449,7 +1451,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) IMB_ERR_JOB_NULL_NEXT_IV); return 1; } - } else if (job->cipher_direction == IMB_DIR_ENCRYPT && + } else if (cipher_direction == IMB_DIR_ENCRYPT && job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); return 1; @@ -1468,19 +1470,19 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_DST); return 1; } - if (job->cipher_direction == IMB_DIR_ENCRYPT && + if (cipher_direction == IMB_DIR_ENCRYPT && job->enc_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(24) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1516,16 +1518,16 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(24) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } - if ((job->cipher_mode == IMB_CIPHER_CNTR && + if ((cipher_mode == IMB_CIPHER_CNTR && job->iv_len_in_bytes != UINT64_C(16) && job->iv_len_in_bytes != UINT64_C(12)) || - (job->cipher_mode == IMB_CIPHER_CNTR_BITLEN && + (cipher_mode == IMB_CIPHER_CNTR_BITLEN && job->iv_len_in_bytes != UINT64_C(16))) { imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; @@ -1565,13 +1567,13 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if ((job->key_len_in_bytes != UINT64_C(16)) && - (job->key_len_in_bytes != UINT64_C(32))) { + if ((key_len_in_bytes != UINT64_C(16)) && + (key_len_in_bytes != UINT64_C(32))) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1603,19 +1605,19 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } /* Same key structure used for encrypt and decrypt */ - if (job->cipher_direction == IMB_DIR_ENCRYPT && + if (cipher_direction == IMB_DIR_ENCRYPT && job->enc_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(24) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1623,13 +1625,13 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; } - if (job->cipher_mode == IMB_CIPHER_GCM && - job->hash_alg != IMB_AUTH_AES_GMAC) { + if (cipher_mode == IMB_CIPHER_GCM && + hash_alg != IMB_AUTH_AES_GMAC) { imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; } - if (job->cipher_mode == IMB_CIPHER_GCM_SGL && - job->hash_alg != IMB_AUTH_GCM_SGL) { + if (cipher_mode == IMB_CIPHER_GCM_SGL && + hash_alg != IMB_AUTH_GCM_SGL) { imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; } @@ -1654,17 +1656,17 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_IV); return 1; } - if (job->cipher_direction == IMB_DIR_ENCRYPT && + if (cipher_direction == IMB_DIR_ENCRYPT && job->enc_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(8)) { + if (key_len_in_bytes != UINT64_C(8)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1695,17 +1697,17 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_IV); return 1; } - if (job->cipher_direction == IMB_DIR_ENCRYPT && + if (cipher_direction == IMB_DIR_ENCRYPT && job->enc_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->cipher_direction == IMB_DIR_DECRYPT && + if (cipher_direction == IMB_DIR_DECRYPT && job->dec_keys == NULL) { imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(8)) { + if (key_len_in_bytes != UINT64_C(8)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1744,8 +1746,8 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } /* currently only AES-CCM-128 and AES-CCM-256 supported */ - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1760,7 +1762,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; } - if (job->hash_alg != IMB_AUTH_AES_CCM) { + if (hash_alg != IMB_AUTH_AES_CCM) { imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; } @@ -1778,7 +1780,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_IV); return 1; } - if (job->key_len_in_bytes != UINT64_C(24)) { + if (key_len_in_bytes != UINT64_C(24)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1795,7 +1797,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; } - if (job->cipher_direction == IMB_DIR_ENCRYPT) { + if (cipher_direction == IMB_DIR_ENCRYPT) { const void * const *ks_ptr = (const void * const *)job->enc_keys; @@ -1851,7 +1853,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, EINVAL); return 1; } - if (job->hash_alg != IMB_AUTH_PON_CRC_BIP) { + if (hash_alg != IMB_AUTH_PON_CRC_BIP) { imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; } @@ -1875,7 +1877,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } - if (job->key_len_in_bytes != UINT64_C(16)) { + if (key_len_in_bytes != UINT64_C(16)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1930,8 +1932,8 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16) && - job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -1940,7 +1942,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); return 1; } - if (job->key_len_in_bytes == UINT64_C(16)) { + if (key_len_in_bytes == UINT64_C(16)) { if (job->iv_len_in_bytes != UINT64_C(16)) { imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; @@ -1970,7 +1972,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16)) { + if (key_len_in_bytes != UINT64_C(16)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -2001,7 +2003,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(16)) { + if (key_len_in_bytes != UINT64_C(16)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -2032,7 +2034,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -2065,7 +2067,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -2097,7 +2099,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); return 1; } - if (job->key_len_in_bytes != UINT64_C(32)) { + if (key_len_in_bytes != UINT64_C(32)) { imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); return 1; } @@ -2105,8 +2107,8 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_IV_LEN); return 1; } - if (job->cipher_mode == IMB_CIPHER_SNOW_V_AEAD && - job->hash_alg != IMB_AUTH_SNOW_V_AEAD) { + if (cipher_mode == IMB_CIPHER_SNOW_V_AEAD && + hash_alg != IMB_AUTH_SNOW_V_AEAD) { imb_set_errno(state, IMB_ERR_HASH_ALGO); return 1; } @@ -2116,7 +2118,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } - switch (job->hash_alg) { + switch (hash_alg) { case IMB_AUTH_HMAC_SHA_1: case IMB_AUTH_MD5: case IMB_AUTH_HMAC_SHA_224: @@ -2128,9 +2130,9 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg] && + auth_tag_len_ipsec[hash_alg] && job->auth_tag_output_len_in_bytes != - auth_tag_len_fips[job->hash_alg]) { + auth_tag_len_fips[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2158,9 +2160,9 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg] && + auth_tag_len_ipsec[hash_alg] && job->auth_tag_output_len_in_bytes != - auth_tag_len_fips[job->hash_alg]) { + auth_tag_len_fips[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2208,7 +2210,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2224,7 +2226,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); return 1; } - if (job->cipher_mode != IMB_CIPHER_GCM) { + if (cipher_mode != IMB_CIPHER_GCM) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2239,7 +2241,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) */ break; case IMB_AUTH_GCM_SGL: - if (job->cipher_mode != IMB_CIPHER_GCM_SGL) { + if (cipher_mode != IMB_CIPHER_GCM_SGL) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2280,7 +2282,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) } /* This GMAC mode is to be used as stand-alone, * not combined with GCM */ - if (job->cipher_mode == IMB_CIPHER_GCM) { + if (cipher_mode == IMB_CIPHER_GCM) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2352,7 +2354,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } - if (job->cipher_mode != IMB_CIPHER_CCM) { + if (cipher_mode != IMB_CIPHER_CCM) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2423,7 +2425,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) case IMB_AUTH_SHA_384: case IMB_AUTH_SHA_512: if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2460,7 +2462,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { /* 64-bits: * - BIP 32-bits * - CRC 32-bits @@ -2468,7 +2470,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } - if (job->cipher_mode != IMB_CIPHER_PON_AES_CNTR) { + if (cipher_mode != IMB_CIPHER_PON_AES_CNTR) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2496,7 +2498,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2552,7 +2554,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) * - encrypt chain order: hash, cipher * - decrypt chain order: cipher, hash */ - if (job->cipher_mode != IMB_CIPHER_DOCSIS_SEC_BPI) { + if (cipher_mode != IMB_CIPHER_DOCSIS_SEC_BPI) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2583,14 +2585,14 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { /* Ethernet FCS CRC is 32-bits */ imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } - if ((job->cipher_direction == IMB_DIR_ENCRYPT && + if ((cipher_direction == IMB_DIR_ENCRYPT && job->chain_order != IMB_ORDER_HASH_CIPHER) || - (job->cipher_direction == IMB_DIR_DECRYPT && + (cipher_direction == IMB_DIR_DECRYPT && job->chain_order != IMB_ORDER_CIPHER_HASH)) { imb_set_errno(state, IMB_ERR_JOB_CHAIN_ORDER); return 1; @@ -2615,7 +2617,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2645,7 +2647,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2668,7 +2670,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2682,7 +2684,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_DST); return 1; } - if (job->cipher_mode != IMB_CIPHER_CHACHA20_POLY1305) { + if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2696,7 +2698,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2710,7 +2712,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) imb_set_errno(state, IMB_ERR_JOB_NULL_DST); return 1; } - if (job->cipher_mode != IMB_CIPHER_CHACHA20_POLY1305_SGL) { + if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305_SGL) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2724,7 +2726,7 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } @@ -2744,11 +2746,11 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job) return 1; } if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[job->hash_alg]) { + auth_tag_len_ipsec[hash_alg]) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; } - if (job->cipher_mode != IMB_CIPHER_SNOW_V_AEAD) { + if (cipher_mode != IMB_CIPHER_SNOW_V_AEAD) { imb_set_errno(state, IMB_ERR_CIPH_MODE); return 1; } @@ -2858,7 +2860,10 @@ submit_job_and_check(IMB_MGR *state, const int run_check) job = JOBS(state, state->next_job); if (run_check) { - if (is_job_invalid(state, job)) { + if (is_job_invalid(state, job, + job->cipher_mode, job->hash_alg, + job->cipher_direction, + job->key_len_in_bytes)) { job->status = IMB_STATUS_INVALID_ARGS; } else { job->status = IMB_STATUS_BEING_PROCESSED; @@ -3075,7 +3080,10 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, } /* validate job */ - if (is_job_invalid(state, job)) { + if (is_job_invalid(state, job, + job->cipher_mode, job->hash_alg, + job->cipher_direction, + job->key_len_in_bytes)) { job->status = IMB_STATUS_INVALID_ARGS; return 0; } @@ -3160,6 +3168,206 @@ SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) return submit_burst_and_check(state, jobs, n_jobs, 0); } +__forceinline +uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + MB_MGR_AES_OOO *aes_ooo = NULL; + + static IMB_JOB * (*submit_fn) + (MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; + static IMB_JOB * (*flush_fn) (MB_MGR_AES_OOO *state) = NULL; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CBC, IMB_AUTH_NULL, + IMB_DIR_ENCRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == 16) { + aes_ooo = state->aes128_ooo; + submit_fn = SUBMIT_JOB_AES128_ENC; + flush_fn = FLUSH_JOB_AES128_ENC; + } else if (key_size == 24) { + aes_ooo = state->aes192_ooo; + submit_fn = SUBMIT_JOB_AES192_ENC; + flush_fn = FLUSH_JOB_AES192_ENC; + } else { /* assume 32 */ + aes_ooo = state->aes256_ooo; + submit_fn = SUBMIT_JOB_AES256_ENC; + flush_fn = FLUSH_JOB_AES256_ENC; + } + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = submit_fn(aes_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while((job = flush_fn(aes_ooo)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + return completed_jobs; +} + +__forceinline +uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + + static void (*submit_fn) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = NULL; + (void) state; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CBC, IMB_AUTH_NULL, + IMB_DIR_DECRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == 16) + submit_fn = AES_CBC_DEC_128; + else if (key_size == 24) + submit_fn = AES_CBC_DEC_192; + else /* assume 32 */ + submit_fn = AES_CBC_DEC_256; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + + return completed_jobs; +} + +__forceinline +uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + switch (cipher) { + case IMB_CIPHER_CBC: + if (dir == IMB_DIR_ENCRYPT) + return submit_aes_cbc_burst_enc(state, jobs, n_jobs, + key_size, run_check); + else + return submit_aes_cbc_burst_dec(state, jobs, n_jobs, + key_size, run_check); + default: + break; + } + + /* unsupported cipher mode */ + imb_set_errno(state, IMB_ERR_CIPH_MODE); + + return 0; +} + +uint32_t +SUBMIT_CIPHER_BURST(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } +#endif + + return submit_cipher_burst_and_check(state, jobs, n_jobs, + cipher, dir, key_size, 1); +} + +uint32_t +SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } +#endif + + return submit_cipher_burst_and_check(state, jobs, n_jobs, + cipher, dir, key_size, 0); +} #endif /* MB_MGR_CODE_H */ diff --git a/lib/include/noaesni.h b/lib/include/noaesni.h index 71a6f773..3d1d7cd2 100644 --- a/lib/include/noaesni.h +++ b/lib/include/noaesni.h @@ -46,6 +46,18 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT void aes_keyexp_128_sse_no_aesni(const void *key, void *enc_exp_keys, void *dec_exp_keys); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 13bab042..ffe044e9 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -761,10 +761,14 @@ typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *); typedef uint32_t (*queue_size_t)(struct IMB_MGR *); - typedef uint32_t (*submit_burst_t)(struct IMB_MGR *, - struct IMB_JOB *, uint32_t); - + struct IMB_JOB *, const uint32_t); +typedef uint32_t (*submit_cipher_burst_t)(struct IMB_MGR *, + struct IMB_JOB *, + const uint32_t, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); typedef void (*keyexp_t)(const void *, void *, void *); typedef void (*cmac_subkey_gen_t)(const void *, void *, void *); typedef void (*hash_one_block_t)(const void *, void *); @@ -1148,6 +1152,8 @@ typedef struct IMB_MGR { submit_burst_t submit_burst; submit_burst_t submit_burst_nocheck; + submit_cipher_burst_t submit_cipher_burst; + submit_cipher_burst_t submit_cipher_burst_nocheck; /* in-order scheduler fields */ int earliest_job; /**< byte offset, -1 if none */ @@ -1353,6 +1359,55 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); + +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); /** * @brief Automatically initialize most performant * Multi-buffer manager based on CPU features @@ -1395,7 +1450,14 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs))) #define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs) \ ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs))) - +#define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, \ + _dir, _key_size) \ + ((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \ + (_cipher), (_dir), (_key_size))) +#define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, \ + _dir, _key_size) \ + ((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\ + (_cipher), (_dir), (_key_size))) /* Key expansion and generation API's */ #define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \ ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key))) diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 77b1ffe7..fc73b03c 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -248,6 +248,8 @@ IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); #define GET_COMPLETED_JOB get_completed_job_sse_no_aesni #define SUBMIT_BURST submit_burst_sse_no_aesni #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_no_aesni +#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_no_aesni +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_no_aesni #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni @@ -920,6 +922,9 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) state->submit_job = submit_job_sse_no_aesni; state->submit_burst = submit_burst_sse_no_aesni; state->submit_burst_nocheck= submit_burst_nocheck_sse_no_aesni; + state->submit_cipher_burst = submit_cipher_burst_sse_no_aesni; + state->submit_cipher_burst_nocheck = + submit_cipher_burst_nocheck_sse_no_aesni; state->submit_job_nocheck = submit_job_nocheck_sse_no_aesni; state->get_completed_job = get_completed_job_sse_no_aesni; state->flush_job = flush_job_sse_no_aesni; diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 0cbe92e3..fcf692cc 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -316,6 +316,8 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define GET_COMPLETED_JOB get_completed_job_sse #define SUBMIT_BURST submit_burst_sse #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse +#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse @@ -1243,6 +1245,8 @@ init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) state->submit_job = submit_job_sse; state->submit_burst = submit_burst_sse; state->submit_burst_nocheck= submit_burst_nocheck_sse; + state->submit_cipher_burst = submit_cipher_burst_sse; + state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse; state->submit_job_nocheck = submit_job_nocheck_sse; state->get_completed_job = get_completed_job_sse; state->flush_job = flush_job_sse; -- GitLab From 0ecdfc85518009ccbf1bf134e8cb6db9110ba9b9 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 16 May 2022 14:46:32 +0000 Subject: [PATCH 126/369] test: [AES-CBC] Test cipher only burst API --- test/aes_test.c | 109 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 14 deletions(-) diff --git a/test/aes_test.c b/test/aes_test.c index fddbdd2a..8aa17004 100644 --- a/test/aes_test.c +++ b/test/aes_test.c @@ -36,6 +36,11 @@ #include "gcm_ctr_vectors_test.h" #include "utils.h" +typedef enum { + BURST_TYPE_GENERIC = 0, + BURST_TYPE_CIPHER, +} BURST_TYPE; + int aes_test(struct IMB_MGR *mb_mgr); struct aes_vector { @@ -1985,12 +1990,13 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, - const int num_jobs) + const int num_jobs, + const BURST_TYPE burst_type) { struct IMB_JOB *job, jobs[24]; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); - int i, jobs_rx = 0, ret = -1; + int i, completed_jobs, jobs_rx = 0, ret = -1; if (targets == NULL) goto end_alloc; @@ -2011,8 +2017,15 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, for (i = 0; i < num_jobs; i++) { job = &jobs[i]; - job->cipher_direction = dir; - job->chain_order = order; + + /* only set fields for generic burst API */ + if (burst_type == BURST_TYPE_GENERIC) { + job->cipher_direction = dir; + job->chain_order = order; + job->key_len_in_bytes = key_len; + job->cipher_mode = cipher; + job->hash_alg = IMB_AUTH_NULL; + } if (!in_place) { job->dst = targets[i] + sizeof(padding); job->src = in_text; @@ -2020,22 +2033,22 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, job->dst = targets[i] + sizeof(padding); job->src = targets[i] + sizeof(padding); } - job->cipher_mode = cipher; + job->enc_keys = enc_keys; job->dec_keys = dec_keys; - job->key_len_in_bytes = key_len; - job->iv = iv; job->iv_len_in_bytes = 16; job->cipher_start_src_offset_in_bytes = 0; job->msg_len_to_cipher_in_bytes = text_len; job->user_data = targets[i]; job->user_data2 = (void *)((uint64_t)i); - - job->hash_alg = IMB_AUTH_NULL; } - int completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + if (burst_type == BURST_TYPE_GENERIC) + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + else + completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, + cipher, dir, key_len); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -2144,7 +2157,8 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, cipher, 0, - vec_tab[vect].Klen, num_jobs)) { + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_GENERIC)) { printf("error #%d encrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2170,7 +2184,8 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, cipher, 0, - vec_tab[vect].Klen, num_jobs)) { + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_GENERIC)) { printf("error #%d decrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2196,7 +2211,8 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, cipher, 1, - vec_tab[vect].Klen, num_jobs)) { + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_GENERIC)) { printf("error #%d encrypt burst in-place\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2222,13 +2238,78 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, cipher, 1, - vec_tab[vect].Klen, num_jobs)) { + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_GENERIC)) { printf("error #%d decrypt burst in-place\n", vect + 1); test_suite_update(ctx, 0, 1); } else { test_suite_update(ctx, 1, 0); } + /** + * Test cipher only burst API + * Currently only AES-CBC supported + */ + if (cipher != IMB_CIPHER_CBC) + continue; + + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, + cipher, 0, + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_CIPHER)) { + printf("error #%d encrypt cipher burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, + cipher, 0, + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_CIPHER)) { + printf("error #%d decrypt cipher burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, + cipher, 1, + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_CIPHER)) { + printf("error #%d encrypt cipher burst " + "in-place\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, + cipher, 1, + vec_tab[vect].Klen, num_jobs, + BURST_TYPE_CIPHER)) { + printf("error #%d decrypt cipher " + "burst in-place\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } } printf("\n"); } -- GitLab From 12d521721cec3478b856215488e0593462047672 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 16 May 2022 15:04:06 +0000 Subject: [PATCH 127/369] lib: Update windows def file with new cipher only burst API --- lib/libIPSec_MB.def | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index 0a93f6e1..0de7330d 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -554,3 +554,13 @@ EXPORTS submit_burst_nocheck_avx @528 submit_burst_nocheck_avx2 @529 submit_burst_nocheck_avx512 @530 + submit_cipher_burst_sse @531 + submit_cipher_burst_sse_no_aesni @532 + submit_cipher_burst_avx @533 + submit_cipher_burst_avx2 @534 + submit_cipher_burst_avx512 @535 + submit_cipher_burst_nocheck_sse @536 + submit_cipher_burst_nocheck_sse_no_aesni @537 + submit_cipher_burst_nocheck_avx @538 + submit_cipher_burst_nocheck_avx2 @539 + submit_cipher_burst_nocheck_avx512 @540 -- GitLab From b733d3b9b8cc8115af5841d798249d006bab304c Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 18 May 2022 12:51:45 +0000 Subject: [PATCH 128/369] perf: Add cipher-only burst API support --- perf/ipsec_perf.c | 111 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 12 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 677af60a..b2ffd8c3 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -951,7 +951,18 @@ static uint32_t pb_mod = 0; static int silent_progress_bar = 0; static int plot_output_option = 0; -static uint32_t burst_api = 0; /* burst API enable/disable flag */ +/* API types */ +typedef enum { + TEST_API_JOB = 0, + TEST_API_BURST, + TEST_API_CIPHER_BURST, + TEST_API_NUMOF +} TEST_API; + +const char *str_api_list[TEST_API_NUMOF] = {"job", "burst", + "cipher-only burst"}; + +static TEST_API test_api = TEST_API_JOB; /* test job API by default */ static uint32_t burst_size = 0; /* num jobs to pass to burst API */ static volatile int timebox_on = 1; /* flag to stop the test loop */ @@ -1668,6 +1679,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, uint32_t aux; uint8_t gcm_key[32]; uint8_t next_iv[IMB_AES_BLOCK_SIZE]; + IMB_JOB jobs[MAX_BURST_SIZE]; memset(&job_template, 0, sizeof(IMB_JOB)); @@ -2002,8 +2014,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif time = __rdtscp(&aux); - if (burst_api) { - IMB_JOB jobs[MAX_BURST_SIZE]; + /* test burst api */ + if (test_api == TEST_API_BURST) { uint32_t num_jobs = num_iter; while (num_jobs && timebox_on) { @@ -2039,7 +2051,71 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } jobs_done = num_iter - num_jobs; - } else { + /* test cipher-only burst api */ + } else if (test_api == TEST_API_CIPHER_BURST) { + IMB_JOB *jt = &job_template; + uint32_t num_jobs = num_iter; + uint32_t list_idx; + + while (num_jobs && timebox_on) { + uint32_t n_jobs = + (num_jobs / burst_size) ? burst_size : num_jobs; + + /* set all job params */ + for (i = 0; i < n_jobs; i++) { + job = &jobs[i]; + + /* If IMIX testing is being done, set the buffer + * size to cipher going through the + * list of sizes precalculated */ + if (imix_list_count != 0) { + list_idx = i & (JOB_SIZE_IMIX_LIST - 1); + job->msg_len_to_cipher_in_bytes = + cipher_size_list[list_idx]; + } else + job->msg_len_to_cipher_in_bytes = + jt->msg_len_to_cipher_in_bytes; + + job->src = get_src_buffer(index, p_buffer); + job->dst = get_dst_buffer(index, p_buffer); + job->enc_keys = job->dec_keys = + (const uint32_t *) + get_key_pointer(index, p_keys); + job->cipher_start_src_offset_in_bytes = + jt->cipher_start_src_offset_in_bytes; + job->iv = jt->iv; + job->iv_len_in_bytes = jt->iv_len_in_bytes; + + index = get_next_index(index); + } + /* submit cipher-only burst */ +#ifdef DEBUG + const uint32_t completed_jobs = + IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, n_jobs, + jt->cipher_mode, + jt->cipher_direction, + jt->key_len_in_bytes); + + if (completed_jobs != n_jobs) { + const int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_cipher_burst error " + "%d : '%s'\n", err, + imb_get_strerror(err)); + } + } +#else + IMB_SUBMIT_CIPHER_BURST_NOCHECK(mb_mgr, jobs, n_jobs, + jt->cipher_mode, + jt->cipher_direction, + jt->key_len_in_bytes); +#endif + num_jobs -= n_jobs; + } + jobs_done = num_iter - num_jobs; + + } else { /* test job api */ for (i = 0; (i < num_iter) && timebox_on; i++) { job = IMB_GET_NEXT_JOB(mb_mgr); *job = job_template; @@ -2082,7 +2158,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, #endif } - } /* if burst api */ + } /* if test_api */ #ifndef _WIN32 if (use_unhalted_cycles) @@ -2878,6 +2954,7 @@ static void usage(void) "--no-time-box: disables 100ms watchdog timer on " "an algorithm@packet-size performance test\n" "--burst-api: use burst API for perf tests\n" + "--cipher-burst-api: use cipher-only burst API for perf tests\n" "--burst-size: number of jobs to submit per burst\n", MAX_NUM_THREADS + 1); } @@ -3451,7 +3528,9 @@ int main(int argc, char *argv[]) i = get_next_num_arg((const char * const *)argv, i, argc, &tag_size, sizeof(tag_size)); } else if (strcmp(argv[i], "--burst-api") == 0) { - burst_api = 1; + test_api = TEST_API_BURST; + } else if (strcmp(argv[i], "--cipher-burst-api") == 0) { + test_api = TEST_API_CIPHER_BURST; } else if (strcmp(argv[i], "--burst-size") == 0) { i = get_next_num_arg((const char * const *)argv, i, argc, &burst_size, @@ -3468,15 +3547,23 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - if (burst_size != 0 && burst_api == 0) { + if (burst_size != 0 && test_api == TEST_API_JOB) { fprintf(stderr, "--burst-size can only be used with " - "--burst-api\n"); + "--burst-api or --cipher-burst-api\n"); return EXIT_FAILURE; } - if (burst_api != 0 && burst_size == 0) + if (test_api != TEST_API_JOB && burst_size == 0) burst_size = DEFAULT_BURST_SIZE; + /* currently only AES-CBC supported by cipher-only burst API */ + if (test_api == TEST_API_CIPHER_BURST && + custom_job_params.cipher_mode != TEST_CBC) { + fprintf(stderr, "Unsupported cipher-only burst " + "API algorithm selected\n"); + return EXIT_FAILURE; + } + if (aead_algo_set == 0 && cipher_algo_set == 0 && hash_algo_set == 0) { fprintf(stderr, "No cipher, hash or " @@ -3634,14 +3721,14 @@ int main(int argc, char *argv[]) "Library version: %s\n", sha_size_incr, IMB_VERSION_STR, imb_get_version_str()); - fprintf(stderr, "API type: %s", burst_api ? "burst" : "job"); - if (burst_api) + fprintf(stderr, "API type: %s", str_api_list[test_api]); + if (test_api != TEST_API_JOB) fprintf(stderr, " (burst size = %u)\n", burst_size); else fprintf(stderr, "\n"); fprintf(stderr, "GCM API type: %s\n", - use_gcm_job_api ? (burst_api ? "burst" : "job") : "direct"); + use_gcm_job_api ? str_api_list[test_api] : "direct"); if (custom_job_params.cipher_mode == TEST_GCM) fprintf(stderr, "GCM AAD = %"PRIu64"\n", gcm_aad_size); -- GitLab From cc3b4d14c5cde96294b66ff469bdfdf43a3bf0ca Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 20:38:28 +0000 Subject: [PATCH 129/369] lib: Add hash-only burst API - Current API implementation is a placeholder - Support for specific hash algorithms will be added in future commits --- lib/avx/mb_mgr_avx.c | 4 +++ lib/avx2/mb_mgr_avx2.c | 4 +++ lib/avx512/mb_mgr_avx512.c | 4 +++ lib/include/ipsec_ooo_mgr.h | 32 +++++++++++++++++++ lib/include/mb_mgr_code.h | 51 ++++++++++++++++++++++++++++++ lib/include/noaesni.h | 8 +++++ lib/intel-ipsec-mb.h | 11 +++++++ lib/libIPSec_MB.def | 10 ++++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 5 +++ lib/sse/mb_mgr_sse.c | 4 +++ 10 files changed, 133 insertions(+) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 9b7efd16..23373612 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -272,6 +272,8 @@ void aes_cmac_256_subkey_gen_avx(const void *key_exp, #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx +#define SUBMIT_HASH_BURST submit_hash_burst_avx +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx /* ====================================================================== */ @@ -932,6 +934,8 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->submit_burst_nocheck= submit_burst_nocheck_avx; state->submit_cipher_burst = submit_cipher_burst_avx; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx; + state->submit_hash_burst = submit_hash_burst_avx; + state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx; state->submit_job_nocheck = submit_job_nocheck_avx; state->get_completed_job = get_completed_job_avx; state->flush_job = flush_job_avx; diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index fbbea8ff..b62368e3 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -275,6 +275,8 @@ IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2 #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2 #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2 +#define SUBMIT_HASH_BURST submit_hash_burst_avx2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2 /* ====================================================================== */ @@ -915,6 +917,8 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) state->submit_burst_nocheck= submit_burst_nocheck_avx2; state->submit_cipher_burst = submit_cipher_burst_avx2; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx2; + state->submit_hash_burst = submit_hash_burst_avx2; + state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx2; state->submit_job_nocheck = submit_job_nocheck_avx2; state->get_completed_job = get_completed_job_avx2; state->flush_job = flush_job_avx2; diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index bca060f2..0cd329ff 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -490,6 +490,8 @@ SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512 #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512 #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512 +#define SUBMIT_HASH_BURST submit_hash_burst_avx512 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512 /* ====================================================================== */ @@ -1929,6 +1931,8 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) state->submit_burst_nocheck= submit_burst_nocheck_avx512; state->submit_cipher_burst = submit_cipher_burst_avx512; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx512; + state->submit_hash_burst = submit_hash_burst_avx512; + state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx512; state->submit_job_nocheck = submit_job_nocheck_avx512; state->get_completed_job = get_completed_job_avx512; state->flush_job = flush_job_avx512; diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 129f607e..62426006 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -404,4 +404,36 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); #endif /* IMB_IPSEC_MB_INTERNAL_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index eec342f9..ba484025 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3370,4 +3370,55 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, cipher, dir, key_size, 0); } +__forceinline +uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash, + const int run_check) +{ + switch (hash) { + default: + break; + } + + /* unsupported hash alg */ + imb_set_errno(state, IMB_ERR_HASH_ALGO); + + return 0; +} + +uint32_t +SUBMIT_HASH_BURST(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } +#endif + return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 1); +} + +uint32_t +SUBMIT_HASH_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } +#endif + return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 0); +} + #endif /* MB_MGR_CODE_H */ diff --git a/lib/include/noaesni.h b/lib/include/noaesni.h index 3d1d7cd2..272b3a75 100644 --- a/lib/include/noaesni.h +++ b/lib/include/noaesni.h @@ -58,6 +58,14 @@ submit_cipher_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT void aes_keyexp_128_sse_no_aesni(const void *key, void *enc_exp_keys, void *dec_exp_keys); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index ffe044e9..779d63a9 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -769,6 +769,10 @@ typedef uint32_t (*submit_cipher_burst_t)(struct IMB_MGR *, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); +typedef uint32_t (*submit_hash_burst_t)(struct IMB_MGR *, + struct IMB_JOB *, + const uint32_t, + const IMB_HASH_ALG hash); typedef void (*keyexp_t)(const void *, void *, void *); typedef void (*cmac_subkey_gen_t)(const void *, void *, void *); typedef void (*hash_one_block_t)(const void *, void *); @@ -1154,6 +1158,8 @@ typedef struct IMB_MGR { submit_burst_t submit_burst_nocheck; submit_cipher_burst_t submit_cipher_burst; submit_cipher_burst_t submit_cipher_burst_nocheck; + submit_hash_burst_t submit_hash_burst; + submit_hash_burst_t submit_hash_burst_nocheck; /* in-order scheduler fields */ int earliest_job; /**< byte offset, -1 if none */ @@ -1458,6 +1464,11 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); _dir, _key_size) \ ((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\ (_cipher), (_dir), (_key_size))) +#define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash) \ + ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash))) +#define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash) \ + ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash))) + /* Key expansion and generation API's */ #define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \ ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key))) diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index 0de7330d..ead3896c 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -564,3 +564,13 @@ EXPORTS submit_cipher_burst_nocheck_avx @538 submit_cipher_burst_nocheck_avx2 @539 submit_cipher_burst_nocheck_avx512 @540 + submit_hash_burst_sse @541 + submit_hash_burst_sse_no_aesni @542 + submit_hash_burst_avx @543 + submit_hash_burst_avx2 @544 + submit_hash_burst_avx512 @545 + submit_hash_burst_nocheck_sse @546 + submit_hash_burst_nocheck_sse_no_aesni @547 + submit_hash_burst_nocheck_avx @548 + submit_hash_burst_nocheck_avx2 @549 + submit_hash_burst_nocheck_avx512 @550 diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index fc73b03c..df558dc3 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -250,6 +250,8 @@ IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_no_aesni #define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_no_aesni #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_no_aesni +#define SUBMIT_HASH_BURST submit_hash_burst_sse_no_aesni +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_no_aesni #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni @@ -925,6 +927,9 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) state->submit_cipher_burst = submit_cipher_burst_sse_no_aesni; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse_no_aesni; + state->submit_hash_burst = submit_hash_burst_sse_no_aesni; + state->submit_hash_burst_nocheck = + submit_hash_burst_nocheck_sse_no_aesni; state->submit_job_nocheck = submit_job_nocheck_sse_no_aesni; state->get_completed_job = get_completed_job_sse_no_aesni; state->flush_job = flush_job_sse_no_aesni; diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index fcf692cc..34c81e5c 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -318,6 +318,8 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse #define SUBMIT_CIPHER_BURST submit_cipher_burst_sse #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse +#define SUBMIT_HASH_BURST submit_hash_burst_sse +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse @@ -1247,6 +1249,8 @@ init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) state->submit_burst_nocheck= submit_burst_nocheck_sse; state->submit_cipher_burst = submit_cipher_burst_sse; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse; + state->submit_hash_burst = submit_hash_burst_sse; + state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_sse; state->submit_job_nocheck = submit_job_nocheck_sse; state->get_completed_job = get_completed_job_sse; state->flush_job = flush_job_sse; -- GitLab From f13258a72b9fea5e1468d2646e4f5ca96d00d30b Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 21:00:30 +0000 Subject: [PATCH 130/369] lib: Add HMAC-SHA1 support to hash-only burst API --- lib/include/mb_mgr_code.h | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index ba484025..faacea55 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3370,6 +3370,69 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, cipher, dir, key_size, 0); } +__forceinline +uint32_t submit_hmac_sha1_burst(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; + + IMB_JOB *(*submit_fn) + (MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) = SUBMIT_JOB_HMAC; + IMB_JOB *(*flush_fn) (MB_MGR_HMAC_SHA_1_OOO *state) = FLUSH_JOB_HMAC; + +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + submit_fn = SUBMIT_JOB_HMAC_NI; + flush_fn = FLUSH_JOB_HMAC_NI; + } +#endif + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_NULL, + IMB_AUTH_HMAC_SHA_1, + IMB_DIR_ENCRYPT, + job->key_len_in_bytes)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = submit_fn(hmac_sha_1_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = flush_fn(hmac_sha_1_ooo)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + return completed_jobs; +} + __forceinline uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -3377,6 +3440,8 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const int run_check) { switch (hash) { + case IMB_AUTH_HMAC_SHA_1: + return submit_hmac_sha1_burst(state, jobs, n_jobs, run_check); default: break; } -- GitLab From b06220ef38f8254005499ff4721d587f5215207f Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 21:02:01 +0000 Subject: [PATCH 131/369] test: [HMAC-SHA1] Extend test to cover hash-only burst API --- test/hmac_sha1_test.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c index 5e4f4b21..64c8e4d4 100644 --- a/test/hmac_sha1_test.c +++ b/test/hmac_sha1_test.c @@ -36,6 +36,11 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr); +typedef enum { + BURST_TYPE_GENERIC = 0, + BURST_TYPE_HASH, +} BURST_TYPE; + #define block_size 64 #define digest_size 20 #define digest96_size 12 @@ -482,7 +487,7 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, static int test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, const struct hmac_sha1_rfc2202_vector *vec, - const uint32_t num_jobs) + const uint32_t num_jobs, BURST_TYPE burst_type) { struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; uint8_t padding[16]; @@ -493,7 +498,7 @@ test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, uint8_t buf[block_size]; DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); - uint32_t key_len = 0; + uint32_t completed_jobs = 0, key_len = 0; if (auths == NULL) { fprintf(stderr, "Can't allocate buffer memory\n"); @@ -563,7 +568,11 @@ test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, } - uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + if (burst_type == BURST_TYPE_GENERIC) + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + else + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + IMB_AUTH_HMAC_SHA_1); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -642,12 +651,19 @@ test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ts, 1, 0); } if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], - num_jobs)) { + num_jobs, BURST_TYPE_GENERIC)) { printf("error #%d - burst API\n", vect); test_suite_update(ts, 0, 1); } else { test_suite_update(ts, 1, 0); } + if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], + num_jobs, BURST_TYPE_HASH)) { + printf("error #%d - hash-only burst API\n", vect); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } } printf("\n"); } -- GitLab From efe5ac5be60f6b757fea2ff2c9bbbab20f494577 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 25 May 2022 10:27:02 +0000 Subject: [PATCH 132/369] perf: Add support for hash-only burst API - Currently only HMAC-SHA1 supported --- perf/ipsec_perf.c | 79 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index b2ffd8c3..7d1e9f06 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -956,11 +956,13 @@ typedef enum { TEST_API_JOB = 0, TEST_API_BURST, TEST_API_CIPHER_BURST, + TEST_API_HASH_BURST, TEST_API_NUMOF } TEST_API; const char *str_api_list[TEST_API_NUMOF] = {"job", "burst", - "cipher-only burst"}; + "cipher-only burst", + "hash-only burst"}; static TEST_API test_api = TEST_API_JOB; /* test job API by default */ static uint32_t burst_size = 0; /* num jobs to pass to burst API */ @@ -2115,6 +2117,67 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } jobs_done = num_iter - num_jobs; + /* test hash-only burst api */ + } else if (test_api == TEST_API_HASH_BURST) { + IMB_JOB *jt = &job_template; + uint32_t num_jobs = num_iter; + uint32_t list_idx; + + while (num_jobs && timebox_on) { + uint32_t n_jobs = + (num_jobs / burst_size) ? burst_size : num_jobs; + + /* set all job params */ + for (i = 0; i < n_jobs; i++) { + job = &jobs[i]; + + /* If IMIX testing is being done, set the buffer + * size to cipher going through the + * list of sizes precalculated */ + if (imix_list_count != 0) { + list_idx = i & (JOB_SIZE_IMIX_LIST - 1); + job->msg_len_to_hash_in_bytes = + hash_size_list[list_idx]; + } else + job->msg_len_to_hash_in_bytes = + jt->msg_len_to_hash_in_bytes; + + job->src = get_src_buffer(index, p_buffer); + job->hash_start_src_offset_in_bytes = + jt->hash_start_src_offset_in_bytes; + job->auth_tag_output_len_in_bytes = + jt->auth_tag_output_len_in_bytes; + job->u.HMAC._hashed_auth_key_xor_ipad = + jt->u.HMAC._hashed_auth_key_xor_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = + jt->u.HMAC._hashed_auth_key_xor_opad; + job->auth_tag_output = jt->auth_tag_output; + + index = get_next_index(index); + } + /* submit hash-only burst */ +#ifdef DEBUG + const uint32_t completed_jobs = + IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, n_jobs, + jt->hash_alg); + + if (completed_jobs != n_jobs) { + const int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_hash_burst error " + "%d : '%s'\n", err, + imb_get_strerror(err)); + } + } +#else + IMB_SUBMIT_HASH_BURST_NOCHECK(mb_mgr, jobs, n_jobs, + jt->hash_alg); +#endif + num_jobs -= n_jobs; + } + jobs_done = num_iter - num_jobs; + } else { /* test job api */ for (i = 0; (i < num_iter) && timebox_on; i++) { job = IMB_GET_NEXT_JOB(mb_mgr); @@ -2955,6 +3018,7 @@ static void usage(void) "an algorithm@packet-size performance test\n" "--burst-api: use burst API for perf tests\n" "--cipher-burst-api: use cipher-only burst API for perf tests\n" + "--hash-burst-api: use hash-only burst API for perf tests\n" "--burst-size: number of jobs to submit per burst\n", MAX_NUM_THREADS + 1); } @@ -3531,6 +3595,8 @@ int main(int argc, char *argv[]) test_api = TEST_API_BURST; } else if (strcmp(argv[i], "--cipher-burst-api") == 0) { test_api = TEST_API_CIPHER_BURST; + } else if (strcmp(argv[i], "--hash-burst-api") == 0) { + test_api = TEST_API_HASH_BURST; } else if (strcmp(argv[i], "--burst-size") == 0) { i = get_next_num_arg((const char * const *)argv, i, argc, &burst_size, @@ -3549,7 +3615,8 @@ int main(int argc, char *argv[]) if (burst_size != 0 && test_api == TEST_API_JOB) { fprintf(stderr, "--burst-size can only be used with " - "--burst-api or --cipher-burst-api\n"); + "--burst-api, --cipher-burst-api or " + "--hash-burst-api options\n"); return EXIT_FAILURE; } @@ -3564,6 +3631,14 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } + /* currently only HMAC-SHA1 supported by hash-only burst API */ + if (test_api == TEST_API_HASH_BURST && + custom_job_params.hash_alg != TEST_SHA1_HMAC) { + fprintf(stderr, + "Unsupported hash-only burst API algorithm selected\n"); + return EXIT_FAILURE; + } + if (aead_algo_set == 0 && cipher_algo_set == 0 && hash_algo_set == 0) { fprintf(stderr, "No cipher, hash or " -- GitLab From 3f01ad168cfa57b0a3cd58a3448267ec5d562ce0 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 25 May 2022 15:46:53 +0000 Subject: [PATCH 133/369] lib: Move burst API prototypes to internal header --- lib/include/ipsec_ooo_mgr.h | 67 ++++++++++++++++++++++++++++++++++++ lib/include/mb_mgr_code.h | 16 --------- lib/intel-ipsec-mb.h | 68 ------------------------------------- 3 files changed, 67 insertions(+), 84 deletions(-) diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 62426006..a12958ea 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -404,6 +404,73 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_EXPORT uint32_t +submit_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); + +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); + +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_hash_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index faacea55..c231d4d9 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3135,10 +3135,6 @@ SUBMIT_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) imb_set_errno(state, 0); #ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } if (jobs == NULL) { imb_set_errno(NULL, IMB_ERR_NULL_JOB); return 0; @@ -3155,10 +3151,6 @@ SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) imb_set_errno(state, 0); #ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } if (jobs == NULL) { imb_set_errno(NULL, IMB_ERR_NULL_JOB); return 0; @@ -3331,10 +3323,6 @@ SUBMIT_CIPHER_BURST(IMB_MGR *state, IMB_JOB *jobs, imb_set_errno(state, 0); #ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } if (jobs == NULL) { imb_set_errno(NULL, IMB_ERR_NULL_JOB); return 0; @@ -3356,10 +3344,6 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, imb_set_errno(state, 0); #ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } if (jobs == NULL) { imb_set_errno(NULL, IMB_ERR_NULL_JOB); return 0; diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 779d63a9..ff44e7ab 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1346,74 +1346,6 @@ IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state); -IMB_DLL_EXPORT uint32_t -submit_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); - -IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs); - -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); - -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); /** * @brief Automatically initialize most performant * Multi-buffer manager based on CPU features -- GitLab From 730aa026a551bb788a145fcc00e06161105b2fee Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 25 May 2022 15:54:17 +0000 Subject: [PATCH 134/369] test: [API] Remove NULL MB_MGR test Arch specific API required for test which is no longer accessible --- test/api_test.c | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/test/api_test.c b/test/api_test.c index 4599117c..7effce3e 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -699,36 +699,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) printf("BURST API behavior test:\n"); - /* ======== test 1 : NULL MB_MGR */ - - /** - * Using sse API directly since IMB_SUBMIT_BURST - * macro cannot be used to test for NULL MB_MGR. - */ - - /* test only when SAFE_PARAM is enabled */ - if ((mb_mgr->features & IMB_FEATURE_SAFE_PARAM) != 0) { - - completed_jobs = submit_burst_sse(NULL, jobs, n_jobs); - if (completed_jobs != 0) { - printf("%s: test %d, unexpected number of " - "completed jobs: %u\n", __func__, - TEST_INVALID_BURST, completed_jobs); - return 1; - } - printf("."); - - err = imb_get_errno(mb_mgr); - if (err != IMB_ERR_NULL_MBMGR) { - printf("%s: test %d, unexpected error: %s\n", - __func__, TEST_INVALID_BURST, - imb_get_strerror(err)); - return 1; - } - printf("."); - } - - /* ======== test 2 : NULL jobs array */ + /* ======== test 1 : NULL jobs array */ completed_jobs = IMB_SUBMIT_BURST(mb_mgr, job, n_jobs); if (completed_jobs != 0) { @@ -746,7 +717,7 @@ test_burst_api(struct IMB_MGR *mb_mgr) } printf("."); - /* ======== test 3 : invalid job */ + /* ======== test 2 : invalid job */ /* fill in valid jobs */ for (i = 0; i < n_jobs; i++) { -- GitLab From 452746bd6f03fa225365359849333813ca01f868 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 10:45:04 +0000 Subject: [PATCH 135/369] lib: Add AES-CTR support to cipher-only burst API --- lib/include/mb_mgr_code.h | 83 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index c231d4d9..2ab31497 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3286,6 +3286,86 @@ uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, return completed_jobs; } +__forceinline +uint32_t submit_aes_ctr_burst(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + if (job == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CNTR, IMB_AUTH_NULL, + IMB_DIR_ENCRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + +#ifdef AVX512 + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + static void (*submit_fn_vaes) (IMB_JOB *job) = NULL; + + if (key_size == 16) + submit_fn_vaes = aes_cntr_128_submit_vaes_avx512; + else if (key_size == 24) + submit_fn_vaes = aes_cntr_192_submit_vaes_avx512; + else /* assume 32 */ + submit_fn_vaes = aes_cntr_256_submit_vaes_avx512; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn_vaes(job); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } else { +#endif + static void (*submit_fn) (const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bytes, + uint64_t iv_len_bytes) = NULL; + + if (key_size == 16) + submit_fn = AES_CNTR_128; + else if (key_size == 24) + submit_fn = AES_CNTR_192; + else /* assume 32 */ + submit_fn = AES_CNTR_256; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } +#ifdef AVX512 + } +#endif + return completed_jobs; +} + __forceinline uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -3302,6 +3382,9 @@ uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, else return submit_aes_cbc_burst_dec(state, jobs, n_jobs, key_size, run_check); + case IMB_CIPHER_CNTR: + return submit_aes_ctr_burst(state, jobs, n_jobs, + key_size, run_check); default: break; } -- GitLab From a09e73a94a4a1a1a2d190ec1409b7a07f2420276 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 10:45:49 +0000 Subject: [PATCH 136/369] test: [AES-CTR] Extend test to cover burst API --- test/ctr_test.c | 303 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 301 insertions(+), 2 deletions(-) diff --git a/test/ctr_test.c b/test/ctr_test.c index 5eb94860..b1f2077a 100644 --- a/test/ctr_test.c +++ b/test/ctr_test.c @@ -34,6 +34,13 @@ #include "gcm_ctr_vectors_test.h" #include "utils.h" +typedef enum { + BURST_TYPE_GENERIC = 0, + BURST_TYPE_CIPHER, +} BURST_TYPE; + +#define MAX_CTR_JOBS 32 + #define BYTE_ROUND_UP(x) ((x + 7) / 8) /* * Test Vector from @@ -1350,8 +1357,8 @@ test_ctr(struct IMB_MGR *mb_mgr, const uint8_t *in_text, const uint8_t *out_text, unsigned text_len, - int dir, - int order, + const IMB_CIPHER_DIRECTION dir, + const IMB_CHAIN_ORDER order, const IMB_CIPHER_MODE alg) { uint32_t text_byte_len; @@ -1445,6 +1452,137 @@ test_ctr(struct IMB_MGR *mb_mgr, return ret; } +static int +test_ctr_burst(struct IMB_MGR *mb_mgr, + const void *expkey, + unsigned key_len, + const void *iv, + unsigned iv_len, + const uint8_t *in_text, + const uint8_t *out_text, + unsigned text_len, + const IMB_CIPHER_DIRECTION dir, + const IMB_CHAIN_ORDER order, + const IMB_CIPHER_MODE alg, + const uint32_t num_jobs, + const BURST_TYPE burst_type) +{ + uint32_t text_byte_len, i, completed_jobs, jobs_rx = 0; + struct IMB_JOB *job, jobs[MAX_CTR_JOBS]; + uint8_t padding[16]; + uint8_t **targets = malloc(num_jobs * sizeof(void *)); + int ret = -1; + + if (targets == NULL) + goto end_alloc; + + /* Get number of bytes (in case algo is CNTR_BITLEN) */ + if (alg == IMB_CIPHER_CNTR) + text_byte_len = text_len; + else + text_byte_len = BYTE_ROUND_UP(text_len); + + memset(targets, 0, num_jobs * sizeof(void *)); + memset(padding, -1, sizeof(padding)); + + for (i = 0; i < num_jobs; i++) { + targets[i] = malloc(text_byte_len + (sizeof(padding) * 2)); + if (targets[i] == NULL) + goto end_alloc; + memset(targets[i], -1, text_byte_len + (sizeof(padding) * 2)); + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->cipher_direction = dir; + job->chain_order = order; + job->dst = targets[i] + sizeof(padding); + job->src = in_text; + job->cipher_mode = alg; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->key_len_in_bytes = key_len; + job->iv = iv; + job->iv_len_in_bytes = iv_len; + job->cipher_start_src_offset_in_bytes = 0; + if (alg == IMB_CIPHER_CNTR) + job->msg_len_to_cipher_in_bytes = text_byte_len; + else + job->msg_len_to_cipher_in_bits = text_len; + job->hash_alg = IMB_AUTH_NULL; + job->user_data = targets[i]; + job->user_data2 = (void *)((uint64_t)i); + } + + + if (burst_type == BURST_TYPE_GENERIC) + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + else + completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, + alg, dir, key_len); + + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (memcmp(out_text, targets[i] + sizeof(padding), + text_byte_len)) { + printf("mismatched\n"); + hexdump(stderr, "Target", targets[i] + sizeof(padding), + text_byte_len); + hexdump(stderr, "Expected", out_text, text_byte_len); + goto end; + } + if (memcmp(padding, targets[i], sizeof(padding))) { + printf("overwrite head\n"); + hexdump(stderr, "Target", targets[i], text_byte_len + + (sizeof(padding) * 2)); + goto end; + } + if (memcmp(padding, targets[i] + sizeof(padding) + + text_byte_len, sizeof(padding))) { + printf("overwrite tail\n"); + hexdump(stderr, "Target", targets[i], text_byte_len + + (sizeof(padding) * 2)); + goto end; + } + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + end: + + end_alloc: + if (targets != NULL) { + for (i = 0; i < num_jobs; i++) + free(targets[i]); + free(targets); + } + + return ret; +} + static void test_ctr_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, @@ -1572,9 +1710,160 @@ test_ctr_vectors(struct IMB_MGR *mb_mgr, printf("\n"); } +static void +test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx128, + struct test_suite_context *ctx192, + struct test_suite_context *ctx256, + const struct gcm_ctr_vector *vectors, + const uint32_t vectors_cnt, const IMB_CIPHER_MODE alg, + const uint32_t num_jobs) +{ + uint32_t vect, burst_type; + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + + printf("AES-CTR standard test vectors - Burst API:\n"); + for (vect = 0; vect < vectors_cnt; vect++) { + struct test_suite_context *ctx; +#ifdef DEBUG + if (alg == IMB_CIPHER_CNTR) + printf("Standard vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d (burst)\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); + else + printf("Bit vector %u/%u Keylen:%d " + "IVlen:%d PTLen:%d (burst)\n", + vect, vectors_cnt - 1, + (int) vectors[vect].Klen, + (int) vectors[vect].IVlen, + (int) vectors[vect].Plen); +#else + printf("."); +#endif + + switch (vectors[vect].Klen) { + case IMB_KEY_128_BYTES: + IMB_AES_KEYEXP_128(mb_mgr, vectors[vect].K, + expkey, dust); + ctx = ctx128; + break; + case IMB_KEY_192_BYTES: + IMB_AES_KEYEXP_192(mb_mgr, vectors[vect].K, + expkey, dust); + ctx = ctx192; + break; + case IMB_KEY_256_BYTES: + IMB_AES_KEYEXP_256(mb_mgr, vectors[vect].K, + expkey, dust); + ctx = ctx256; + break; + default: + return; + } + + for (burst_type = BURST_TYPE_GENERIC; + burst_type <= BURST_TYPE_CIPHER; burst_type++) { + const char *bt = (burst_type == BURST_TYPE_CIPHER) ? + "cipher-only burst" : "burst"; + + /* skip bitlen cipher-only burst api tests */ + if (alg == IMB_CIPHER_CNTR_BITLEN && + burst_type == BURST_TYPE_CIPHER) + continue; + + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].P, vectors[vect].C, + (unsigned) vectors[vect].Plen, + IMB_DIR_ENCRYPT, + IMB_ORDER_CIPHER_HASH, alg, + num_jobs, burst_type)) { + printf("error #%u encrypt %s\n", vect + 1, bt); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].C, vectors[vect].P, + (unsigned) vectors[vect].Plen, + IMB_DIR_DECRYPT, + IMB_ORDER_HASH_CIPHER, alg, + num_jobs, burst_type)) { + printf("error #%u decrypt %s\n", vect + 1, bt); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (vectors[vect].IVlen == 12 && + alg == IMB_CIPHER_CNTR) { + /* IV in the table didn't + * include block counter (12 bytes). + * Let's encrypt & decrypt the same but + * with 16 byte IV that includes block counter. + */ + const unsigned new_iv_len = 16; + const unsigned orig_iv_len = 12; + uint8_t local_iv[16]; + + memcpy(local_iv, vectors[vect].IV, orig_iv_len); + /* 32-bit 0x1 in BE == 0x01000000 in LE */ + local_iv[12] = 0x00; + local_iv[13] = 0x00; + local_iv[14] = 0x00; + local_iv[15] = 0x01; + + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + local_iv, new_iv_len, + vectors[vect].P, + vectors[vect].C, (unsigned) + vectors[vect].Plen, + IMB_DIR_ENCRYPT, + IMB_ORDER_CIPHER_HASH, + alg, num_jobs, burst_type)) { + printf("error #%u encrypt %s\n", + vect + 1, bt); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + local_iv, new_iv_len, + vectors[vect].C, + vectors[vect].P, (unsigned) + vectors[vect].Plen, + IMB_DIR_DECRYPT, + IMB_ORDER_HASH_CIPHER, + alg, num_jobs, burst_type)) { + printf("error #%u decrypt %s\n", + vect + 1, bt); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } + } + } + printf("\n"); +} + int ctr_test(struct IMB_MGR *mb_mgr) { + uint32_t i; int errors = 0; struct test_suite_context ctx128; struct test_suite_context ctx192; @@ -1590,6 +1879,11 @@ ctr_test(struct IMB_MGR *mb_mgr) &ctx128, &ctx192, &ctx256, ctr_vectors, ctr_vec_cnt, IMB_CIPHER_CNTR); + for (i = 1; i <= MAX_CTR_JOBS; i++) + test_ctr_vectors_burst(mb_mgr, + &ctx128, &ctx192, &ctx256, + ctr_vectors, ctr_vec_cnt, + IMB_CIPHER_CNTR, i); errors += test_suite_end(&ctx128); errors += test_suite_end(&ctx192); errors += test_suite_end(&ctx256); @@ -1601,6 +1895,11 @@ ctr_test(struct IMB_MGR *mb_mgr) test_ctr_vectors(mb_mgr, &ctx128, &ctx192, &ctx256, ctr_bit_vectors, ctr_bit_vec_cnt, IMB_CIPHER_CNTR_BITLEN); + for (i = 1; i <= MAX_CTR_JOBS; i++) + test_ctr_vectors_burst(mb_mgr, + &ctx128, &ctx192, &ctx256, + ctr_bit_vectors, ctr_bit_vec_cnt, + IMB_CIPHER_CNTR_BITLEN, i); errors += test_suite_end(&ctx128); errors += test_suite_end(&ctx192); errors += test_suite_end(&ctx256); -- GitLab From 172099319a034327df39e57e9bd52cb11bc7177a Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 24 May 2022 10:46:50 +0000 Subject: [PATCH 137/369] perf: Allow AES-CTR when testing cipher-only burst API --- perf/ipsec_perf.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 7d1e9f06..191987f3 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -3623,9 +3623,10 @@ int main(int argc, char *argv[]) if (test_api != TEST_API_JOB && burst_size == 0) burst_size = DEFAULT_BURST_SIZE; - /* currently only AES-CBC supported by cipher-only burst API */ + /* currently only AES-CBC & CTR supported by cipher-only burst API */ if (test_api == TEST_API_CIPHER_BURST && - custom_job_params.cipher_mode != TEST_CBC) { + (custom_job_params.cipher_mode != TEST_CBC && + custom_job_params.cipher_mode != TEST_CNTR)) { fprintf(stderr, "Unsupported cipher-only burst " "API algorithm selected\n"); return EXIT_FAILURE; -- GitLab From 6f0e6baf789b450eb26b4e31f4f6182ca6cd1e30 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 27 May 2022 09:09:41 +0000 Subject: [PATCH 138/369] lib: Extend hash-only burst API to support all HMAC-SHAx algos --- lib/include/mb_mgr_code.h | 137 ++++++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 19 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 2ab31497..45e612ae 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -3438,24 +3438,17 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, } __forceinline -uint32_t submit_hmac_sha1_burst(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) +uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check, + const IMB_HASH_ALG hash_alg, + void *ooo_mgr, + IMB_JOB *(*submit_fn)(void *, IMB_JOB *), + IMB_JOB *(*flush_fn)(void *)) { uint32_t i, completed_jobs = 0; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - - IMB_JOB *(*submit_fn) - (MB_MGR_HMAC_SHA_1_OOO *state, IMB_JOB *job) = SUBMIT_JOB_HMAC; - IMB_JOB *(*flush_fn) (MB_MGR_HMAC_SHA_1_OOO *state) = FLUSH_JOB_HMAC; -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - submit_fn = SUBMIT_JOB_HMAC_NI; - flush_fn = FLUSH_JOB_HMAC_NI; - } -#endif if (run_check) { /* validate jobs */ for (i = 0; i < n_jobs; i++) { @@ -3469,7 +3462,7 @@ uint32_t submit_hmac_sha1_burst(IMB_MGR *state, /* validate job */ if (is_job_invalid(state, job, IMB_CIPHER_NULL, - IMB_AUTH_HMAC_SHA_1, + hash_alg, IMB_DIR_ENCRYPT, job->key_len_in_bytes)) { job->status = IMB_STATUS_INVALID_ARGS; @@ -3481,7 +3474,7 @@ uint32_t submit_hmac_sha1_burst(IMB_MGR *state, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - job = submit_fn(hmac_sha_1_ooo, job); + job = submit_fn(ooo_mgr, job); if (job != NULL) { job->status = IMB_STATUS_COMPLETED; completed_jobs++; @@ -3491,7 +3484,7 @@ uint32_t submit_hmac_sha1_burst(IMB_MGR *state, if (completed_jobs != n_jobs) { IMB_JOB *job = NULL; - while ((job = flush_fn(hmac_sha_1_ooo)) != NULL) { + while ((job = flush_fn(ooo_mgr)) != NULL) { job->status = IMB_STATUS_COMPLETED; completed_jobs++; } @@ -3500,6 +3493,99 @@ uint32_t submit_hmac_sha1_burst(IMB_MGR *state, return completed_jobs; } +__forceinline +uint32_t submit_burst_hmac_sha_1(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_1, + (void *)state->hmac_sha_1_ooo, + (void *)SUBMIT_JOB_HMAC_NI, + (void *)FLUSH_JOB_HMAC_NI); + } +#endif + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_1, + (void *)state->hmac_sha_1_ooo, + (void *)SUBMIT_JOB_HMAC, + (void *)FLUSH_JOB_HMAC); +} + +__forceinline +uint32_t submit_burst_hmac_sha_224(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_224, + (void *)state->hmac_sha_224_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_224_NI, + (void *)FLUSH_JOB_HMAC_SHA_224_NI); + } +#endif + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_224, + (void *)state->hmac_sha_224_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_224, + (void *)FLUSH_JOB_HMAC_SHA_224); + +} + +__forceinline +uint32_t submit_burst_hmac_sha_256(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_256, + (void *)state->hmac_sha_256_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_256_NI, + (void *)FLUSH_JOB_HMAC_SHA_256_NI); + } +#endif + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_256, + (void *)state->hmac_sha_256_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_256, + (void *)FLUSH_JOB_HMAC_SHA_256); +} + +__forceinline +uint32_t submit_burst_hmac_sha_384(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_384, + (void *)state->hmac_sha_384_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_384, + (void *)FLUSH_JOB_HMAC_SHA_384); +} + +__forceinline +uint32_t submit_burst_hmac_sha_512(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_512, + (void *)state->hmac_sha_512_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_512, + (void *)FLUSH_JOB_HMAC_SHA_512); +} + __forceinline uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -3508,7 +3594,20 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, { switch (hash) { case IMB_AUTH_HMAC_SHA_1: - return submit_hmac_sha1_burst(state, jobs, n_jobs, run_check); + return submit_burst_hmac_sha_1(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_224: + return submit_burst_hmac_sha_224(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_256: + return submit_burst_hmac_sha_256(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_384: + return submit_burst_hmac_sha_384(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_512: + return submit_burst_hmac_sha_512(state, jobs, + n_jobs, run_check); default: break; } -- GitLab From b7eaf847c300fccc29aa043e4f08a2321a69b971 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 27 May 2022 09:11:15 +0000 Subject: [PATCH 139/369] test: [HMAC-SHAx] Add hash-only burst API tests for HMAC-SHAx Tests: - HMAC-SHA224 - HMAC-SHA256 - HMAC-SHA385 - HMAC-SHA512 --- test/hmac_sha256_sha512_test.c | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index 3403598d..1e7b25ed 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -38,6 +38,11 @@ int hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr); +typedef enum { + BURST_TYPE_GENERIC = 0, + BURST_TYPE_HASH, +} BURST_TYPE; + /* * Test vectors from https://tools.ietf.org/html/rfc4231 */ @@ -1044,12 +1049,13 @@ static int test_hmac_shax_burst(struct IMB_MGR *mb_mgr, const struct hmac_rfc4231_vector *vec, const uint32_t num_jobs, - const int sha_type) + const int sha_type, + const BURST_TYPE burst_type) { struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); - uint32_t i = 0, jobs_rx = 0; + uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; int ret = -1; uint8_t key[IMB_SHA_512_BLOCK_SIZE]; uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; @@ -1215,7 +1221,11 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, } - uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + if (burst_type == BURST_TYPE_GENERIC) + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + else + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + job->hash_alg); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -1313,12 +1323,23 @@ test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, } if (test_hmac_shax_burst(mb_mgr, &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type)) { + num_jobs, sha_type, + BURST_TYPE_GENERIC)) { printf("error #%d - burst API\n", vect); test_suite_update(ts, 0, 1); } else { test_suite_update(ts, 1, 0); } + if (test_hmac_shax_burst(mb_mgr, + &hmac_sha256_sha512_vectors[idx], + num_jobs, sha_type, + BURST_TYPE_HASH)) { + printf("error #%d - hash-only burst API\n", vect); + test_suite_update(ts, 0, 1); + } else { + test_suite_update(ts, 1, 0); + } + } printf("\n"); } -- GitLab From 00b60b5fa1545b6f3bf05da5bc5f5a89bc6dbf3b Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 27 May 2022 09:14:16 +0000 Subject: [PATCH 140/369] perf: Allow all HMAC-SHAx algos with hash-only burst API --- perf/ipsec_perf.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 191987f3..3ea2cf8c 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -3632,9 +3632,13 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - /* currently only HMAC-SHA1 supported by hash-only burst API */ + /* currently only HMAC-SHAx algs supported by hash-only burst API */ if (test_api == TEST_API_HASH_BURST && - custom_job_params.hash_alg != TEST_SHA1_HMAC) { + ((custom_job_params.hash_alg != TEST_SHA1_HMAC) && + (custom_job_params.hash_alg != TEST_SHA_224_HMAC) && + (custom_job_params.hash_alg != TEST_SHA_256_HMAC) && + (custom_job_params.hash_alg != TEST_SHA_384_HMAC) && + (custom_job_params.hash_alg != TEST_SHA_512_HMAC))) { fprintf(stderr, "Unsupported hash-only burst API algorithm selected\n"); return EXIT_FAILURE; -- GitLab From 061b0306cd761e17a2b28487f1e6c9d7d267947f Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 13 May 2022 07:15:15 +0000 Subject: [PATCH 141/369] avx512: [ZUC-EIA3-256] Do not reset tag with a new job For ZUC-EIA2-256, it is not needed to reset the memory for the tag, as initialization function overwrites it. --- lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm index dedda8b4..aa5cb101 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm @@ -871,17 +871,9 @@ FLUSH_JOB_ZUC256_EEA3: not tmp and [state + _zuc_unused_lane_bitmask], WORD(tmp) ; Reset temporary digest for the lane -%if %%TAG_SIZE == 4 +%if %%KEY_SIZE == 128 mov dword [state + _zuc_args_digest + lane*4], 0 -%elif %%TAG_SIZE == 8 - mov qword [state + _zuc_args_digest + lane*8], 0 -%else ; %%TAG_SIZE == 16 - vpxor xmm0, xmm0 - shl lane, 4 - vmovdqa [state + _zuc_args_digest + lane], xmm0 - shr lane, 4 %endif - mov tmp, [job + _src] add tmp, [job + _hash_start_src_offset_in_bytes] mov [state + _zuc_args_in + lane*8], tmp -- GitLab From bf32e8a2c2e998a9a07daca74665bc8aab1d324e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 23 May 2022 16:28:50 +0000 Subject: [PATCH 142/369] avx512: [ZUC-EIA3] use stack to store intermediate products of digests In case of 8 and 16 byte tags, 2 or 4 intermediate products are calculated with pclmul's to generate the tag. Instead of shuffling the products of the digests to have the values positioned correctly every iteration (every 16 bytes of data), these products are stored and XOR'd in stack, until the last iteration, where they are positioned in the right order and saved in OOO manager. --- lib/avx512/zuc_x16_avx512.asm | 218 ++++++++++++++++++++++++++++++++-- 1 file changed, 205 insertions(+), 13 deletions(-) diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512/zuc_x16_avx512.asm index 04822742..0b1a26d9 100755 --- a/lib/avx512/zuc_x16_avx512.asm +++ b/lib/avx512/zuc_x16_avx512.asm @@ -439,6 +439,13 @@ times 4 db 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf ;; - To access a 16-byte chunk inside a 64-byte chunk, ks_idx is used %define GET_KS(base, lane4_idx, bytes16_idx, ks_idx) (base + lane4_idx * 512 + bytes16_idx * 64 + ks_idx * 16) +; Define Stack Layout +START_FIELDS +;;; name size align +FIELD _TEMP_DIGEST_SAVE, 16*64, 64 +FIELD _RSP, 8, 8 +%assign STACK_SPACE _FIELD_OFFSET + mksection .text align 64 @@ -1693,7 +1700,7 @@ init_for_auth_tag_8B: ;; To use it with YMM and ZMM registers, VPCMULQDQ must be ;; supported. ;; -%macro DIGEST_DATA 14 +%macro DIGEST_DATA 14-16 %define %%DATA %1 ; [in] Input data (16 bytes) per buffer %define %%KS_L %2 ; [in/clobbered] Lower 16 bytes of KS per buffer %define %%KS_H %3 ; [in/clobbered] Higher 16 bytes of KS per buffer @@ -1707,8 +1714,13 @@ init_for_auth_tag_8B: %define %%TMP4 %11 ; [clobbered] Temporary XMM/YMM/ZMM register %define %%TMP5 %12 ; [clobbered] Temporary XMM/YMM/ZMM register %define %%TMP6 %13 ; [clobbered] Temporary XMM/YMM/ZMM register -%define %%TAG_SIZE %14 ; [in] Tag size (4, 8 or 16 bytes) +%define %%TAG_SIZE %14 ; [constant] Tag size (4, 8 or 16 bytes) +%define %%LANE_GROUP %15 ; [constant] Lane group (0-3) +%define %%IDX %16 ; [constant] Index inside lane group (0-3) +%if %0 == 15 +%define %%IDX 0 +%endif ;; Set up KS ;; ;; KS_L contains bytes 15:0 of KS (for 1, 2 or 4 buffers) @@ -1745,10 +1757,14 @@ init_for_auth_tag_8B: %endif ; %%TAG_SIZE == 4 %if %%TAG_SIZE >= 8 ; Move previous result to low 32 bits and XOR with previous digest +%if %0 > 14 + vpternlogq %%TMP5, %%TMP6, [rsp + 256*%%LANE_GROUP + %%IDX*16], 0x96 + vmovdqa64 [rsp + 256*%%LANE_GROUP + %%IDX*16], %%TMP5 +%else vpxorq %%TMP5, %%TMP5, %%TMP6 vpshufb %%TMP5, %%TMP5, [rel shuf_mask_0_0_0_dw1] - vpxorq %%IN_OUT, %%IN_OUT, %%TMP5 +%endif vpclmulqdq %%TMP3, %%TMP1, %%KS_L, 0x10 vpclmulqdq %%TMP4, %%TMP1, %%KS_M1, 0x01 @@ -1757,12 +1773,16 @@ init_for_auth_tag_8B: ; XOR all the products and keep only 32-63 bits vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 +%if %0 > 14 + vpternlogq %%TMP5, %%TMP6, [rsp + 256*%%LANE_GROUP + 64 + %%IDX*16], 0x96 + vmovdqa64 [rsp + 256*%%LANE_GROUP + 64 + %%IDX*16], %%TMP5 +%else vpxorq %%TMP5, %%TMP5, %%TMP6 vpandq %%TMP5, %%TMP5, [rel bits_32_63] ; XOR with bits 32-63 of previous digest vpxorq %%IN_OUT, %%TMP5 - +%endif %if %%TAG_SIZE == 16 ; Prepare data and calculate bits 95-64 of tag vpclmulqdq %%TMP3, %%TMP1, %%KS_M1, 0x00 @@ -1772,11 +1792,16 @@ init_for_auth_tag_8B: ; XOR all the products and move bits 63-32 to bits 95-64 vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 +%if %0 > 14 + vpternlogq %%TMP5, %%TMP6, [rsp + 256*%%LANE_GROUP + 64*2 + %%IDX*16], 0x96 + vmovdqa64 [rsp + 256*%%LANE_GROUP + 64*2 + %%IDX*16], %%TMP5 +%else vpxorq %%TMP5, %%TMP5, %%TMP6 vpshufb %%TMP5, %%TMP5, [rel shuf_mask_0_dw1_0_0] ; XOR with previous bits 64-95 of previous digest vpxorq %%IN_OUT, %%TMP5 +%endif ; Prepare data and calculate bits 127-96 of tag vpclmulqdq %%TMP3, %%TMP1, %%KS_M1, 0x10 @@ -1786,11 +1811,16 @@ init_for_auth_tag_8B: ; XOR all the products and move bits 63-32 to bits 127-96 vpternlogq %%TMP5, %%TMP3, %%TMP4, 0x96 +%if %0 > 14 + vpternlogq %%TMP5, %%TMP6, [rsp + 256*%%LANE_GROUP + 64*3 + %%IDX*16], 0x96 + vmovdqa64 [rsp + 256*%%LANE_GROUP + 64*3 + %%IDX*16], %%TMP5 +%else vpxorq %%TMP5, %%TMP5, %%TMP6 vpshufb %%TMP5, %%TMP5, [rel shuf_mask_dw1_0_0_0] ; XOR with lower 96 bits, to construct 128 bits of tag vpxorq %%IN_OUT, %%TMP5 +%endif %endif ; %%TAG_SIZE == 16 %endif ; %%TAG_SIZE >= 8 @@ -1971,6 +2001,21 @@ init_for_auth_tag_8B: %define %%TMP_KMASK1 k4 %define %%TMP_KMASK2 k5 +%if %%TAG_SIZE != 4 + mov %%TMP, rsp + ; Reserve stack space to store temporary digest products + sub rsp, STACK_SPACE + and rsp, ~63 + mov [rsp + _RSP], %%TMP + + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + 64*%%I], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif + xor %%OFFSET, %%OFFSET mov DWORD(%%TMP), 0xAAAAAAAA @@ -1982,10 +2027,12 @@ init_for_auth_tag_8B: mov DWORD(%%TMP), 0x55555555 kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP) +%if %%TAG_SIZE == 4 vpxorq %%DIGEST_0, %%DIGEST_0 vpxorq %%DIGEST_1, %%DIGEST_1 vpxorq %%DIGEST_2, %%DIGEST_2 vpxorq %%DIGEST_3, %%DIGEST_3 +%endif %if USE_GFNI_VAES_VPCLMUL == 0 vmovdqa64 %%REV_TABLE_L, [rel bit_reverse_table_l] @@ -2059,7 +2106,8 @@ init_for_auth_tag_8B: ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers DIGEST_DATA %%ZTMP7, %%ZKS_L, %%ZKS_H, %%ZTMP8, %%ZTMP10, \ APPEND(%%DIGEST_, %%LANE_GROUP), %%SHUF_DATA_KMASK, \ - %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, %%TAG_SIZE + %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, %%ZTMP6, \ + %%TAG_SIZE, %%LANE_GROUP %else ; USE_GFNI_VAES_VPCLMUL == 1 ;; If VPCMUL is NOT available, read chunks of 16 bytes of data @@ -2068,9 +2116,11 @@ init_for_auth_tag_8B: %assign %%J 0 %rep 4 +%if %%TAG_SIZE == 4 %if %%idx == 0 ; Reset temporary digests (for the first 16 bytes) vpxorq APPEND(%%XDIGEST_, %%J), APPEND(%%XDIGEST_, %%J) +%endif %endif ; Read the next 2 blocks of 16 bytes of KS vmovdqa64 %%KS_L, [GET_KS(%%KS, %%LANE_GROUP, %%idx, %%J)] @@ -2082,21 +2132,26 @@ init_for_auth_tag_8B: %%REV_AND_TABLE, %%XTMP2, %%XTMP3 ; Digest 16 bytes of data with 24 bytes of KS, for one buffer - DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, %%XTMP8, %%XTMP10, APPEND(%%XDIGEST_, %%J), \ - %%SHUF_DATA_KMASK, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE + DIGEST_DATA %%XTMP7, %%KS_L, %%KS_H, %%XTMP8, %%XTMP10, \ + APPEND(%%XDIGEST_, %%J), %%SHUF_DATA_KMASK, \ + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE, \ + %%LANE_GROUP, %%J ; Once all 64 bytes of data have been digested, insert them in temporary ZMM register +%if %%TAG_SIZE == 4 %if %%idx == 3 vinserti32x4 %%Z_TEMP_DIGEST, APPEND(%%XDIGEST_, %%J), %%J %endif +%endif %assign %%J (%%J + 1) %endrep ; %rep 4 %%J ; XOR with previous digest +%if %%TAG_SIZE == 4 %if %%idx == 3 vpxorq APPEND(%%DIGEST_, %%LANE_GROUP), %%Z_TEMP_DIGEST %endif +%endif %endif ;; USE_GFNI_VAES_VPCLMUL == 0 %assign %%idx (%%idx + 1) %assign %%N %%N+1 @@ -2124,6 +2179,29 @@ init_for_auth_tag_8B: dec %%NROUNDS jnz %%_loop + ; Read from stack to extract the products and arrange them to XOR later + ; against previous digests (only for 8-byte and 16-byte tag) +%if %%TAG_SIZE != 4 +%assign %%I 0 +%rep 4 + vmovdqa64 %%ZTMP1, [rsp + %%I*256] + vmovdqa64 %%ZTMP2, [rsp + %%I*256 + 64] + vpshufb %%ZTMP1, %%ZTMP1, [rel shuf_mask_0_0_0_dw1] + vpandq %%ZTMP2, %%ZTMP2, [rel bits_32_63] +%if %%TAG_SIZE == 16 + vmovdqa64 %%ZTMP3, [rsp + %%I*256 + 64*2] + vmovdqa64 %%ZTMP4, [rsp + %%I*256 + 64*3] + vpshufb %%ZTMP3, %%ZTMP3, [rel shuf_mask_0_dw1_0_0] + vpshufb %%ZTMP4, %%ZTMP4, [rel shuf_mask_dw1_0_0_0] + vpternlogq %%ZTMP1, %%ZTMP2, %%ZTMP3, 0x96 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP4 +%else ; %%TAG_SIZE == 8 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP2 +%endif +%assign %%I (%%I + 1) +%endrep +%endif ; %%TAG_SIZE != 4 + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 @@ -2149,6 +2227,18 @@ init_for_auth_tag_8B: vpsubw YWORD(%%ZTMP2){%%TMP_KMASK1}, YWORD(%%ZTMP1) vmovdqa64 [%%LEN], YWORD(%%ZTMP2) +%if %%TAG_SIZE != 4 +%ifdef SAFE_DATA + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + %%I*64], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif + + mov rsp, [rsp + _RSP] +%endif %endmacro ;; @@ -2762,6 +2852,21 @@ _no_final_rounds: %define %%YTMP1 YWORD(%%ZTMP1) +%if %%TAG_SIZE != 4 + mov %%TMP1, rsp + ; Reserve stack space to store temporary digest products + sub rsp, STACK_SPACE + and rsp, ~63 + mov [rsp + _RSP], %%TMP1 + + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + 64*%%I], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif ; %%TAG_SIZE != 4 + mov DWORD(%%TMP1), 0x55555555 kmovd %%SHUF_DATA_KMASK, DWORD(%%TMP1) ;; Read first buffers 0,4,8,12; then 1,5,9,13, and so on, @@ -2771,8 +2876,9 @@ _no_final_rounds: ;; for buffers 1,5,9,13, and so on. %assign %%IDX 0 %rep 4 +%if %%TAG_SIZE == 4 vpxorq APPEND(%%DIGEST_, %%IDX), APPEND(%%DIGEST_, %%IDX) - +%endif mov %%DATA_ADDR0, [%%DATA + %%IDX*8 + 0*32] mov %%DATA_ADDR1, [%%DATA + %%IDX*8 + 1*32] mov %%DATA_ADDR2, [%%DATA + %%IDX*8 + 2*32] @@ -2796,7 +2902,7 @@ _no_final_rounds: DIGEST_DATA %%ZTMP1, APPEND(%%KS_TRANS, %%I), APPEND(%%KS_TRANS, %%J), \ %%ZTMP8, %%ZTMP9, APPEND(%%DIGEST_, %%IDX), %%SHUF_DATA_KMASK, \ %%ZTMP2, %%ZTMP3, %%ZTMP4, %%ZTMP5, \ - %%ZTMP6, %%ZTMP7, %%TAG_SIZE + %%ZTMP6, %%ZTMP7, %%TAG_SIZE, %%IDX %assign %%J (%%J + 1) %assign %%I (%%I + 1) @@ -2815,6 +2921,29 @@ _no_final_rounds: %assign %%IDX (%%IDX + 1) %endrep + ; Read from stack to extract the products and arrange them to XOR later + ; against previous digests (only for 8-byte and 16-byte tag) +%if %%TAG_SIZE != 4 +%assign %%I 0 +%rep 4 + vmovdqa64 %%ZTMP1, [rsp + %%I*256] + vmovdqa64 %%ZTMP2, [rsp + %%I*256 + 64] + vpshufb %%ZTMP1, %%ZTMP1, [rel shuf_mask_0_0_0_dw1] + vpandq %%ZTMP2, %%ZTMP2, [rel bits_32_63] +%if %%TAG_SIZE == 16 + vmovdqa64 %%ZTMP3, [rsp + %%I*256 + 64*2] + vmovdqa64 %%ZTMP4, [rsp + %%I*256 + 64*3] + vpshufb %%ZTMP3, %%ZTMP3, [rel shuf_mask_0_dw1_0_0] + vpshufb %%ZTMP4, %%ZTMP4, [rel shuf_mask_dw1_0_0_0] + vpternlogq %%ZTMP1, %%ZTMP2, %%ZTMP3, 0x96 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP4 +%else ; %%TAG_SIZE == 8 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP2 +%endif +%assign %%I (%%I + 1) +%endrep +%endif ; %%TAG_SIZE != 4 + UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 @@ -2828,11 +2957,23 @@ _no_final_rounds: vmovdqu64 [%%DATA + 64], %%ZTMP2 ; Update array of lengths (subtract 512 bits from all lengths if valid lane) - vmovdqa %%YTMP1, [LEN] + vmovdqa64 %%YTMP1, [LEN] vpcmpw %%TMP_KMASK1, %%YTMP1, [rel all_ffs], 4 vpsubw %%YTMP1{%%TMP_KMASK1}, [rel all_512w] - vmovdqa [%%LEN], %%YTMP1 + vmovdqa64 [%%LEN], %%YTMP1 + +%if %%TAG_SIZE != 4 +%ifdef SAFE_DATA + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + %%I*64], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif + mov rsp, [rsp + _RSP] +%endif ; %%TAG_SIZE != 4 %endmacro @@ -2885,6 +3026,21 @@ _no_final_rounds: %define %%DATA_ADDR %%TMP3 +%if %%TAG_SIZE != 4 + mov %%TMP1, rsp + ; Reserve stack space to store temporary digest products + sub rsp, STACK_SPACE + and rsp, ~63 + mov [rsp + _RSP], %%TMP1 + + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + 64*%%I], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif ; %%TAG_SIZE != 4 + vmovdqa %%REV_TABLE_L, [rel bit_reverse_table_l] vmovdqa %%REV_TABLE_H, [rel bit_reverse_table_h] vmovdqa %%REV_AND_TABLE, [rel bit_reverse_and_table] @@ -2902,7 +3058,9 @@ _no_final_rounds: %assign %%J 0 %rep 4 +%if %%TAG_SIZE == 4 vpxor %%TEMP_DIGEST, %%TEMP_DIGEST +%endif mov %%DATA_ADDR, [%%DATA + 8*(%%J*4 + %%I)] %assign %%K 0 @@ -2924,7 +3082,7 @@ _no_final_rounds: vmovdqu %%KS_H, [%%KS + (16*%%J + %%I*512) + (%%K + 1)*(16*4)] ; Digest 16 bytes of data with 24 bytes of KS, for 4 buffers DIGEST_DATA %%XDATA, %%KS_L, %%KS_H, %%XTMP7, %%XTMP8, %%TEMP_DIGEST, %%SHUF_DATA_KMASK, \ - %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE + %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5, %%XTMP6, %%TAG_SIZE, %%I, %%J %assign %%K (%%K + 1) %endrep @@ -2944,6 +3102,28 @@ _no_final_rounds: %assign %%I (%%I + 1) %endrep + ; Read from stack to extract the products and arrange them to XOR later + ; against previous digests (only for 8-byte and 16-byte tag) +%if %%TAG_SIZE != 4 +%assign %%I 0 +%rep 4 + vmovdqa64 %%ZTMP1, [rsp + %%I*256] + vmovdqa64 %%ZTMP2, [rsp + %%I*256 + 64] + vpshufb %%ZTMP1, %%ZTMP1, [rel shuf_mask_0_0_0_dw1] + vpandq %%ZTMP2, %%ZTMP2, [rel bits_32_63] +%if %%TAG_SIZE == 16 + vmovdqa64 %%ZTMP3, [rsp + %%I*256 + 64*2] + vmovdqa64 %%ZTMP4, [rsp + %%I*256 + 64*3] + vpshufb %%ZTMP3, %%ZTMP3, [rel shuf_mask_0_dw1_0_0] + vpshufb %%ZTMP4, %%ZTMP4, [rel shuf_mask_dw1_0_0_0] + vpternlogq %%ZTMP1, %%ZTMP2, %%ZTMP3, 0x96 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP4 +%else ; %%TAG_SIZE == 8 + vpxorq APPEND(%%DIGEST_, %%I), %%ZTMP1, %%ZTMP2 +%endif +%assign %%I (%%I + 1) +%endrep +%endif ; %%TAG_SIZE != 4 UPDATE_TAGS %%T, %%TAG_SIZE, order_0_4_8_12, %%TMP1, %%TMP_KMASK1, %%TMP_KMASK2, \ %%DIGEST_0, %%DIGEST_1, %%DIGEST_2, %%DIGEST_3, \ %%ZTMP1, %%ZTMP2, %%ZTMP3, %%ZTMP4 @@ -2962,6 +3142,18 @@ _no_final_rounds: vpsubw %%YTMP1{%%TMP_KMASK1}, [rel all_512w] vmovdqa64 [%%LEN], %%YTMP1 +%if %%TAG_SIZE != 4 +%ifdef SAFE_DATA + vpxorq %%ZTMP1, %%ZTMP1 +%assign %%I 0 +%rep 16 + vmovdqa64 [rsp + %%I*64], %%ZTMP1 +%assign %%I (%%I + 1) +%endrep +%endif + + mov rsp, [rsp + _RSP] +%endif ; %%TAG_SIZE != 4 %endmacro ;; -- GitLab From 06a5613482cbc72a4c42acb6033c88023fe4f2a1 Mon Sep 17 00:00:00 2001 From: Kamila Lipinska Date: Thu, 19 May 2022 11:51:28 +0000 Subject: [PATCH 143/369] sse: [SNOW3G-UEA2] Add new implementation --- lib/include/transpose_sse.asm | 70 +++ lib/sse/snow3g_uea2_by4_sse.asm | 924 ++++++++++++++++++++++++++++++++ 2 files changed, 994 insertions(+) create mode 100644 lib/include/transpose_sse.asm create mode 100644 lib/sse/snow3g_uea2_by4_sse.asm diff --git a/lib/include/transpose_sse.asm b/lib/include/transpose_sse.asm new file mode 100644 index 00000000..74797788 --- /dev/null +++ b/lib/include/transpose_sse.asm @@ -0,0 +1,70 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%ifndef _TRANSPOSE_SSE_ASM_ +%define _TRANSPOSE_SSE_ASM_ + +;; transpose r0, r1, r2, r3, t0, t1 +;; "transpose" data in {r0..r3} using temps {t0..t3} +;; Input looks like: {r0 r1 r2 r3} +;; r0 = {a3 a2 a1 a0} +;; r1 = {b3 b2 b1 b0} +;; r2 = {c3 c2 c1 c0} +;; r3 = {d3 d2 d1 d0} +;; +;; output looks like: {t0 r1 r0 r3} +;; t0 = {d0 c0 b0 a0} +;; r1 = {d1 c1 b1 a1} +;; r0 = {d2 c2 b2 a2} +;; r3 = {d3 c3 b3 a3} + +%macro TRANSPOSE4_U32 6 +%define %%r0 %1 +%define %%r1 %2 +%define %%r2 %3 +%define %%r3 %4 +%define %%t0 %5 +%define %%t1 %6 + movdqa %%t0, %%r0 + shufps %%t0, %%r1, 0x44 ; t0 = {b1 b0 a1 a0} + shufps %%r0, %%r1, 0xEE ; r0 = {b3 b2 a3 a2} + + movdqa %%t1, %%r2 + shufps %%t1, %%r3, 0x44 ; t1 = {d1 d0 c1 c0} + shufps %%r2, %%r3, 0xEE ; r2 = {d3 d2 c3 c2} + + movdqa %%r1, %%t0 + shufps %%r1, %%t1, 0xDD ; r1 = {d1 c1 b1 a1} + + movdqa %%r3, %%r0 + shufps %%r3, %%r2, 0xDD ; r3 = {d3 c3 b3 a3} + + shufps %%r0, %%r2, 0x88 ; r0 = {d2 c2 b2 a2} + shufps %%t0, %%t1, 0x88 ; t0 = {d0 c0 b0 a0} +%endmacro + +%endif ;; _TRANSPOSE_SSE_ASM_ \ No newline at end of file diff --git a/lib/sse/snow3g_uea2_by4_sse.asm b/lib/sse/snow3g_uea2_by4_sse.asm new file mode 100644 index 00000000..8d7a29e8 --- /dev/null +++ b/lib/sse/snow3g_uea2_by4_sse.asm @@ -0,0 +1,924 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "include/reg_sizes.asm" +%include "include/memcpy.asm" +%include "include/imb_job.asm" +%include "include/clear_regs.asm" +%include "include/cet.inc" +%include "include/mb_mgr_datastruct.asm" +%include "include/memcpy.asm" +%include "include/transpose_sse.asm" + +align 64 +const_fixup_mask: +times 2 dq 0x7272727272727272 + +align 64 +const_byte_mix_col_rev: +dd 0x00030201, 0x04070605, 0x080b0a09, 0x0c0f0e0d + +align 16 +snow3g_inv_SR_SQ: +db 0xC2, 0xA6, 0x8F, 0x0A, 0x0D, 0xBE, 0xA7, 0x08 +db 0x1D, 0x99, 0x45, 0x59, 0x13, 0xD2, 0x11, 0x9F +db 0xAE, 0xE6, 0xD4, 0xA4, 0x92, 0x8D, 0x58, 0xC1 +db 0xD0, 0x97, 0xC8, 0x84, 0x9D, 0x4F, 0xBC, 0x3B +db 0x2D, 0xEB, 0x27, 0x53, 0x72, 0x4E, 0xE3, 0xEE +db 0xDA, 0x7F, 0xAA, 0x4D, 0x5C, 0x2F, 0x44, 0xDB +db 0x3E, 0x3A, 0x67, 0xC5, 0xC3, 0x6A, 0x16, 0x4C +db 0x38, 0xCC, 0xD7, 0xDD, 0x70, 0x62, 0xF2, 0x19 +db 0x10, 0x09, 0x98, 0x4B, 0x61, 0xC9, 0x86, 0x03 +db 0xA8, 0x6B, 0x5A, 0x33, 0x6E, 0x54, 0x5D, 0x8C +db 0x41, 0x1A, 0xF7, 0xF6, 0x82, 0xC6, 0xF8, 0x80 +db 0xC0, 0xC7, 0xFE, 0xB3, 0x65, 0x2C, 0x7B, 0xBA +db 0xB4, 0xFC, 0x2A, 0x22, 0x0C, 0x73, 0xF5, 0x5F +db 0x64, 0x68, 0x2E, 0x94, 0xB2, 0x24, 0x35, 0x14 +db 0x78, 0xFB, 0xBF, 0x48, 0xDE, 0xED, 0x43, 0x07 +db 0xB6, 0x32, 0xE4, 0xBD, 0x74, 0x7D, 0x57, 0x46 +db 0x3C, 0x37, 0xC4, 0xB7, 0x51, 0x8A, 0xF3, 0x55 +db 0x6C, 0xCF, 0x79, 0xAB, 0x77, 0xA3, 0xE1, 0x93 +db 0xD5, 0x6D, 0x81, 0x5B, 0x2B, 0x9A, 0x7E, 0x8B +db 0x04, 0xB5, 0x85, 0xD3, 0x91, 0xA1, 0x47, 0x52 +db 0xA5, 0xEC, 0xD6, 0xBB, 0x20, 0x87, 0x26, 0xF0 +db 0xAF, 0x4A, 0x89, 0xF4, 0xCE, 0x25, 0xCB, 0x50 +db 0x00, 0x3F, 0xD9, 0x42, 0x90, 0x21, 0x3D, 0xA9 +db 0xE7, 0x29, 0x01, 0xF1, 0x36, 0x5E, 0xFA, 0xCD +db 0xE5, 0x31, 0x1B, 0x05, 0xFD, 0x9E, 0xA0, 0x76 +db 0x30, 0xB1, 0x75, 0xB0, 0x9B, 0x56, 0xEA, 0x1C +db 0xEF, 0x06, 0x69, 0x7A, 0x95, 0x88, 0x15, 0xFF +db 0xCA, 0xAC, 0x0E, 0x23, 0xD8, 0x0F, 0x28, 0x0B +db 0x18, 0xF9, 0x63, 0x1E, 0x83, 0x66, 0x39, 0x9C +db 0xE2, 0x49, 0x1F, 0xE8, 0xD1, 0x34, 0x7C, 0xA2 +db 0xB9, 0xE0, 0x02, 0x12, 0xE9, 0xDF, 0xAD, 0x71 +db 0x96, 0x8E, 0x6F, 0xB8, 0x40, 0x60, 0x17, 0xDC + +align 64 +xmm_bswap: +dd 0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f + +;; used for inverse of AESENC shift rows operation +align 64 +const_fixed_rotate_mask: +dq 0x0b0e0104070a0d00, 0x0306090c0f020508 + +align 64 +idx_rows_sse: +times 4 dd 0x00000000 +times 4 dd 0x10101010 +times 4 dd 0x20202020 +times 4 dd 0x30303030 +times 4 dd 0x40404040 +times 4 dd 0x50505050 +times 4 dd 0x60606060 +times 4 dd 0x70707070 +times 4 dd 0x80808080 +times 4 dd 0x90909090 +times 4 dd 0xa0a0a0a0 +times 4 dd 0xb0b0b0b0 +times 4 dd 0xc0c0c0c0 +times 4 dd 0xd0d0d0d0 +times 4 dd 0xe0e0e0e0 +times 4 dd 0xf0f0f0f0 + +align 64 +ms_byte_mask: +dd 0x0f0b0703 +dd 0x80808080 +dd 0x80808080 +dd 0x80808080 + +align 64 +ls_byte_mask: +dd 0x0c080400 +dd 0x80808080 +dd 0x80808080 +dd 0x80808080 + +align 64 +low_nibble_byte_mask: +times 4 dd 0x0f0f0f0f + +align 64 +mul_alpha: +db 0x00, 0x13, 0x26, 0x35, 0x4C, 0x5F, 0x6A, 0x79 +db 0x98, 0x8B, 0xBE, 0xAD, 0xD4, 0xC7, 0xF2, 0xE1 +db 0x00, 0xCF, 0x37, 0xF8, 0x6E, 0xA1, 0x59, 0x96 +db 0xDC, 0x13, 0xEB, 0x24, 0xB2, 0x7D, 0x85, 0x4A +db 0x00, 0x9F, 0x97, 0x08, 0x87, 0x18, 0x10, 0x8F +db 0xA7, 0x38, 0x30, 0xAF, 0x20, 0xBF, 0xB7, 0x28 +db 0x00, 0xE1, 0x6B, 0x8A, 0xD6, 0x37, 0xBD, 0x5C +db 0x05, 0xE4, 0x6E, 0x8F, 0xD3, 0x32, 0xB8, 0x59 +db 0x00, 0x99, 0x9B, 0x02, 0x9F, 0x06, 0x04, 0x9D +db 0x97, 0x0E, 0x0C, 0x95, 0x08, 0x91, 0x93, 0x0A +db 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 +db 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF +db 0x00, 0xE7, 0x67, 0x80, 0xCE, 0x29, 0xA9, 0x4E +db 0x35, 0xD2, 0x52, 0xB5, 0xFB, 0x1C, 0x9C, 0x7B +db 0x00, 0x0A, 0x14, 0x1E, 0x28, 0x22, 0x3C, 0x36 +db 0x50, 0x5A, 0x44, 0x4E, 0x78, 0x72, 0x6C, 0x66 + +align 64 +div_alpha: +db 0x00, 0xCD, 0x33, 0xFE, 0x66, 0xAB, 0x55, 0x98 +db 0xCC, 0x01, 0xFF, 0x32, 0xAA, 0x67, 0x99, 0x54 +db 0x00, 0x40, 0x80, 0xC0, 0xA9, 0xE9, 0x29, 0x69 +db 0xFB, 0xBB, 0x7B, 0x3B, 0x52, 0x12, 0xD2, 0x92 +db 0x00, 0x0F, 0x1E, 0x11, 0x3C, 0x33, 0x22, 0x2D +db 0x78, 0x77, 0x66, 0x69, 0x44, 0x4B, 0x5A, 0x55 +db 0x00, 0x18, 0x30, 0x28, 0x60, 0x78, 0x50, 0x48 +db 0xC0, 0xD8, 0xF0, 0xE8, 0xA0, 0xB8, 0x90, 0x88 +db 0x00, 0x31, 0x62, 0x53, 0xC4, 0xF5, 0xA6, 0x97 +db 0x21, 0x10, 0x43, 0x72, 0xE5, 0xD4, 0x87, 0xB6 +db 0x00, 0x5F, 0xBE, 0xE1, 0xD5, 0x8A, 0x6B, 0x34 +db 0x03, 0x5C, 0xBD, 0xE2, 0xD6, 0x89, 0x68, 0x37 +db 0x00, 0xF0, 0x49, 0xB9, 0x92, 0x62, 0xDB, 0x2B +db 0x8D, 0x7D, 0xC4, 0x34, 0x1F, 0xEF, 0x56, 0xA6 +db 0x00, 0x29, 0x52, 0x7B, 0xA4, 0x8D, 0xF6, 0xDF +db 0xE1, 0xC8, 0xB3, 0x9A, 0x45, 0x6C, 0x17, 0x3E + +align 64 +all_fs: +times 4 dd 0xffffffff + +mksection .text + +struc STACK +_keystream: resb (4 * 16) +_gpr_save: resq 6 +_rsp_save: resq 1 +endstruc + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Stores register contents and create the stack frame +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_FUNC_START 0 + mov rax, rsp + sub rsp, STACK_size + and rsp, ~63 + + mov [rsp + _gpr_save + 8 * 0], rbx + mov [rsp + _gpr_save + 8 * 1], rbp + mov [rsp + _gpr_save + 8 * 2], r12 + mov [rsp + _gpr_save + 8 * 3], rsi + mov [rsp + _gpr_save + 8 * 4], rdi +%ifdef LINUX + mov [rsp + _gpr_save + 8 * 5], r9 +%else + mov [rsp + _gpr_save + 8 * 5], rcx + mov [rsp + _gpr_save + 8 * 6], rdx +%endif + mov [rsp + _rsp_save], rax ;; original SP +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Restores register contents and removes the stack frame +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_FUNC_END 0 + mov rbx, [rsp + _gpr_save + 8 * 0] + mov rbp, [rsp + _gpr_save + 8 * 1] + mov r12, [rsp + _gpr_save + 8 * 2] + mov rsi, [rsp + _gpr_save + 8 * 3] + mov rdi, [rsp + _gpr_save + 8 * 4] +%ifdef LINUX + mov r9, [rsp + _gpr_save + 8 * 5] +%else + mov rcx, [rsp + _gpr_save + 8 * 5] + mov rdx, [rsp + _gpr_save + 8 * 6] +%endif + mov rsp, [rsp + _rsp_save] ;; original SP +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SSE_LOOKUP_16X8BIT: Search 16 8-bit values in lookup table +;; arg 1 [in] : xmm register with 16x8bit indexes to search +;; arg 2 [in] : memory with 16 8-bit indices to be looked up +;; arg3-15 [clobbered]: xmm registers used as temp variables +;; arg 16 [out]: xmm register to write 16 8-bit values from the table +;; in and out xmm register must be different registers +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SSE_LOOKUP_16X8BIT 15 +%define %%IN_INDEXES_H %1 ;; [in/out] xmm reg with indexes +%define %%TMP_INDEXES_L %2 ;; [clobbered] xmm register +%define %%TMP1 %3 ;; [clobbered] xmm register +%define %%TMP2 %4 ;; [clobbered] xmm register +%define %%TMP3 %5 ;; [clobbered] xmm register +%define %%TMP4 %6 ;; [clobbered] xmm register +%define %%TMP5 %7 ;; [clobbered] xmm register +%define %%TMP6 %8 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_0 %9 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_1 %10 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_2 %11 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_3 %12 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_4 %13 ;; [clobbered] xmm register +%define %%TMPARG_TAB_VAL_5 %14 ;; [clobbered] xmm register +%define %%OUT_SUBSTITUTE_VAL %15 ;; [out] xmm register + + + movdqa %%TMP1, [rel idx_rows_sse + (15 * 16)] ;; 4x0xf0f0f0f0 + movdqa %%TMP2, %%TMP1 + psrlq %%TMP2, 4 ;; 4x0x0f0f0f0f + movdqa %%TMP_INDEXES_L, %%IN_INDEXES_H + pand %%IN_INDEXES_H, %%TMP1 ;; index top nibble + pand %%TMP_INDEXES_L, %%TMP2 ;; index low nibble + + movdqa %%TMP1, %%IN_INDEXES_H + movdqa %%TMP3, %%IN_INDEXES_H + movdqa %%TMP4, %%IN_INDEXES_H + movdqa %%TMP5, %%IN_INDEXES_H + movdqa %%TMP6, %%IN_INDEXES_H + movdqa %%TMP2, %%IN_INDEXES_H + pcmpeqb %%TMP1, [rel idx_rows_sse + (0 * 16)] + movdqa %%TMPARG_TAB_VAL_0, [rel snow3g_inv_SR_SQ + (0 * 16)] + pcmpeqb %%TMP3, [rel idx_rows_sse + (1 * 16)] + movdqa %%TMPARG_TAB_VAL_1, [rel snow3g_inv_SR_SQ + (1 * 16)] + pcmpeqb %%TMP4, [rel idx_rows_sse + (2 * 16)] + movdqa %%TMPARG_TAB_VAL_2, [rel snow3g_inv_SR_SQ + (2 * 16)] + pcmpeqb %%TMP5, [rel idx_rows_sse + (3 * 16)] + movdqa %%TMPARG_TAB_VAL_3, [rel snow3g_inv_SR_SQ + (3 * 16)] + pcmpeqb %%TMP6, [rel idx_rows_sse + (4 * 16)] + movdqa %%TMPARG_TAB_VAL_4, [rel snow3g_inv_SR_SQ + (4 * 16)] + pcmpeqb %%TMP2, [rel idx_rows_sse + (5 * 16)] + movdqa %%TMPARG_TAB_VAL_5, [rel snow3g_inv_SR_SQ + (5 * 16)] + + pshufb %%TMPARG_TAB_VAL_0, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_1, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_2, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_3, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_4, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_5, %%TMP_INDEXES_L + + pand %%TMP1, %%TMPARG_TAB_VAL_0 + pand %%TMP3, %%TMPARG_TAB_VAL_1 + pand %%TMP4, %%TMPARG_TAB_VAL_2 + pand %%TMP5, %%TMPARG_TAB_VAL_3 + pand %%TMP6, %%TMPARG_TAB_VAL_4 + pand %%TMP2, %%TMPARG_TAB_VAL_5 + + por %%TMP1, %%TMP3 + por %%TMP4, %%TMP5 + por %%TMP2, %%TMP6 + movdqa %%OUT_SUBSTITUTE_VAL, %%TMP1 + por %%OUT_SUBSTITUTE_VAL, %%TMP4 + + ;; %%OUT_SUBSTITUTE_VAL & %%TMP2 carry current OR result. + + movdqa %%TMP1, %%IN_INDEXES_H + movdqa %%TMP3, %%IN_INDEXES_H + movdqa %%TMP4, %%IN_INDEXES_H + movdqa %%TMP5, %%IN_INDEXES_H + movdqa %%TMP6, %%IN_INDEXES_H + + pcmpeqb %%TMP1, [rel idx_rows_sse + (6 * 16)] + movdqa %%TMPARG_TAB_VAL_0, [rel snow3g_inv_SR_SQ + (6 * 16)] + pcmpeqb %%TMP3, [rel idx_rows_sse + (7 * 16)] + movdqa %%TMPARG_TAB_VAL_1, [rel snow3g_inv_SR_SQ + (7 * 16)] + pcmpeqb %%TMP4, [rel idx_rows_sse + (8 * 16)] + movdqa %%TMPARG_TAB_VAL_2, [rel snow3g_inv_SR_SQ + (8 * 16)] + pcmpeqb %%TMP5, [rel idx_rows_sse + (9 * 16)] + movdqa %%TMPARG_TAB_VAL_3, [rel snow3g_inv_SR_SQ + (9 * 16)] + pcmpeqb %%TMP6, [rel idx_rows_sse + (10 * 16)] + movdqa %%TMPARG_TAB_VAL_4, [rel snow3g_inv_SR_SQ + (10 * 16)] + + pshufb %%TMPARG_TAB_VAL_0, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_1, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_2, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_3, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_4, %%TMP_INDEXES_L + + pand %%TMP1, %%TMPARG_TAB_VAL_0 + pand %%TMP3, %%TMPARG_TAB_VAL_1 + pand %%TMP4, %%TMPARG_TAB_VAL_2 + pand %%TMP5, %%TMPARG_TAB_VAL_3 + pand %%TMP6, %%TMPARG_TAB_VAL_4 + + por %%TMP1, %%TMP3 + por %%TMP4, %%TMP5 + por %%TMP2, %%TMP6 + por %%OUT_SUBSTITUTE_VAL, %%TMP1 + por %%TMP2, %%TMP4 + + ;; %%OUT_SUBSTITUTE_VAL & %%TMP1 carry current OR result + + movdqa %%TMP1, %%IN_INDEXES_H + movdqa %%TMP3, %%IN_INDEXES_H + movdqa %%TMP4, %%IN_INDEXES_H + movdqa %%TMP5, %%IN_INDEXES_H + movdqa %%TMP6, %%IN_INDEXES_H + + pcmpeqb %%TMP1, [rel idx_rows_sse + (11 * 16)] + movdqa %%TMPARG_TAB_VAL_0, [rel snow3g_inv_SR_SQ + (11 * 16)] + pcmpeqb %%TMP3, [rel idx_rows_sse + (12 * 16)] + movdqa %%TMPARG_TAB_VAL_1, [rel snow3g_inv_SR_SQ + (12 * 16)] + pcmpeqb %%TMP4, [rel idx_rows_sse + (13 * 16)] + movdqa %%TMPARG_TAB_VAL_2, [rel snow3g_inv_SR_SQ + (13 * 16)] + pcmpeqb %%TMP5, [rel idx_rows_sse + (14 * 16)] + movdqa %%TMPARG_TAB_VAL_3, [rel snow3g_inv_SR_SQ + (14 * 16)] + pcmpeqb %%TMP6, [rel idx_rows_sse + (15 * 16)] + movdqa %%TMPARG_TAB_VAL_4, [rel snow3g_inv_SR_SQ + (15 * 16)] + + pshufb %%TMPARG_TAB_VAL_0, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_1, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_2, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_3, %%TMP_INDEXES_L + pshufb %%TMPARG_TAB_VAL_4, %%TMP_INDEXES_L + + pand %%TMP1, %%TMPARG_TAB_VAL_0 + pand %%TMP3, %%TMPARG_TAB_VAL_1 + pand %%TMP4, %%TMPARG_TAB_VAL_2 + pand %%TMP5, %%TMPARG_TAB_VAL_3 + pand %%TMP6, %%TMPARG_TAB_VAL_4 + + por %%TMP1, %%TMP3 + por %%TMP4, %%TMP5 + por %%TMP2, %%TMP6 + por %%OUT_SUBSTITUTE_VAL, %%TMP1 + por %%TMP2, %%TMP4 + por %%OUT_SUBSTITUTE_VAL, %%TMP2 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SNOW3G S2 box calculation for 4 32-bit values passed in 1st input parameter. +;; Clobbers all 15 input xmm registers +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro S2_BOX_SSE 15 +%define %%TMP1 %1 ;; [in/clobbered] xmm containing 4 dwords +%define %%TMP2 %2 ;; [clobbered] temp xmm register +%define %%TMP3 %3 ;; [clobbered] temp xmm register +%define %%TMP4 %4 ;; [clobbered] temp xmm register +%define %%TMP5 %5 ;; [clobbered] temp xmm register +%define %%TMP6 %6 ;; [clobbered] temp xmm register +%define %%TMP7 %7 ;; [clobbered] temp xmm register +%define %%TMP8 %8 ;; [clobbered] temp xmm register +%define %%TMP9 %9 ;; [clobbered] temp xmm register +%define %%TMP10 %10 ;; [clobbered] temp xmm register +%define %%TMP11 %11 ;; [clobbered] temp xmm register +%define %%TMP12 %12 ;; [clobbered] temp xmm register +%define %%TMP13 %13 ;; [clobbered] temp xmm register +%define %%TMP14 %14 ;; [clobbered] temp xmm register +%define %%TMPOUT %15 ;; [out] xmm containing S2 box for 4 input dwords + + ;; Perform invSR(SQ(x)) transform + SSE_LOOKUP_16X8BIT %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, \ + %%TMP11, %%TMP12, %%TMP13, %%TMP14, \ + %%TMPOUT + + pshufb %%TMPOUT, [rel const_fixed_rotate_mask] + pxor %%TMP1, %%TMP1 + movdqa %%TMP2, %%TMPOUT + + ;; aesenclast does not perform mix column operation and + ;; allows to determine the fix-up value to be applied + ;; on result of aesenc to produce correct result for SNOW3G + aesenclast %%TMP2, %%TMP1 + aesenc %%TMPOUT, %%TMP1 + + ;; Using signed compare to return 0xFF when the most significant bit of + ;; no_mixc is set + pcmpgtb %%TMP1, %%TMP2 + movdqa %%TMP5, %%TMP1 + pshufb %%TMP5, [rel const_byte_mix_col_rev] + pxor %%TMP1, %%TMP5 + pand %%TMP1, [rel const_fixup_mask] + pxor %%TMPOUT, %%TMP1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Perform SNOW3G FSM clock operation for 4 buffers. +;; Passed addresses for FSM_R1-FSM_R3 and LFSR_5 are interpreted as list of 4 +;; 32-bit values each. +;; Values under FSM_R1-FSM_R3 are updated as a result of this macro. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_FSM_CLOCK 20 +%define %%FSM_R1 %1 ;; [in] address of 4 FSM values R1 +%define %%FSM_R2 %2 ;; [in] address of 4 FSM values R2 +%define %%FSM_R3 %3 ;; [in] address of 4 FSM values R3 +%define %%TMP1 %4 ;; [clobbered] temp xmm register +%define %%TMP2 %5 ;; [clobbered] temp xmm register +%define %%TMP3 %6 ;; [clobbered] temp xmm register +%define %%TMP4 %7 ;; [clobbered] temp xmm register +%define %%TMP5 %8 ;; [clobbered] temp xmm register +%define %%TMP6 %9 ;; [clobbered] temp xmm register +%define %%TMP7 %10 ;; [clobbered] temp xmm register +%define %%TMP8 %11 ;; [clobbered] temp xmm register +%define %%TMP9 %12 ;; [clobbered] temp xmm register +%define %%TMP10 %13 ;; [clobbered] temp xmm register +%define %%TMP11 %14 ;; [clobbered] temp xmm register +%define %%TMP12 %15 ;; [clobbered] temp xmm register +%define %%TMP13 %16 ;; [clobbered] temp xmm register +%define %%TMP14 %17 ;; [clobbered] temp xmm register +%define %%TMP15 %18 ;; [clobbered] temp xmm register +%define %%TMP16 %19 ;; [clobbered] temp xmm register +%define %%LFSR_5 %20 ;; [in] address of 4 LFSR 5 values + + movdqa %%TMP1, [ %%FSM_R2 ] + + ;; FSM_3 = S2_box(FSM_2) + S2_BOX_SSE %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, \ + %%TMP11, %%TMP12, %%TMP13, %%TMP14, %%TMP15 + ;; R = (FSM_R3 ^ LFSR_5) + FSM_R2 + movdqa %%TMP16, [%%FSM_R3] + pxor %%TMP16, [%%LFSR_5] + paddd %%TMP16, [%%FSM_R2] + + ;; FSM_3 = S2_box(FSM_2) + movdqa [%%FSM_R3], %%TMP15 + + ;; FSM_R2 = S1_box(FSM_R1) + movdqa %%TMP3, [%%FSM_R1] + + ;; S1 box calculation + pshufb %%TMP3, [rel const_fixed_rotate_mask] + pxor %%TMP2, %%TMP2 + aesenc %%TMP3, %%TMP2 + + movdqa [%%FSM_R2], %%TMP3 + + ;; FSM_1 = R + movdqa [%%FSM_R1], %%TMP16 + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Perform SNOW3G mul_alpha or div_alpha depending on table passed in %2: +;; (MULxPOW(c, 23, 0xA9) || MULxPOW(c, 245, 0xA9) || MULxPOW(c, 48, 0xA9) +;; || MULxPOW(c, 239, 0xA9)) +;; c = %%LFSR_X +;; Result of mul_alpha and div_alpha operations are precalculated and expected +;; under %%OP_TABLE address. This function searches those tables. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro ALPHA_OP 7 +%define %%LFSR_X %1 ;; [in/clobbered] xmm with LFSR value +%define %%OP_TABLE %2 ;; [in] address of mulalpha/divalpha val table +%define %%TMP1 %3 ;; [out] temporary xmm register +%define %%TMP2 %4 ;; [clobbered] temporary xmm register +%define %%TMP3 %5 ;; [clobbered] temporary xmm register +%define %%TMP4 %6 ;; [clobbered] temporary xmm register +%define %%TMP5 %7 ;; [clobbered] temporary xmm register + + movdqa %%TMP2, [rel low_nibble_byte_mask] + pand %%TMP2, %%LFSR_X ;; lower part of each byte of LFSR + movdqa %%TMP1, [rel %%OP_TABLE] + pshufb %%TMP1, %%TMP2 + movdqa %%TMP3, [rel %%OP_TABLE + 16] + movdqa %%TMP4, [rel %%OP_TABLE + 32] + movdqa %%TMP5, [rel %%OP_TABLE + 48] + + pshufb %%TMP3, %%TMP2 + pshufb %%TMP4, %%TMP2 + pshufb %%TMP5, %%TMP2 + + punpcklbw %%TMP1, %%TMP3 + punpcklbw %%TMP4, %%TMP5 + movdqa %%TMP2, %%TMP1 + punpcklwd %%TMP2, %%TMP4 + + movdqa %%TMP1, [rel low_nibble_byte_mask] + psrld %%LFSR_X, 4 + pand %%LFSR_X, %%TMP1 + + movdqa %%TMP1, [rel %%OP_TABLE + 64] + movdqa %%TMP3, [rel %%OP_TABLE + 80] + movdqa %%TMP4, [rel %%OP_TABLE + 96] + movdqa %%TMP5, [rel %%OP_TABLE + 112] + + pshufb %%TMP1, %%LFSR_X + pshufb %%TMP3, %%LFSR_X + pshufb %%TMP4, %%LFSR_X + pshufb %%TMP5, %%LFSR_X + + punpcklbw %%TMP1, %%TMP3 + punpcklbw %%TMP4, %%TMP5 + punpcklwd %%TMP1, %%TMP4 + pxor %%TMP1, %%TMP2 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Perform SNOW3G LFSR shift operation. +;; This operation is common for initialization mode and keystream mode, the only +;; difference is in init mode %1 = keystream otherwise %1 needs ti be set to 0. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SHIFT_LFSRS 9 +%define %%STATE %1 ;; [in] state ptr +%define %%KEYSTREAM %2 ;; [in] in init mode keystream, else 0 +%define %%TMP1 %3 ;; [clobbered] temporary xmm register +%define %%TMP2 %4 ;; [clobbered] temporary xmm register +%define %%TMP3 %5 ;; [clobbered] temporary xmm register +%define %%TMP4 %6 ;; [clobbered] temporary xmm register +%define %%TMP5 %7 ;; [clobbered] temporary xmm register +%define %%TMP6 %8 ;; [clobbered] temporary xmm register +%define %%TMP7 %9 ;; [clobbered] temporary xmm register + + ;; LFSR_0:LFSR_15: LFSR_i = LFSR_(i + 1); + ;; LFSR_15 = keystream / 0 + movdqa %%TMP1, [%%STATE + _snow3g_args_LFSR_1] + movdqa %%TMP2, [%%STATE + _snow3g_args_LFSR_2] + movdqa %%TMP3, [%%STATE + _snow3g_args_LFSR_3] + movdqa %%TMP4, [%%STATE + _snow3g_args_LFSR_4] + movdqa %%TMP5, [%%STATE + _snow3g_args_LFSR_5] + movdqa %%TMP6, [%%STATE + _snow3g_args_LFSR_6] + movdqa [%%STATE + _snow3g_args_LFSR_0], %%TMP1 + movdqa [%%STATE + _snow3g_args_LFSR_1], %%TMP2 + movdqa [%%STATE + _snow3g_args_LFSR_2], %%TMP3 + movdqa [%%STATE + _snow3g_args_LFSR_3], %%TMP4 + movdqa [%%STATE + _snow3g_args_LFSR_4], %%TMP5 + movdqa [%%STATE + _snow3g_args_LFSR_5], %%TMP6 + + movdqa %%TMP1, [%%STATE + _snow3g_args_LFSR_7] + movdqa %%TMP2, [%%STATE + _snow3g_args_LFSR_8] + movdqa %%TMP3, [%%STATE + _snow3g_args_LFSR_9] + movdqa %%TMP4, [%%STATE + _snow3g_args_LFSR_10] + movdqa %%TMP5, [%%STATE + _snow3g_args_LFSR_11] + movdqa %%TMP6, [%%STATE + _snow3g_args_LFSR_12] + movdqa [%%STATE + _snow3g_args_LFSR_6], %%TMP1 + movdqa [%%STATE + _snow3g_args_LFSR_7], %%TMP2 + movdqa [%%STATE + _snow3g_args_LFSR_8], %%TMP3 + movdqa [%%STATE + _snow3g_args_LFSR_9 ], %%TMP4 + movdqa [%%STATE + _snow3g_args_LFSR_10], %%TMP5 + movdqa [%%STATE + _snow3g_args_LFSR_11], %%TMP6 + + movdqa %%TMP1, [%%STATE + _snow3g_args_LFSR_13] + movdqa %%TMP2, [%%STATE + _snow3g_args_LFSR_14] + movdqa %%TMP3, [%%STATE + _snow3g_args_LFSR_15] + + movdqa [%%STATE + _snow3g_args_LFSR_12], %%TMP1 + movdqa [%%STATE + _snow3g_args_LFSR_13], %%TMP2 + movdqa [%%STATE + _snow3g_args_LFSR_14], %%TMP3 + + movdqa [%%STATE + _snow3g_args_LFSR_15], %%KEYSTREAM +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Generate SNOW3G keystream per 4 buffers. Update LFSR/FSM state. +;; This macro is used both in initialization and keystream modes. +;; In initialization mode F is stored on stack. +;; In keystream mode keystream is stored on stack. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_KEY_GEN_SSE 18 +%define %%STATE %1 ;; [in] ptr to LFSR/FSM struct +%define %%TMP1 %2 ;; [clobbered] temporary xmm register +%define %%TMP2 %3 ;; [clobbered] temporary xmm register +%define %%TMP3 %4 ;; [clobbered] temporary xmm register +%define %%TMP4 %5 ;; [clobbered] temporary xmm register +%define %%TMP5 %6 ;; [clobbered] temporary xmm register +%define %%TMP6 %7 ;; [clobbered] temporary xmm register +%define %%TMP7 %8 ;; [clobbered] temporary xmm register +%define %%TMP8 %9 ;; [clobbered] temporary xmm register +%define %%TMP9 %10 ;; [clobbered] temporary xmm register +%define %%TMP10 %11 ;; [clobbered] temporary xmm register +%define %%TMP11 %12 ;; [clobbered] temporary xmm register +%define %%TMP12 %13 ;; [clobbered] temporary xmm register +%define %%TMP13 %14 ;; [clobbered] temporary xmm register +%define %%TMP14 %15 ;; [clobbered] temporary xmm register +%define %%TMP15 %16 ;; [out] generated keystream +%define %%TMP16 %17 ;; [out] generated keystream +%define %%DWORD_ITER %18 ;; [in] offset for stack storing keystream + + ;; Calculate F = (LFSR_S[15] + FSM_R1) ^ FSM_R2; + movdqa %%TMP1, [ %%STATE + _snow3g_args_LFSR_15 ] + paddd %%TMP1, [ %%STATE + _snow3g_args_FSM_1 ] + pxor %%TMP1, [ %%STATE + _snow3g_args_FSM_2 ] + + ;; Store F/keystream on stack + movdqa %%TMP2, [%%STATE + _snow3g_args_LD_ST_MASK + 4*4] + pandn %%TMP2, %%TMP1 ;; zero in keystream mode + movdqa %%TMP3, [%%STATE + _snow3g_args_LD_ST_MASK + 4*4] + ;; keystream mode: ks = F xor LFSR_0 + pxor %%TMP1, [%%STATE + _snow3g_args_LFSR_0] + pand %%TMP3, %%TMP1 ;; zero in init mode + por %%TMP2, %%TMP3 + movdqa [rsp + _keystream + %%DWORD_ITER * 16], %%TMP2 + + ;; FSM Clock + SNOW3G_FSM_CLOCK {%%STATE + _snow3g_args_FSM_1}, \ + {%%STATE + _snow3g_args_FSM_2}, \ + {%%STATE + _snow3g_args_FSM_3}, %%TMP1, %%TMP2, \ + %%TMP3, %%TMP4, %%TMP5, %%TMP6, %%TMP7, \ + %%TMP8, %%TMP9, %%TMP10, %%TMP11, %%TMP12, \ + %%TMP13, %%TMP14, %%TMP15, %%TMP16, \ + %%STATE + _snow3g_args_LFSR_5 + + ;; LFSR clock: mul alpha + movdqa %%TMP15, [ %%STATE + _snow3g_args_LFSR_0 ] + movdqa %%TMP2, [rel ms_byte_mask] + pshufb %%TMP15, %%TMP2 + ALPHA_OP %%TMP15, mul_alpha, %%TMP2, %%TMP3, %%TMP4, \ + %%TMP5, %%TMP6 + + ;; LFSR clock: div alpha + movdqa %%TMP15, [%%STATE + _snow3g_args_LFSR_11 ] + movdqa %%TMP7, [rel ls_byte_mask] + pshufb %%TMP15, %%TMP7 + ALPHA_OP %%TMP15, div_alpha, %%TMP7, %%TMP3, %%TMP4, \ + %%TMP5, %%TMP6 + + movdqa %%TMP15, [%%STATE + _snow3g_args_LFSR_2 ] + pxor %%TMP15, %%TMP2 + pxor %%TMP15, %%TMP7 + + movdqa %%TMP3, [%%STATE + _snow3g_args_LFSR_0 ] + movdqa %%TMP4, [%%STATE + _snow3g_args_LFSR_11 ] + pslld %%TMP3, 8 + psrld %%TMP4, 8 + pxor %%TMP15, %%TMP3 + pxor %%TMP15, %%TMP4 + + ;; in init mode mask is 0, so this is applies only in init mode + movdqa %%TMP2, [%%STATE + _snow3g_args_LD_ST_MASK + 4*4] + pandn %%TMP2, [rsp + _keystream + %%DWORD_ITER * 16] + pxor %%TMP15, %%TMP2 + + SHIFT_LFSRS %%STATE, %%TMP15, %%TMP1, %%TMP2, %%TMP3, \ + %%TMP4, %%TMP5, %%TMP6, %%TMP7 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Read and transpose keystreams from stack +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro TRANSPOSE_4X32 6 +%define %%OUT_XMM_LANE_0 %1 ;; [out] 128bit keystream for lane 0 +%define %%OUT_XMM_LANE_1 %2 ;; [out] 128bit keystream for lane 1 +%define %%OUT_XMM_LANE_2 %3 ;; [out] 128bit keystream for lane 0 +%define %%OUT_XMM_LANE_3 %4 ;; [out] 128bit keystream for lane 3 +%define %%TMPXMM %5 ;; [clobbered] temporary xmm register +%define %%TMPXMM_1 %6 ;; [clobbered] temporary xmm register + + movdqa %%OUT_XMM_LANE_2, [rsp + _keystream + 0 * 16] + movdqa %%OUT_XMM_LANE_1, [rsp + _keystream + 1 * 16] + movdqa %%TMPXMM_1, [rsp + _keystream + 2 * 16] + movdqa %%OUT_XMM_LANE_3, [rsp + _keystream + 3 * 16] + + ;; output looks like: {t0 r1 r0 r3} + TRANSPOSE4_U32 %%OUT_XMM_LANE_2, %%OUT_XMM_LANE_1, %%TMPXMM_1, \ + %%OUT_XMM_LANE_3, %%OUT_XMM_LANE_0, %%TMPXMM + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Checks the masks for init/keystream phases and outputs keystream from stack +;; xored with input, depending on given lane, if phase is keystream. +;; Input arguments and size of output data is controlled by %SIZE: +;; - 16: 8 arguments, output full xmm value passed by %8 +;; - 4: 7 arguments, output exactly 1 DW +;; - other: 7 arguments, output 1-4 bytes depending on %7 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_OUTPUT 7-10 +%define %%SIZE %1 ;; [in] size indicating nr of bytes to output +%define %%STATE %2 ;; [in] ptr to LFSR/FSM struct +%define %%LANE %3 ;; [in] lane nr +%define %%TMP64 %4 ;; [clobbered] r64 gp reg temp +%define %%IN_PTR %5 ;; [clobbered] r64 gp reg temp +%define %%OUT_PTR %6 ;; [clobbered] r64 gp reg temp +%define %%TMP %7 ;; [in] temp xmm, if size is 16 +%define %%LENGTH %7 ;; [clobbered] r64 gp reg temp, if size is not 16 +%define %%VALUE %8 ;; [in] xmm_val, if size is 16 + + + ;; Check if phase for given lane + mov DWORD(%%TMP64), \ + [%%STATE + _snow3g_args_LD_ST_MASK + %%LANE*4] + or DWORD(%%TMP64),DWORD(%%TMP64) + je %%no_output ;; skip output if in init phase + ;; read in/out ptrs + mov %%IN_PTR, [%%STATE + _snow3g_args_in + %%LANE * 8] + mov %%OUT_PTR, [%%STATE + _snow3g_args_out + %%LANE * 8] + + ;; output == input XOR keysteram +%ifidn %%SIZE, 16 + movdqu %%TMP, [%%IN_PTR] + pshufb %%VALUE, [rel xmm_bswap] + pxor %%VALUE, %%TMP + movdqu [%%OUT_PTR], %%VALUE + add %%IN_PTR, %%SIZE + add %%OUT_PTR, %%SIZE +%else + ;; there is always at least 1DW of keystream generated on stack + mov DWORD(%%TMP64), [rsp + _keystream + %%LANE*4] + bswap DWORD(%%TMP64) +%ifidn %%SIZE, 4 + xor DWORD(%%TMP64), [%%IN_PTR] + mov [%%OUT_PTR], DWORD(%%TMP64) +%else ;; up to 4 bytes (defined by %%length) + mov DWORD(%%LENGTH), [%%STATE + _snow3g_lens + %%LANE * 4] + cmp %%LENGTH, 4 + jne %%_not_dw + xor DWORD(%%TMP64), [%%IN_PTR] + mov dword [%%OUT_PTR], DWORD(%%TMP64) + jmp %%_write_done +%%_not_dw: + cmp %%LENGTH, 2 + jl %%_write_single_byte + + ;; write 2 bytes + xor WORD(%%TMP64), [%%IN_PTR] + mov word [%%OUT_PTR], WORD(%%TMP64) + add %%IN_PTR, 2 + add %%OUT_PTR, 2 + and %%LENGTH, 1 + je %%_write_done + +%%_write_single_byte: + xor BYTE(%%TMP64), [%%IN_PTR] + mov byte [%%OUT_PTR], BYTE(%%TMP64) +%%_write_done: +%endif + ;; if %%LENGTH is less then 4 bytes per given lane that's the last bytes + ;; of total request so pointers are never used again + add %%IN_PTR, 4 + add %%OUT_PTR, 4 +%endif + ;; Update input/output pointers + mov [%%STATE + _snow3g_args_in + %%LANE*8], %%IN_PTR + mov [%%STATE + _snow3g_args_out + %%LANE*8], %%OUT_PTR + +%%no_output: +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Initialize LFSR, FSM registers and write mask for given lane +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_INIT_LANE_SSE 7 +%define %%STATE %1 ;; [in] ptr to MB_MGR_SNOW3G_OOO structure +%define %%LANE %2 ;; [in] nr of lane initialize data in +%define %%P_KEY %3 ;; [in] ptr to key +%define %%P_IV %4 ;; [in*] ptr to IV copy (IV bytes swapped in each dw) +%define %%TMPXMM_1 %5 ;; [clobbered] temporary xmm reg +%define %%TMPXMM_2 %6 ;; [clobbered] temporary xmm reg +%define %%TMPXMM_3 %7 ;; [clobbered] temporary xmm reg + + movd %%TMPXMM_1, [%%P_KEY] ;; key + movdqa %%TMPXMM_2, %%TMPXMM_1 + pxor %%TMPXMM_2, [rel all_fs] ;; ~key + + movdqu %%TMPXMM_3, [%%P_IV] + pshufb %%TMPXMM_3, [rel xmm_bswap] + + ;; temporarily store swapped IV on stack + movdqu [rsp + _keystream], %%TMPXMM_3 + + ;; LFSR initialisation + movd [%%STATE + _snow3g_args_LFSR_0 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_8 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_4 + 4*%%LANE], %%TMPXMM_1 + movd %%TMPXMM_3, [rsp + _keystream + 8] + pxor %%TMPXMM_1, %%TMPXMM_3 ;; LFSR_12 ^= IV[2](swapped) + movd [%%STATE + _snow3g_args_LFSR_12 + 4*%%LANE], %%TMPXMM_1 + + movd %%TMPXMM_1, [%%P_KEY + 4] + movd %%TMPXMM_2, [%%P_KEY + 4] + pxor %%TMPXMM_2, [rel all_fs] + + movd [%%STATE + _snow3g_args_LFSR_1 + 4*%%LANE], %%TMPXMM_2 + movd %%TMPXMM_3, [rsp + _keystream] + pxor %%TMPXMM_2, %%TMPXMM_3 ;; LFSR_9 ^= IV[0](swapped) + movd [%%STATE + _snow3g_args_LFSR_9 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_5 + 4*%%LANE], %%TMPXMM_1 + movd [%%STATE + _snow3g_args_LFSR_13 + 4*%%LANE], %%TMPXMM_1 + + movd %%TMPXMM_1, [%%P_KEY + 8] + movd %%TMPXMM_2, [%%P_KEY + 8] + pxor %%TMPXMM_2, [rel all_fs] + + movd [%%STATE + _snow3g_args_LFSR_2 + 4*%%LANE], %%TMPXMM_2 + movd %%TMPXMM_3, [rsp + _keystream + 4] + pxor %%TMPXMM_2, %%TMPXMM_3 ;; LFSR_10 ^= IV[1](swapped) + movd [%%STATE + _snow3g_args_LFSR_10 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_6 + 4*%%LANE], %%TMPXMM_1 + movd [%%STATE + _snow3g_args_LFSR_14 + 4*%%LANE], %%TMPXMM_1 + + movd %%TMPXMM_1, [%%P_KEY + 12] + movdqa %%TMPXMM_2, %%TMPXMM_1 + pxor %%TMPXMM_2, [rel all_fs] + + movd [%%STATE + _snow3g_args_LFSR_3 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_11 + 4*%%LANE], %%TMPXMM_2 + movd [%%STATE + _snow3g_args_LFSR_7 + 4*%%LANE], %%TMPXMM_1 + movd %%TMPXMM_3, [rsp + _keystream + 12] + pxor %%TMPXMM_1, %%TMPXMM_3 ;; LFSR_15 ^= IV[3](swapped) + movd [%%STATE + _snow3g_args_LFSR_15 + 4*%%LANE], %%TMPXMM_1 + + ; FSM initialization: FSM_1 = FSM_2 = FSM_3 = 0 + pxor %%TMPXMM_1, %%TMPXMM_1 + movd [%%STATE + _snow3g_args_FSM_1 + 4*%%LANE], %%TMPXMM_1 + movd [%%STATE + _snow3g_args_FSM_2 + 4*%%LANE], %%TMPXMM_1 + movd [%%STATE + _snow3g_args_FSM_3 + 4*%%LANE], %%TMPXMM_1 + + movd [%%STATE + _snow3g_args_LD_ST_MASK + 4*%%LANE], %%TMPXMM_1 + movd [%%STATE + _snow3g_args_LD_ST_MASK + 16+ 4*%%LANE], %%TMPXMM_1 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Perform SNOW3G encrypt/decrypt operation steps for 4 buffers. Generate number +;; of dwords indicated by %%COMMON_LEN, update LFSR, FSM state. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_ENC_DEC 23 +%define %%STATE %1 ;; [in] ptr to LFSR/FSM struct +%define %%COMMON_LEN %2 ;; [in/clobbered] dw aligned common length +%define %%IN %3 ;; [clobbered] r64 gp reg temp +%define %%OUT %4 ;; [clobbered] r64 gp reg temp +%define %%LENGTH %5 ;; [clobbered] r64 gp reg temp +%define %%TMP1_64 %6 ;; [clobbered] r64 gp reg temp +%define %%TMP2_64 %7 ;; [clobbered] r64 gp reg temp +%define %%TMP1 %8 ;; [clobbered] temporary xmm register +%define %%TMP2 %9 ;; [clobbered] temporary xmm register +%define %%TMP3 %10 ;; [clobbered] temporary xmm register +%define %%TMP4 %11 ;; [clobbered] temporary xmm register +%define %%TMP5 %12 ;; [clobbered] temporary xmm register +%define %%TMP6 %13 ;; [clobbered] temporary xmm register +%define %%TMP7 %14 ;; [clobbered] temporary xmm register +%define %%TMP8 %15 ;; [clobbered] temporary xmm register +%define %%TMP9 %16 ;; [clobbered] temporary xmm register +%define %%TMP10 %17 ;; [clobbered] temporary xmm register +%define %%TMP11 %18 ;; [clobbered] temporary xmm register +%define %%TMP12 %19 ;; [clobbered] temporary xmm register +%define %%TMP13 %20 ;; [clobbered] temporary xmm register +%define %%TMP14 %21 ;; [clobbered] temporary xmm register +%define %%TMP15 %22 ;; [clobbered] temporary xmm register +%define %%TMP16 %23 ;; [clobbered] temporary xmm register + + sub %%COMMON_LEN, 1 + mov %%LENGTH, %%COMMON_LEN + + shr %%LENGTH, 2 + je %%no_dqws + +%%next_dqw: +%assign i 0 +%rep 4 + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, i + +%assign i (i+1) +%endrep + + TRANSPOSE_4X32 %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, %%TMP6 + SNOW3G_OUTPUT 16, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%TMP5, %%TMP1 + SNOW3G_OUTPUT 16, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%TMP5, %%TMP2 + SNOW3G_OUTPUT 16, %%STATE, 2, %%IN, %%OUT, %%TMP1_64, %%TMP5, %%TMP3 + SNOW3G_OUTPUT 16, %%STATE, 3, %%IN, %%OUT, %%TMP1_64, %%TMP5, %%TMP4 + + sub %%LENGTH, 1 + jne %%next_dqw + +%%no_dqws: + and %%COMMON_LEN, 0x3 + cmp %%COMMON_LEN, 0 + + je %%no_full_dws_to_write_out + +%%next_dw: + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + + SNOW3G_OUTPUT 4, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 + SNOW3G_OUTPUT 4, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 + SNOW3G_OUTPUT 4, %%STATE, 2, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 + SNOW3G_OUTPUT 4, %%STATE, 3, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 + + sub %%COMMON_LEN, 1 + jne %%next_dw + +%%no_full_dws_to_write_out: + ;; Process last dw/bytes: + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + + SNOW3G_OUTPUT 3, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%LENGTH + SNOW3G_OUTPUT 3, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%LENGTH + SNOW3G_OUTPUT 3, %%STATE, 2, %%IN, %%OUT, %%TMP1_64, %%LENGTH + SNOW3G_OUTPUT 3, %%STATE, 3, %%IN, %%OUT, %%TMP1_64, %%LENGTH + +%%zero_bytes: +%endmacro + +mksection stack-noexec -- GitLab From 80688b8e7ac7c34b1ca5d0329820db231ef7d36e Mon Sep 17 00:00:00 2001 From: Kamila Lipinska Date: Thu, 2 Jun 2022 14:16:14 +0000 Subject: [PATCH 144/369] sse: [SNOW3G-UEA2] Integrate into OOO manager --- README | 2 +- README.md | 2 +- lib/Makefile | 3 +- ...mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 426 ++++++++++++++++++ lib/sse/mb_mgr_sse.c | 54 +++ lib/sse/snow3g_sse.c | 4 + lib/win_x64.mak | 2 + 7 files changed, 490 insertions(+), 3 deletions(-) create mode 100644 lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm diff --git a/README b/README index 96061bf7..73e60732 100644 --- a/README +++ b/README @@ -90,7 +90,7 @@ Table 1. List of supported cipher algorithms and their implementations. | KASUMI-F8 | Y | N | N | N | N | N | | ZUC-EEA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | | ZUC-EEA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | - | SNOW3G-UEA2 | N | Y | Y | Y | Y x16 | Y x16 | + | SNOW3G-UEA2 | N | Y x4 | Y | Y | Y x16 | Y x16 | | AES128-CBCS(9) | N | Y(1) | Y(3) | N | N | Y(6) | | Chacha20 | N | Y | Y | Y | Y | N | | Chacha20 AEAD | N | Y | Y | Y | Y | N | diff --git a/README.md b/README.md index 88775b74..cebef20a 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Table 1. List of supported cipher algorithms and their implementations. | KASUMI-F8 | Y | N | N | N | N | N | | ZUC-EEA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | | ZUC-EEA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | -| SNOW3G-UEA2 | N | Y | Y | Y | Y x16 | Y x16 | +| SNOW3G-UEA2 | N | Y x4 | Y | Y | Y x16 | Y x16 | | AES128-CBCS(9) | N | Y(1) | Y(3) | N | N | Y(6) | | Chacha20 | N | Y | Y | Y | Y | N | | Chacha20 AEAD | N | Y | Y | Y | Y | N | diff --git a/lib/Makefile b/lib/Makefile index ea5e1b1d..9eea98d4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -452,6 +452,7 @@ asm_sse_lib_objs := \ mb_mgr_hmac_sha256_submit_ni_sse.o \ mb_mgr_zuc_submit_flush_sse.o \ mb_mgr_zuc_submit_flush_gfni_sse.o \ + mb_mgr_snow3g_uea2_submit_flush_x4_sse.o \ ethernet_fcs_sse.o \ crc16_x25_sse.o \ crc32_sctp_sse.o \ @@ -464,7 +465,7 @@ asm_sse_lib_objs := \ crc32_iuup_sse.o \ crc32_wimax_sse.o \ chacha20_sse.o \ - memcpy_sse.o \ + memcpy_sse.o \ snow_v_sse.o \ snow3g_uia2_by4_sse.o diff --git a/lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm new file mode 100644 index 00000000..38392efe --- /dev/null +++ b/lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -0,0 +1,426 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "include/imb_job.asm" +%include "include/mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" +%include "include/cet.inc" +%include "sse/snow3g_uea2_by4_sse.asm" + +%define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_sse +%define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_sse + +mksection .rodata +default rel + +align 64 +last_3_bytes: +dd 0x00000003, 0x00000003, 0x00000003, 0x00000003 +align 64 +zero_xmm: +dd 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +mksection .text +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define tmp_gp1 rcx +%define tmp_gp2 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define tmp_gp1 rdi +%define tmp_gp2 rsi +%endif + +%define tmp_gp3 rbx +%define tmp_gp4 rbp +%define tmp_gp5 r9 +%define tmp_gp6 r10 +%define tmp_gp7 r11 +%define tmp_gp8 r12 + +%define state arg1 +%define job arg2 + +%define job_rax rax + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Get lane nr from ptr to the list of unused lanes. +;; Remove returned lane nr from the list +;; Increase lanes in use. +;; Put job ptr in appropriate lane field in state (arg %3) +;; Assumptions: +;; In (arg %1) single lane nr takes 4 bits and 1st free lane nr is lowest 4 bits +;; Job ptr in (arg %3) takes 8 bytes +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GET_UNUSED_LANE_SSE 6 +%define %%LANE_LIST %1 ;; [in] ptr to unused lane list +%define %%LANES_IN_USE %2 ;; [in] ptr to lanes in use count +%define %%JOB_LANES %3 ;; [in] ptr to list of jobs +%define %%JOB %4 ;; [in] ptr to job structure +%define %%LANE_NR %5 ;; [out] GP register to fill with unused lane nr +%define %%UNUSED_LANES %6 ;; [clobbered] GP register + + mov DWORD(%%UNUSED_LANES), dword [%%LANE_LIST] + mov %%LANE_NR, %%UNUSED_LANES + and %%LANE_NR, 0x3 + ;; remove picked lane nr from list of unused lanes + shr %%UNUSED_LANES, 4 + mov dword [%%LANE_LIST], DWORD(%%UNUSED_LANES) + + add word [%%LANES_IN_USE], 1 + mov [%%JOB_LANES + %%LANE_NR*8], %%JOB +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Find minimum value in table of 4 dwords +;; Outputs (%4) min value and (%5) position of that value in the table +;; Additionally (%2) contains list of lane lengths extracted from (%1) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GET_MIN_LENGTH_X4_SSE 4 +%define %%LANE_LENS_LIST_PTR %1 ;; [in] ptr to list of lane lengths +%define %%TEMP_64 %2 ;; [clobbered] tmp 64bit register +%define %%LENGTH %3 ;; [out] gp register to put min length in +%define %%INDEX %4 ;; [out] gp register to put index in + + mov DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*0] + xor %%INDEX, %%INDEX + mov %%TEMP_64, 1 + + cmp DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*1] + cmova DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*1] + cmova %%INDEX, %%TEMP_64 + inc %%TEMP_64 + + cmp DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*2] + cmova DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*2] + cmova %%INDEX, %%TEMP_64 + inc %%TEMP_64 + + cmp DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*3] + cmova DWORD(%%LENGTH), [%%LANE_LENS_LIST_PTR + 4*3] + cmova %%INDEX, %%TEMP_64 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Depending on %1: +;; submit: Submit single SNOW3G request to be later processed, setup masks and +;; initial FSM/LFSR state. After that, if there is full 4 requests +;; submitted proceed with flush operation. +;; flush: Do SNOW3G encrypt/decrypt processing for 4 buffers until one of them +;; is fully processed. Return job pointer corresponding to finished +;; request. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SUBMIT_FLUSH_JOB_SNOW3G_UEA2_SSE 25 +%define %%SUBMIT_FLUSH %1 ;; [in] submit/flush selector +%define %%UNUSED_LANES %2 ;; [clobbered] GP register +%define %%LANE %3 ;; [clobbered] GP register +%define %%TGP0 %4 ;; [clobbered] GP register +%define %%TGP1 %5 ;; [clobbered] GP register +%define %%TGP2 %6 ;; [clobbered] GP register +%define %%TGP3 %7 ;; [clobbered] GP register +%define %%TGP4 %8 ;; [clobbered] GP register +%define %%TGP5 %9 ;; [clobbered] GP register +%define %%TMP_XMM_0 %10 ;; [clobbered] xmm register +%define %%TMP_XMM_1 %11 ;; [clobbered] xmm register +%define %%TMP_XMM_2 %12 ;; [clobbered] xmm register +%define %%TMP_XMM_3 %13 ;; [clobbered] xmm register +%define %%TMP_XMM_4 %14 ;; [clobbered] xmm register +%define %%TMP_XMM_5 %15 ;; [clobbered] xmm register +%define %%TMP_XMM_6 %16 ;; [clobbered] xmm register +%define %%TMP_XMM_7 %17 ;; [clobbered] xmm register +%define %%TMP_XMM_8 %18 ;; [clobbered] xmm register +%define %%TMP_XMM_9 %19 ;; [clobbered] xmm register +%define %%TMP_XMM_10 %20 ;; [clobbered] xmm register +%define %%TMP_XMM_11 %21 ;; [clobbered] xmm register +%define %%TMP_XMM_12 %22 ;; [clobbered] xmm register +%define %%TMP_XMM_13 %23 ;; [clobbered] xmm register +%define %%TMP_XMM_14 %24 ;; [clobbered] xmm register +%define %%TMP_XMM_15 %25 ;; [clobbered] xmm register + + SNOW3G_FUNC_START + xor job_rax, job_rax ;; assume NULL return job + +%ifidn %%SUBMIT_FLUSH, submit + GET_UNUSED_LANE_SSE state + _snow3g_unused_lanes, \ + state + _snow3g_lanes_in_use, \ + state + _snow3g_job_in_lane, \ + job, %%LANE, %%UNUSED_LANES + + mov %%TGP1, [job + _enc_keys] + mov %%TGP2, [job + _iv] + + ;; --------------------------------------------------------------------- + ;; Initialize LFSR and FSM registers + ;; [LD_ST_MASK + 4*%%LANE] = 0 + ;; [LD_ST_MASK + 4*4+ 4*%%LANE] = 0 + ;; LD_ST_MASK field from state is later used to determine if any data + ;; should be read from src and written to dst. + ;; When mask is set to 0 so no reads/writes occur. + SNOW3G_INIT_LANE_SSE state, %%LANE, %%TGP1, %%TGP2, %%TMP_XMM_0, \ + %%TMP_XMM_1, %%TMP_XMM_2 + + ;; 32 iterations in Init mode are required + ;; details of _snow3g_lens dw fields are in FLUSH section + mov dword [state + _snow3g_lens + 32 + 4*%%LANE], 32 + mov dword [state + _snow3g_lens + 4*%%LANE], 4 + + mov %%TGP0, [job + _msg_len_to_cipher_in_bits] + shr %%TGP0, 3 + + mov dword [state + _snow3g_args_byte_length + %%LANE*4], DWORD(%%TGP0) + + mov %%TGP0, [job + _cipher_start_offset_in_bits] + shr %%TGP0, 3 ;; convert from bits to bytes (src & dst) + mov %%TGP1, [job + _dst] + add %%TGP1, %%TGP0 + add %%TGP0, [job + _src] + + mov [state + _snow3g_args_in + %%LANE*8], %%TGP0 + mov [state + _snow3g_args_out + %%LANE*8], %%TGP1 + + cmp word [state + _snow3g_lanes_in_use], 4 + jne %%return_uea2 + ;; if all lanes are busy fall through to %%process_job_uea2 + +%else ;; FLUSH + ;; --------------------------------------------------------------------- + ;; All lanes are busy or flush is called - process used lanes until + ;; one job is done. + ;; --------------------------------------------------------------------- + ;; Each of the lanes can be in any stage: INIT1, INIT2, KEYGEN, FINISHED + ;; and they can be processed in parallel by the algorithmic code. + ;; START -> INIT1 -> INIT2 -> KEYGEN -> COMPLETE + ;; --------------------------------------------------------------------- + ;; State of the job is identified with: + ;; _snow3g_args_LD_ST_MASK + ;; *dwords 4:7 - determines if INIT1 phase is done + ;; *dwords 0:3 - determines if lane is KEYGEN state + ;; -> yes: all bits set in dw per given lane + ;; -> no : set to 0 + ;; _snow3g_args_byte_length + ;; message lengths to be processed (bytes). Decreased appropriately + ;; if particular lane is in KEYGEN phase + ;; _snow3g_lens: + ;; *dwords 0:3 - indicating final 0-4 bytes to be outputted for the + ;; lane per SNOW3G_INIT_LANE_SSE macro call + ;; *dword 4 - common minimum length in double words (rounded up) + ;; *dwords 5:7 - unused + ;; *dwords 8:11 - length in dwords to be processed per lane in + ;; given processing phase(rounded up) + ;; --------------------------------------------------------------------- + cmp word [state + _snow3g_lanes_in_use], 0 + je %%return_uea2 +%endif + +%%_find_min: +%define ROUNDED_DW_LENS _snow3g_lens+32 +%define KEYGEN_STAGE _snow3g_args_LD_ST_MASK +%define INIT1_DONE _snow3g_args_LD_ST_MASK+16 + ;; Find minimum length. If lane is empty min length is set to 0xffffffff + GET_MIN_LENGTH_X4_SSE state + ROUNDED_DW_LENS, %%TGP1, %%TGP0, %%LANE + or %%TGP0, %%TGP0 + jz %%_len_is_0 + + movd %%TMP_XMM_0, DWORD(%%TGP0) + pshufd %%TMP_XMM_0, %%TMP_XMM_0, 0 + + movdqa %%TMP_XMM_5, [state + ROUNDED_DW_LENS] + pxor %%TMP_XMM_6, %%TMP_XMM_6 + pxor %%TMP_XMM_7, %%TMP_XMM_7 + + ;; Create mask with lanes in use + pcmpeqq %%TMP_XMM_6, [state + _snow3g_job_in_lane] + pcmpeqq %%TMP_XMM_7, [state + _snow3g_job_in_lane + 16] + pshufd %%TMP_XMM_6, %%TMP_XMM_6, 0x88 ;; lane order: 1,0,1,0 + pshufd %%TMP_XMM_7, %%TMP_XMM_7, 0x88 ;; lane order: 3,2,3,2 + pblendw %%TMP_XMM_6, %%TMP_XMM_7, 0xf0 + + pxor %%TMP_XMM_6, [rel all_fs] + pand %%TMP_XMM_6, %%TMP_XMM_0 + psubd %%TMP_XMM_5, %%TMP_XMM_6 + movdqa [state + ROUNDED_DW_LENS], %%TMP_XMM_5 + + pslld %%TMP_XMM_0, 2 + pand %%TMP_XMM_0, [state + KEYGEN_STAGE] + movdqa %%TMP_XMM_1, %%TMP_XMM_0 + + ;; Set all bits in dws where rounded dw length is bigger than original + ;; byte lengths + pcmpgtd %%TMP_XMM_1, [state + _snow3g_args_byte_length] + pand %%TMP_XMM_1, [state + _snow3g_args_byte_length] + ;; Write outstanding bytes to _snow3g_lens dwords [0:3] and adjust + ;; _snow3g_args_byte_length so after common dw length subtraction + ;; it is set to 0 + pand %%TMP_XMM_1, [rel last_3_bytes] + pcmpeqd %%TMP_XMM_1, [rel zero_xmm] + movdqa %%TMP_XMM_4, %%TMP_XMM_1 + pxor %%TMP_XMM_4, [rel all_fs] + pand %%TMP_XMM_4, [state+_snow3g_lens] + pand %%TMP_XMM_1, [state + _snow3g_args_byte_length] + por %%TMP_XMM_4, %%TMP_XMM_1 + movdqa [state + _snow3g_args_byte_length], %%TMP_XMM_4 + + movdqa %%TMP_XMM_3, [state+_snow3g_args_byte_length] + psubd %%TMP_XMM_3, %%TMP_XMM_0 + movdqa [state+_snow3g_args_byte_length], %%TMP_XMM_3 + ;; Do cipher / clock operation for all lanes and given common length + SNOW3G_ENC_DEC state, %%TGP0, %%TGP1, %%TGP2, %%TGP3, %%TGP4, %%TGP5, \ + %%TMP_XMM_0, %%TMP_XMM_1, %%TMP_XMM_2, %%TMP_XMM_3, \ + %%TMP_XMM_4, %%TMP_XMM_5, %%TMP_XMM_6, %%TMP_XMM_7, \ + %%TMP_XMM_8, %%TMP_XMM_9, %%TMP_XMM_10, %%TMP_XMM_11, \ + %%TMP_XMM_12, %%TMP_XMM_13, %%TMP_XMM_14, %%TMP_XMM_15 + + jmp %%_find_min + +%%_len_is_0: + ;; --------------------------------------------------------------------- + ;; 3 states are possible here for the lane with length 0: + ;; INIT1 done -> set DW length to 1 and update LD_ST_MASK + ;; INIT2 done -> set DW length to bytelength rounded up to dws and + ;; update LD_ST_MASK + ;; COMPLETED -> update length and return job + ;; check if the job is in one of INIT1 or INIT2 + ;; lane with len 0 index is %%LANE + ;; --------------------------------------------------------------------- + test dword [state + KEYGEN_STAGE + %%LANE*4], 0xffffffff + jne %%process_completed_job_submit_uea2 + + ;; check if INIT1 stage is done + test dword [state + INIT1_DONE + %%LANE*4], 0xffffffff + jne %%_init_done + + ;; mark INIT1 completed and set length to 1DW for INIT2 stage + mov dword [state + INIT1_DONE + %%LANE*4], 0xffffffff + mov dword [state + ROUNDED_DW_LENS + %%LANE*4], 1 + + jmp %%_find_min + +%%_init_done: + mov dword [state + KEYGEN_STAGE + %%LANE*4], 0xffffffff + + ;; length in double words = original length in bytes / 4 + ;; odd bytes are rounded up + mov DWORD(%%TGP0), [state + _snow3g_args_byte_length + %%LANE*4] + mov DWORD(%%TGP1), DWORD(%%TGP0) + shr %%TGP0, 2 + and %%TGP1, 3 + je %%_no_rounding_up + inc %%TGP0 +%%_no_rounding_up: + mov dword [state + ROUNDED_DW_LENS + %%LANE*4], DWORD (%%TGP0) + jmp %%_find_min + +%%process_completed_job_submit_uea2: + ;; COMPLETE: return job, change job dw length to UINT32_MAX, set masks + ;; to not initialized + mov dword [state + ROUNDED_DW_LENS + %%LANE*4], 0xffffffff + mov dword [state + KEYGEN_STAGE + %%LANE*4], 0 + mov dword [state + INIT1_DONE + %%LANE*4], 0 + + ;; decrement number of jobs in use + dec word [state + _snow3g_lanes_in_use] + + mov job_rax, [state + _snow3g_job_in_lane + %%LANE*8] + or qword [job_rax + _status], IMB_STATUS_COMPLETED_CIPHER + + mov %%UNUSED_LANES, [state + _snow3g_unused_lanes] + mov qword [state + _snow3g_job_in_lane + %%LANE*8], 0 + shl %%UNUSED_LANES, 4 + or %%UNUSED_LANES, %%LANE + mov [state + _snow3g_unused_lanes], %%UNUSED_LANES + +%ifdef SAFE_DATA + ;; clear finished job lane, %%LANE is an index of finished job + mov dword [state + _snow3g_args_LFSR_0 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_1 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_2 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_3 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_4 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_5 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_6 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_7 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_8 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_9 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_10 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_11 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_12 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_13 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_14 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_LFSR_15 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_FSM_1 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_FSM_2 + 4*%%LANE], 0 + mov dword [state + _snow3g_args_FSM_3 + 4*%%LANE], 0 + + ;; clear key stream stack frame + pxor %%TMP_XMM_0, %%TMP_XMM_0 + ;; _keystream clean is part of submit as well under return_uea2 label + movdqa [rsp + _keystream + 1 * 16], %%TMP_XMM_0 + movdqa [rsp + _keystream + 2 * 16], %%TMP_XMM_0 + movdqa [rsp + _keystream + 3 * 16], %%TMP_XMM_0 +%endif + +%%return_uea2: + +%ifdef SAFE_DATA + ;; clear temporarily stored swapped IV (done inside of submit) + pxor %%TMP_XMM_0, %%TMP_XMM_0 + movdqa [rsp + _keystream], %%TMP_XMM_0 +%endif + + SNOW3G_FUNC_END + +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; JOB* SUBMIT_JOB_SNOW3G_UEA2(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job) +;; arg 1 : state +;; arg 2 : job +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +MKGLOBAL(SUBMIT_JOB_SNOW3G_UEA2,function,internal) +SUBMIT_JOB_SNOW3G_UEA2: + endbranch64 + SUBMIT_FLUSH_JOB_SNOW3G_UEA2_SSE submit, tmp_gp1, tmp_gp2, tmp_gp3, \ + tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, \ + tmp_gp8, xmm0, xmm1, xmm2, xmm3, xmm4, \ + xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, \ + xmm11, xmm12, xmm13, xmm14, xmm15 + + ret + +MKGLOBAL(FLUSH_JOB_SNOW3G_UEA2,function,internal) +FLUSH_JOB_SNOW3G_UEA2: + endbranch64 + SUBMIT_FLUSH_JOB_SNOW3G_UEA2_SSE flush, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4,\ + tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, xmm0, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, \ + xmm14, xmm15 + ret +mksection stack-noexec \ No newline at end of file diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index 34c81e5c..fe5dd951 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -39,6 +39,7 @@ #include "include/snow3g.h" #include "include/gcm.h" #include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" #include "include/save_xmms.h" #include "include/asm.h" @@ -212,6 +213,9 @@ IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); IMB_JOB *snow_v_sse(IMB_JOB *job); IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); +IMB_JOB *submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state); + void *poly1305_mac_scalar(IMB_JOB *job); #define SAVE_XMMS save_xmms @@ -366,6 +370,9 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define SUBMIT_JOB_SNOW_V snow_v_sse #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse + /* ====================================================================== */ /* @@ -595,6 +602,26 @@ submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) return job; } +static IMB_JOB * +submit_snow3g_uea2_job_sse(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_sse(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + IMB_DLL_LOCAL IMB_JOB * submit_job_aes_cntr_sse(IMB_JOB *job) { @@ -727,6 +754,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, sizeof(aes128_ooo->lens)); @@ -1191,6 +1219,32 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < SSE_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; + + /* Init SNOW3G-UEA out-of-order fields */ + memset(&snow3g_uea2_ooo->args, 0, sizeof(snow3g_uea2_ooo->args)); + memset(snow3g_uea2_ooo->job_in_lane, 0, + sizeof(snow3g_uea2_ooo->job_in_lane)); + snow3g_uea2_ooo->unused_lanes = 0x3210; + snow3g_uea2_ooo->num_lanes_inuse = 0; + /* + * lens[0:3] are used to indicate outstanding bytes after rounding + * up length to dwords + * lens[4] is used for common min length for all lanes in dwords + * lens[8:11] are used for keeping lengths rounded up to dwords + * lens [5:7], [12:15] are unused + */ + snow3g_uea2_ooo->lens[0]=0; + snow3g_uea2_ooo->lens[1]=0; + snow3g_uea2_ooo->lens[2]=0; + snow3g_uea2_ooo->lens[3]=0; + /* + * Dwords rounded lengths need to be initialized to max length not to + * interfere when searching for minimum length. + */ + snow3g_uea2_ooo->lens[8]=0xffffffff; + snow3g_uea2_ooo->lens[9]=0xffffffff; + snow3g_uea2_ooo->lens[10]=0xffffffff; + snow3g_uea2_ooo->lens[11]=0xffffffff; } IMB_DLL_LOCAL void diff --git a/lib/sse/snow3g_sse.c b/lib/sse/snow3g_sse.c index 4a32b3b1..664007b0 100644 --- a/lib/sse/snow3g_sse.c +++ b/lib/sse/snow3g_sse.c @@ -40,3 +40,7 @@ #define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse #include "include/snow3g_common.h" +#include "include/ipsec_ooo_mgr.h" + +IMB_DLL_LOCAL void submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *, IMB_JOB *); +IMB_DLL_LOCAL void flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO); diff --git a/lib/win_x64.mak b/lib/win_x64.mak index d76a3849..c780763c 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -264,6 +264,7 @@ lib_objs1 = \ $(OBJ_DIR)\zuc_x16_vaes_avx512.obj \ $(OBJ_DIR)\zuc_iv.obj \ $(OBJ_DIR)\snow3g_sse.obj \ + $(OBJ_DIR)\snow3g_uea2_by4_sse.obj \ $(OBJ_DIR)\snow3g_uia2_by4_sse.obj \ $(OBJ_DIR)\snow3g_avx.obj \ $(OBJ_DIR)\snow3g_avx2.obj \ @@ -274,6 +275,7 @@ lib_objs1 = \ $(OBJ_DIR)\snow3g_uia2_by32_vaes_avx512.obj \ $(OBJ_DIR)\mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.obj \ $(OBJ_DIR)\mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.obj \ + $(OBJ_DIR)\mb_mgr_snow3g_uea2_submit_flush_x4_sse.obj \ $(OBJ_DIR)\aes_xcbc_expand_key.obj \ $(OBJ_DIR)\md5_one_block.obj \ $(OBJ_DIR)\sha_sse.obj \ -- GitLab From 00cd9db7b3d4edc07a85041969431638d33bfa20 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Wed, 1 Jun 2022 12:16:31 +0000 Subject: [PATCH 145/369] [lib]: fix cet issues --- lib/avx/aes128_cbc_dec_by8_avx.asm | 3 ++- lib/avx/aes128_cntr_by8_avx.asm | 2 ++ lib/avx/aes192_cbc_dec_by8_avx.asm | 2 ++ lib/avx/aes192_cntr_by8_avx.asm | 2 ++ lib/avx/aes256_cbc_dec_by8_avx.asm | 3 ++- lib/avx/aes256_cntr_by8_avx.asm | 2 ++ lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm | 1 + lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm | 1 + lib/sse/aes128_cntr_by8_sse.asm | 2 ++ lib/sse/aes192_cntr_by8_sse.asm | 2 ++ lib/sse/aes256_cntr_by8_sse.asm | 2 ++ 11 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/avx/aes128_cbc_dec_by8_avx.asm b/lib/avx/aes128_cbc_dec_by8_avx.asm index 978e9326..3cfaa81c 100644 --- a/lib/avx/aes128_cbc_dec_by8_avx.asm +++ b/lib/avx/aes128_cbc_dec_by8_avx.asm @@ -31,6 +31,7 @@ %include "include/os.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -215,7 +216,7 @@ mksection .text MKGLOBAL(AES_CBC_DEC_128_X8,function,internal) AES_CBC_DEC_128_X8: - + endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %else diff --git a/lib/avx/aes128_cntr_by8_avx.asm b/lib/avx/aes128_cntr_by8_avx.asm index a74b8599..a5459096 100644 --- a/lib/avx/aes128_cntr_by8_avx.asm +++ b/lib/avx/aes128_cntr_by8_avx.asm @@ -31,6 +31,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -599,6 +600,7 @@ aes_cntr_ccm_128_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_128_avx,function,internal) aes_cntr_128_avx: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_128_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/aes192_cbc_dec_by8_avx.asm b/lib/avx/aes192_cbc_dec_by8_avx.asm index 13f85efb..dabecd1d 100644 --- a/lib/avx/aes192_cbc_dec_by8_avx.asm +++ b/lib/avx/aes192_cbc_dec_by8_avx.asm @@ -30,6 +30,7 @@ ; XMM registers are clobbered. Saving/restoring must be done at a higher level %include "include/os.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -230,6 +231,7 @@ mksection .text ;; aes_cbc_dec_192_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(aes_cbc_dec_192_avx,function,internal) aes_cbc_dec_192_avx: + endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/avx/aes192_cntr_by8_avx.asm b/lib/avx/aes192_cntr_by8_avx.asm index c6e928c0..017b837b 100644 --- a/lib/avx/aes192_cntr_by8_avx.asm +++ b/lib/avx/aes192_cntr_by8_avx.asm @@ -30,6 +30,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -495,6 +496,7 @@ aes_cntr_ccm_192_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_192_avx,function,internal) aes_cntr_192_avx: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_192_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/aes256_cbc_dec_by8_avx.asm b/lib/avx/aes256_cbc_dec_by8_avx.asm index 44ac3c6c..44f76834 100644 --- a/lib/avx/aes256_cbc_dec_by8_avx.asm +++ b/lib/avx/aes256_cbc_dec_by8_avx.asm @@ -30,6 +30,7 @@ ; XMM registers are clobbered. Saving/restoring must be done at a higher level %include "include/os.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -246,7 +247,7 @@ mksection .text ;; aes_cbc_dec_256_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(aes_cbc_dec_256_avx,function,internal) aes_cbc_dec_256_avx: - + endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/avx/aes256_cntr_by8_avx.asm b/lib/avx/aes256_cntr_by8_avx.asm index 3d5d3b6a..bde04636 100644 --- a/lib/avx/aes256_cntr_by8_avx.asm +++ b/lib/avx/aes256_cntr_by8_avx.asm @@ -31,6 +31,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -593,6 +594,7 @@ aes_cntr_ccm_256_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_256_avx,function,internal) aes_cntr_256_avx: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_256_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm b/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm index 994ea9df..5f6c6167 100644 --- a/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm @@ -116,6 +116,7 @@ endstruc ; arg 2 : job MKGLOBAL(FLUSH_JOB_AES_ENC,function,internal) FLUSH_JOB_AES_ENC: + endbranch64 mov rax, rsp sub rsp, STACK_size diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm b/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm index db9cb4e2..c563589e 100644 --- a/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm @@ -89,6 +89,7 @@ endstruc ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_ENC,function,internal) SUBMIT_JOB_AES_ENC: + endbranch64 mov rax, rsp sub rsp, STACK_size diff --git a/lib/sse/aes128_cntr_by8_sse.asm b/lib/sse/aes128_cntr_by8_sse.asm index d0ea42ba..d0b0ebc7 100644 --- a/lib/sse/aes128_cntr_by8_sse.asm +++ b/lib/sse/aes128_cntr_by8_sse.asm @@ -31,6 +31,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -571,6 +572,7 @@ AES_CNTR_CCM_128: ;; aes_cntr_128_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) MKGLOBAL(AES_CNTR_128,function,internal) AES_CNTR_128: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_128_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) diff --git a/lib/sse/aes192_cntr_by8_sse.asm b/lib/sse/aes192_cntr_by8_sse.asm index 7939aa1c..ec894769 100644 --- a/lib/sse/aes192_cntr_by8_sse.asm +++ b/lib/sse/aes192_cntr_by8_sse.asm @@ -30,6 +30,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -496,6 +497,7 @@ align 32 ;; aes_cntr_192_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) MKGLOBAL(AES_CNTR_192,function,internal) AES_CNTR_192: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_192_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) diff --git a/lib/sse/aes256_cntr_by8_sse.asm b/lib/sse/aes256_cntr_by8_sse.asm index ee9e7fe3..3964bc10 100644 --- a/lib/sse/aes256_cntr_by8_sse.asm +++ b/lib/sse/aes256_cntr_by8_sse.asm @@ -31,6 +31,7 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" +%include "include/cet.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -599,6 +600,7 @@ AES_CNTR_CCM_256: ;; aes_cntr_256_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) MKGLOBAL(AES_CNTR_256,function,internal) AES_CNTR_256: + endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_256_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) -- GitLab From 6c0c3876cbe5bb14b2cb5caf75fae0f20953b54f Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 30 May 2022 08:27:36 +0000 Subject: [PATCH 146/369] test: [ACVP] support AES-CBC --- test/acvp_app_main.c | 116 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 104 insertions(+), 12 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 59c636a8..0ee30cba 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -46,6 +46,100 @@ IMB_MGR *mb_mgr = NULL; int verbose = 0; int direct_api = 0; /* job API by default */ +static int aes_cbc_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); + static uint8_t next_iv[16]; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + fprintf(stderr, "Unsupported direction\n"); + return EXIT_FAILURE; + } + + switch (tc->key_len) { + case 128: + IMB_AES_KEYEXP_128(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 192: + IMB_AES_KEYEXP_192(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 256: + IMB_AES_KEYEXP_256(mb_mgr, tc->key, enc_keys, dec_keys); + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len >> 3; + job->cipher_mode = IMB_CIPHER_CBC; + job->hash_alg = IMB_AUTH_NULL; + /* + * If Monte-carlo test, use the IV from the ciphertext of + * the previous iteration + */ + if (tc->test_type == ACVP_SYM_TEST_TYPE_MCT && + tc->mct_index != 0) + job->iv = next_iv; + else + job->iv = tc->iv; + + job->iv_len_in_bytes = tc->iv_len; + job->cipher_start_src_offset_in_bytes = 0; + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + tc->ct_len = tc->pt_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } else /* DECRYPT */ { + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + tc->pt_len = tc->ct_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + /* + * If Monte-carlo test, copy the ciphertext for + * the IV of the next iteration + */ + if (tc->test_type == ACVP_SYM_TEST_TYPE_MCT) + memcpy(next_iv, tc->ct, 16); + + return EXIT_SUCCESS; +} + static int aes_gcm_handler(ACVP_TEST_CASE *test_case) { ACVP_SYM_CIPHER_TC *tc; @@ -55,7 +149,6 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) aes_gcm_enc_dec_finalize_t finalize_enc = mb_mgr->gcm128_enc_finalize; aes_gcm_enc_dec_update_t update_dec = mb_mgr->gcm128_dec_update; aes_gcm_enc_dec_finalize_t finalize_dec = mb_mgr->gcm128_dec_finalize; - int ret = 0; struct gcm_key_data key; struct gcm_context_data ctx; @@ -67,7 +160,6 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { - fprintf(stderr, "Unsupported direction\n"); return EXIT_FAILURE; } @@ -83,8 +175,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - ret = 1; - goto end; + return EXIT_FAILURE; } if (direct_api == 1) { @@ -108,8 +199,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) break; default: fprintf(stderr, "Unsupported AES key length\n"); - ret = 1; - goto end; + return EXIT_FAILURE; } } else { job = IMB_GET_NEXT_JOB(mb_mgr); @@ -146,8 +236,8 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) if (job == NULL) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { - ret = 1; fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; } } } else /* DECRYPT */ { @@ -171,13 +261,11 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) if (job == NULL) job = IMB_FLUSH_JOB(mb_mgr); if (job->status != IMB_STATUS_COMPLETED) { - ret = 1; fprintf(stderr, "Invalid job\n"); - goto end; + return EXIT_FAILURE; } } if (memcmp(res_tag, tc->tag, tc->tag_len) != 0) { - ret = 1; if (verbose) { hexdump(stdout, "result tag: ", res_tag, tc->tag_len); @@ -185,10 +273,10 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) tc->tag, tc->tag_len); fprintf(stderr, "Invalid tag\n"); } + return EXIT_FAILURE; } } -end: - return ret; + return EXIT_SUCCESS; } static void usage(const char *app_name) @@ -296,6 +384,10 @@ main(int argc, char **argv) &aes_gcm_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CBC, + &aes_cbc_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From c4606fc74593c464c4ca445cbd6f11666cb5fb5b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 23 May 2022 19:44:15 +0000 Subject: [PATCH 147/369] lib: [sse] function prototypes moved from mb_mgr_sse.c into separate header files --- lib/include/arch_sse_type1.h | 225 +++++++++++++++++++++++++++++++ lib/include/arch_sse_type2.h | 50 +++++++ lib/include/arch_sse_type3.h | 115 ++++++++++++++++ lib/include/asm.h | 71 ---------- lib/sse/mb_mgr_sse.c | 187 +------------------------ lib/x86_64/aes_xcbc_expand_key.c | 1 + 6 files changed, 394 insertions(+), 255 deletions(-) create mode 100644 lib/include/arch_sse_type1.h create mode 100644 lib/include/arch_sse_type2.h create mode 100644 lib/include/arch_sse_type3.h diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h new file mode 100644 index 00000000..f2de7321 --- /dev/null +++ b/lib/include/arch_sse_type1.h @@ -0,0 +1,225 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* ARCH SSE TYPE 1: SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP */ + +#ifndef IMB_ARCH_SSE_TYPE1_H +#define IMB_ARCH_SSE_TYPE1_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* AES-CBC */ +void aes_cbc_enc_128_x8(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_192_x8(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_256_x8(AES_ARGS *args, uint64_t len_in_bytes); + + +void aes_cbc_dec_128_sse(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); +void aes_cbc_dec_192_sse(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); +void aes_cbc_dec_256_sse(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); + +/* AES-CTR */ +void aes_cntr_256_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_192_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_128_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); + +/* AES-CTR-BITLEN */ +void aes_cntr_bit_256_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_192_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_128_sse(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); + +/* AES-CCM */ +IMB_JOB *aes_cntr_ccm_128_sse(IMB_JOB *job); + +IMB_JOB *aes_cntr_ccm_256_sse(IMB_JOB *job); + +/* AES-ECB */ +void aes_ecb_enc_256_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_by4_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +/* AES128-ECBENC */ +void aes128_ecbenc_x3_sse(const void *in, void *keys, + void *out1, void *out2, void *out3); + +/* AES-CBCS */ +void aes_cbcs_1_9_dec_128_sse(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes, void *next_iv); + +/* stitched AES128-CNTR, CRC32 and BIP */ +IMB_JOB *submit_job_pon_enc_sse(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_sse(IMB_JOB *job); + +IMB_JOB *submit_job_pon_enc_no_ctr_sse(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_no_ctr_sse(IMB_JOB *job); + +/* CRC */ +uint32_t ethernet_fcs_sse(const void *msg, const uint64_t len); +uint32_t crc16_x25_sse(const void *msg, const uint64_t len); +uint32_t crc32_sctp_sse(const void *msg, const uint64_t len); +uint32_t crc24_lte_a_sse(const void *msg, const uint64_t len); +uint32_t crc24_lte_b_sse(const void *msg, const uint64_t len); +uint32_t crc16_fp_data_sse(const void *msg, const uint64_t len); +uint32_t crc11_fp_header_sse(const void *msg, const uint64_t len); +uint32_t crc7_fp_header_sse(const void *msg, const uint64_t len); +uint32_t crc10_iuup_data_sse(const void *msg, const uint64_t len); +uint32_t crc6_iuup_header_sse(const void *msg, const uint64_t len); +uint32_t crc32_wimax_ofdma_data_sse(const void *msg, const uint64_t len); +uint32_t crc8_wimax_ofdma_hcs_sse(const void *msg, const uint64_t len); + + +/* moved from MB MGR */ +IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_enc_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes192_enc_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes192_enc_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes256_enc_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes256_enc_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state); + +IMB_JOB *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state); + + +IMB_JOB *submit_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state); + +IMB_JOB *submit_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *submit_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state); + +IMB_JOB *flush_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes_cntr_sse(IMB_JOB *job); + +IMB_JOB *submit_job_aes_cntr_bit_sse(IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); + +void aes_cmac_256_subkey_gen_sse(const void *key_exp, + void *key1, void *key2); +uint32_t hec_32_sse(const uint8_t *in); +uint64_t hec_64_sse(const uint8_t *in); + +IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); + +IMB_JOB *snow_v_sse(IMB_JOB *job); +IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); + +void *poly1305_mac_scalar(IMB_JOB *job); + +IMB_JOB *submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state); + +#endif /* IMB_ARCH_SSE_TYPE1_H */ diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h new file mode 100644 index 00000000..32a7e377 --- /dev/null +++ b/lib/include/arch_sse_type2.h @@ -0,0 +1,50 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* ARCH SSE TYPE 2: SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP, SHANI */ + +#ifndef IMB_ARCH_SSE_TYPE2_H +#define IMB_ARCH_SSE_TYPE2_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* Moved from MB MGR */ + +IMB_JOB *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state); + +IMB_JOB *submit_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); + +#endif /* IMB_ARCH_SSE_TYPE2_H */ diff --git a/lib/include/arch_sse_type3.h b/lib/include/arch_sse_type3.h new file mode 100644 index 00000000..41e1ef5e --- /dev/null +++ b/lib/include/arch_sse_type3.h @@ -0,0 +1,115 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* ARCH SSE TYPE 3: SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP, SHANI, GFNI */ + +#ifndef IMB_ASM_SSE_T3_H +#define IMB_ASM_SSE_T3_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* AES-CBC */ +void aes_cbc_enc_128_x8_sse(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_192_x8_sse(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_256_x8_sse(AES_ARGS *args, uint64_t len_in_bytes); + +void aes_cbc_dec_128_by8_sse(const void *in, const uint8_t *IV, + const void *keys, void *out, uint64_t len_bytes); +void aes_cbc_dec_192_by8_sse(const void *in, const uint8_t *IV, + const void *keys, void *out, uint64_t len_bytes); +void aes_cbc_dec_256_by8_sse(const void *in, const uint8_t *IV, + const void *keys, void *out, uint64_t len_bytes); + +/* AES-ECB */ +void aes_ecb_enc_256_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_by8_sse(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +/* moved from MB MGR */ + +IMB_JOB *submit_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +#endif /* IMB_ASM_SSE_T3_H */ diff --git a/lib/include/asm.h b/lib/include/asm.h index f46bf955..24a75fc4 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -57,20 +57,6 @@ void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_128_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_192_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_256_sse(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); - -void aes_cbc_dec_128_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_192_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_256_by8_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, uint64_t len_bytes); - void aes_cbc_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes); @@ -82,13 +68,6 @@ void aes_cbc_dec_256_sse_no_aesni(const void *in, const uint8_t *IV, uint64_t len_bytes); /* AES-CTR */ -void aes_cntr_256_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - void aes_cntr_256_sse_no_aesni(const void *in, const void *IV, const void *keys, void *out, uint64_t len_bytes, uint64_t IV_len); void aes_cntr_192_sse_no_aesni(const void *in, const void *IV, const void *keys, @@ -109,13 +88,6 @@ void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); /* AES-CTR-BITLEN */ -void aes_cntr_bit_256_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_sse(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); - void aes_cntr_bit_256_sse_no_aesni(const void *in, const void *IV, const void *keys, void *out, uint64_t len_bits, uint64_t IV_len); @@ -138,10 +110,6 @@ void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); /* AES-CCM */ -IMB_JOB *aes_cntr_ccm_128_sse(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_sse(IMB_JOB *job); - IMB_JOB *aes_cntr_ccm_128_sse_no_aesni(IMB_JOB *job); IMB_JOB *aes_cntr_ccm_256_sse_no_aesni(IMB_JOB *job); @@ -151,20 +119,6 @@ IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); /* AES-ECB */ -void aes_ecb_enc_256_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_enc_256_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_enc_256_sse_no_aesni(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_192_sse_no_aesni(const void *in, const void *keys, @@ -194,20 +148,6 @@ void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_by4_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_dec_256_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_by8_sse(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_dec_256_sse_no_aesni(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_sse_no_aesni(const void *in, const void *keys, @@ -238,17 +178,12 @@ void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); /* AES128-ECBENC */ -void aes128_ecbenc_x3_sse(const void *in, void *keys, - void *out1, void *out2, void *out3); void aes128_ecbenc_x3_sse_no_aesni(const void *in, void *keys, void *out1, void *out2, void *out3); void aes128_ecbenc_x3_avx(const void *in, void *keys, void *out1, void *out2, void *out3); /* AES-CBCS */ -void aes_cbcs_1_9_dec_128_sse(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); void aes_cbcs_1_9_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); @@ -264,9 +199,6 @@ void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_enc_sse(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_sse(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_sse_no_aesni(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_sse_no_aesni(IMB_JOB *job); @@ -276,9 +208,6 @@ IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_enc_no_ctr_avx(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_no_ctr_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_enc_no_ctr_sse(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_sse(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_no_ctr_sse_no_aesni(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_no_ctr_sse_no_aesni(IMB_JOB *job); diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse/mb_mgr_sse.c index fe5dd951..56e5d836 100644 --- a/lib/sse/mb_mgr_sse.c +++ b/lib/sse/mb_mgr_sse.c @@ -42,181 +42,15 @@ #include "include/snow3g_submit.h" #include "include/save_xmms.h" -#include "include/asm.h" #include "include/des.h" #include "include/cpu_feature.h" #include "include/noaesni.h" #include "include/aesni_emu.h" #include "include/error.h" -IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_x8_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state); - - -IMB_JOB *submit_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_sse(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_x8_sse(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *flush_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_x8_sse(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes_cntr_sse(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_sse(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_gfni_sse(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_gfni_sse(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); - -void aes_cmac_256_subkey_gen_sse(const void *key_exp, - void *key1, void *key2); -uint32_t hec_32_sse(const uint8_t *in); -uint64_t hec_64_sse(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); - -IMB_JOB *snow_v_sse(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); - -IMB_JOB *submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state); - -void *poly1305_mac_scalar(IMB_JOB *job); +#include "include/arch_sse_type1.h" +#include "include/arch_sse_type2.h" +#include "include/arch_sse_type3.h" #define SAVE_XMMS save_xmms #define RESTORE_XMMS restore_xmms @@ -325,9 +159,6 @@ void *poly1305_mac_scalar(IMB_JOB *job); #define SUBMIT_HASH_BURST submit_hash_burst_sse #define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse #define QUEUE_SIZE queue_size_sse /* ====================================================================== */ @@ -390,18 +221,6 @@ ethernet_fcs_sse_local(const void *msg, const uint64_t len, #define ETHERNET_FCS ethernet_fcs_sse_local -uint32_t ethernet_fcs_sse(const void *msg, const uint64_t len); -uint32_t crc16_x25_sse(const void *msg, const uint64_t len); -uint32_t crc32_sctp_sse(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_sse(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_sse(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_sse(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_sse(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_sse(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_sse(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_sse(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_sse(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_sse(const void *msg, const uint64_t len); /* ====================================================================== */ diff --git a/lib/x86_64/aes_xcbc_expand_key.c b/lib/x86_64/aes_xcbc_expand_key.c index 0f3d73b5..42689c4b 100644 --- a/lib/x86_64/aes_xcbc_expand_key.c +++ b/lib/x86_64/aes_xcbc_expand_key.c @@ -34,6 +34,7 @@ #include "asm.h" #include "include/clear_regs_mem.h" #include "include/error.h" +#include "include/arch_sse_type1.h" static uint32_t in[4*3] = { 0x01010101, 0x01010101, 0x01010101, 0x01010101, -- GitLab From 7c2c06d9bc8859af2dbfee00a4af6ef809df6327 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 24 May 2022 22:39:22 +0000 Subject: [PATCH 148/369] lib: [no-aesni] function prototypes moved from mb_mgr_sse_no_aesni.c into separate header file --- lib/include/arch_noaesni.h | 193 +++++++++++++++++++++++++++++ lib/include/asm.h | 61 --------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 120 +----------------- lib/x86_64/aes_xcbc_expand_key.c | 1 + 4 files changed, 196 insertions(+), 179 deletions(-) create mode 100644 lib/include/arch_noaesni.h diff --git a/lib/include/arch_noaesni.h b/lib/include/arch_noaesni.h new file mode 100644 index 00000000..6ab55351 --- /dev/null +++ b/lib/include/arch_noaesni.h @@ -0,0 +1,193 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* interface to asm routines */ + +#ifndef IMB_ARCH_NOAESNI_H +#define IMB_ARCH_NOAESNI_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* AES-CBC */ + +void aes_cbc_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); +void aes_cbc_dec_192_sse_no_aesni(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); +void aes_cbc_dec_256_sse_no_aesni(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); + +/* AES-CTR */ +void aes_cntr_256_sse_no_aesni(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_192_sse_no_aesni(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_128_sse_no_aesni(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); + +/* AES-CTR-BITLEN */ +void aes_cntr_bit_256_sse_no_aesni(const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_192_sse_no_aesni(const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_128_sse_no_aesni(const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bits, uint64_t IV_len); + +/* AES-CCM */ +IMB_JOB *aes_cntr_ccm_128_sse_no_aesni(IMB_JOB *job); +IMB_JOB *aes_cntr_ccm_256_sse_no_aesni(IMB_JOB *job); + +/* AES-ECB */ +void aes_ecb_enc_256_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_sse_no_aesni(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +/* AES128-ECBENC */ +void aes128_ecbenc_x3_sse_no_aesni(const void *in, void *keys, + void *out1, void *out2, void *out3); + +/* AES-CBCS */ +void aes_cbcs_1_9_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes, void *next_iv); + +/* stitched AES128-CNTR, CRC32 and BIP */ +IMB_JOB *submit_job_pon_enc_sse_no_aesni(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_sse_no_aesni(IMB_JOB *job); + +IMB_JOB *submit_job_pon_enc_no_ctr_sse_no_aesni(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_no_ctr_sse_no_aesni(IMB_JOB *job); + +/* moved from MB MGR */ +IMB_JOB *submit_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state); + +IMB_JOB *submit_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job); + +IMB_JOB *submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +uint32_t hec_32_sse_no_aesni(const uint8_t *in); +uint64_t hec_64_sse_no_aesni(const uint8_t *in); + +IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state); + +void *poly1305_mac_scalar(IMB_JOB *job); + +IMB_JOB *snow_v_sse_no_aesni(IMB_JOB *job); +IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); + +void aes128_cbc_mac_x4_no_aesni(AES_ARGS *args, uint64_t len); + +uint32_t ethernet_fcs_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc16_x25_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc32_sctp_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc24_lte_a_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc24_lte_b_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc16_fp_data_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc11_fp_header_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc7_fp_header_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc10_iuup_data_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc6_iuup_header_sse_no_aesni(const void *msg, const uint64_t len); +uint32_t crc32_wimax_ofdma_data_sse_no_aesni(const void *msg, + const uint64_t len); +uint32_t crc8_wimax_ofdma_hcs_sse_no_aesni(const void *msg, const uint64_t len); + +uint32_t +ethernet_fcs_sse_no_aesni_local(const void *msg, const uint64_t len, + const void *tag_ouput); + +#endif /* IMB_ARCH_NOAESNI_H */ + + diff --git a/lib/include/asm.h b/lib/include/asm.h index 24a75fc4..fa61031c 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -57,25 +57,7 @@ void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes); -void aes_cbc_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_192_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_256_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); - /* AES-CTR */ -void aes_cntr_256_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_sse_no_aesni(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - - void aes_cntr_256_avx(const void *in, const void *IV, const void *keys, void *out, uint64_t len_bytes, uint64_t IV_len); void aes_cntr_192_avx(const void *in, const void *IV, const void *keys, @@ -88,16 +70,6 @@ void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); /* AES-CTR-BITLEN */ -void aes_cntr_bit_256_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_sse_no_aesni(const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bits, uint64_t IV_len); - void aes_cntr_bit_256_avx(const void *in, const void *IV, const void *keys, void *out, uint64_t len_bits, uint64_t IV_len); void aes_cntr_bit_192_avx(const void *in, const void *IV, const void *keys, @@ -110,23 +82,10 @@ void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); /* AES-CCM */ -IMB_JOB *aes_cntr_ccm_128_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_sse_no_aesni(IMB_JOB *job); - IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); - IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); /* AES-ECB */ -void aes_ecb_enc_256_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); - - void aes_ecb_enc_256_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_192_avx(const void *in, const void *keys, @@ -148,14 +107,6 @@ void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_sse_no_aesni(const void *in, const void *keys, - void *out, uint64_t len_bytes); - - void aes_ecb_dec_256_avx(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_avx(const void *in, const void *keys, @@ -178,15 +129,10 @@ void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); /* AES128-ECBENC */ -void aes128_ecbenc_x3_sse_no_aesni(const void *in, void *keys, - void *out1, void *out2, void *out3); void aes128_ecbenc_x3_avx(const void *in, void *keys, void *out1, void *out2, void *out3); /* AES-CBCS */ -void aes_cbcs_1_9_dec_128_sse_no_aesni(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); void aes_cbcs_1_9_dec_128_avx(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); @@ -194,23 +140,16 @@ void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); - /* stitched AES128-CNTR, CRC32 and BIP */ IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_enc_sse_no_aesni(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_sse_no_aesni(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_enc_no_ctr_avx(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_no_ctr_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_enc_no_ctr_sse_no_aesni(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_sse_no_aesni(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_no_ctr_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_no_ctr_vaes_avx512(IMB_JOB *job); diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index df558dc3..ea9ae3a3 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -39,113 +39,15 @@ #include "include/chacha20_poly1305.h" #include "include/save_xmms.h" -#include "include/asm.h" #include "include/des.h" #include "include/gcm.h" #include "include/noaesni.h" #include "include/error.h" +#include "include/arch_noaesni.h" +#include "include/arch_sse_type1.h" /* ====================================================================== */ -IMB_JOB *submit_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_sse_no_aesni(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_sse_no_aesni(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_sse_no_aesni(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sse(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_sse(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_sse(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_sse(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_sse_no_aesni(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_cmac_auth_sse_no_aesni(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_sse_no_aesni(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_sse_no_aesni(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_sse_no_aesni(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); - -uint32_t hec_32_sse_no_aesni(const uint8_t *in); -uint64_t hec_64_sse_no_aesni(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); - -void *poly1305_mac_scalar(IMB_JOB *job); - -IMB_JOB *snow_v_sse_no_aesni(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); - #define SAVE_XMMS save_xmms #define RESTORE_XMMS restore_xmms @@ -271,7 +173,6 @@ IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); #define AES_CFB_128_ONE aes_cfb_128_one_sse_no_aesni #define AES_CFB_256_ONE aes_cfb_256_one_sse_no_aesni -void aes128_cbc_mac_x4_no_aesni(AES_ARGS *args, uint64_t len); #define AES128_CBC_MAC aes128_cbc_mac_x4_no_aesni @@ -307,25 +208,8 @@ void aes128_cbc_mac_x4_no_aesni(AES_ARGS *args, uint64_t len); /* ====================================================================== */ -uint32_t -ethernet_fcs_sse_no_aesni_local(const void *msg, const uint64_t len, - const void *tag_ouput); - #define ETHERNET_FCS ethernet_fcs_sse_no_aesni_local -uint32_t ethernet_fcs_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc16_x25_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc32_sctp_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t -crc32_wimax_ofdma_data_sse_no_aesni(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_sse_no_aesni(const void *msg, const uint64_t len); /* ====================================================================== */ diff --git a/lib/x86_64/aes_xcbc_expand_key.c b/lib/x86_64/aes_xcbc_expand_key.c index 42689c4b..af9d4f1a 100644 --- a/lib/x86_64/aes_xcbc_expand_key.c +++ b/lib/x86_64/aes_xcbc_expand_key.c @@ -34,6 +34,7 @@ #include "asm.h" #include "include/clear_regs_mem.h" #include "include/error.h" +#include "include/arch_noaesni.h" #include "include/arch_sse_type1.h" static uint32_t in[4*3] = { -- GitLab From 4423ab43a555e9a8abb28545f31978544908c729 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 25 May 2022 11:25:07 +0000 Subject: [PATCH 149/369] lib: [avx] function prototypes moved from mb_mgr_avx.c into separate header file --- lib/avx/mb_mgr_avx.c | 122 +--------------- lib/avx2/mb_mgr_avx2.c | 32 +---- lib/avx512/mb_mgr_avx512.c | 60 +------- lib/include/arch_avx_type1.h | 230 +++++++++++++++++++++++++++++++ lib/include/asm.h | 56 -------- lib/x86_64/aes_xcbc_expand_key.c | 1 + 6 files changed, 235 insertions(+), 266 deletions(-) create mode 100644 lib/include/arch_avx_type1.h diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 23373612..2914aeaf 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -40,70 +40,11 @@ #include "include/gcm.h" #include "include/chacha20_poly1305.h" #include "include/save_xmms.h" -#include "include/asm.h" #include "include/des.h" #include "include/cpu_feature.h" -#include "include/noaesni.h" #include "include/aesni_emu.h" #include "include/error.h" - -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *flush_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); - -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_chacha20_enc_dec_avx(IMB_JOB *job); - -void *poly1305_mac_scalar(IMB_JOB *job); - -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); +#include "include/arch_avx_type1.h" #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -201,53 +142,6 @@ IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx -IMB_JOB *submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -void aes_cmac_256_subkey_gen_avx(const void *key_exp, - void *key1, void *key2); - #define SUBMIT_JOB_HMAC submit_job_hmac_avx #define FLUSH_JOB_HMAC flush_job_hmac_avx #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx @@ -286,7 +180,6 @@ void aes_cmac_256_subkey_gen_avx(const void *key_exp, #define AES_CFB_128_ONE aes_cfb_128_one_avx #define AES_CFB_256_ONE aes_cfb_256_one_avx -void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); #define AES128_CBC_MAC aes128_cbc_mac_x8 @@ -310,19 +203,6 @@ ethernet_fcs_avx_local(const void *msg, const uint64_t len, #define ETHERNET_FCS ethernet_fcs_avx_local -uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - /* ====================================================================== */ #define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index b62368e3..eda22e6f 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -48,25 +48,8 @@ #include "include/aesni_emu.h" #include "include/error.h" -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); +#include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_avx_type1.h" IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *job); @@ -94,20 +77,9 @@ IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, void aes_cmac_256_subkey_gen_avx2(const void *key_exp, void *key1, void *key2); -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); IMB_JOB *submit_job_chacha20_enc_dec_avx2(IMB_JOB *job); -void *poly1305_mac_scalar(IMB_JOB *job); - -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); - #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 0cd329ff..3c27f9a5 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -50,21 +50,7 @@ #include "include/aesni_emu.h" #include "include/error.h" -IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); +#include "include/arch_avx_type1.h" IMB_JOB *submit_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state, IMB_JOB *job); @@ -109,10 +95,6 @@ IMB_JOB *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); -IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); - IMB_JOB *submit_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); @@ -160,8 +142,6 @@ IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, void aes_cmac_256_subkey_gen_avx512(const void *key_exp, void *key1, void *key2); -uint32_t hec_32_avx(const uint8_t *in); -uint64_t hec_64_avx(const uint8_t *in); IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, IMB_JOB *job); @@ -171,10 +151,6 @@ IMB_JOB *submit_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state); -IMB_JOB *snow_v_avx(IMB_JOB *job); -IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); - - IMB_JOB *submit_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); @@ -312,17 +288,6 @@ IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - - IMB_JOB *submit_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, IMB_JOB *job); @@ -333,21 +298,11 @@ IMB_JOB *submit_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, IMB_JOB *flush_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - IMB_JOB *submit_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - IMB_JOB *submit_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, IMB_JOB *job); @@ -541,19 +496,6 @@ crc32_wimax_ofdma_data_avx512(const void *msg, const uint64_t len); extern uint32_t crc8_wimax_ofdma_hcs_avx512(const void *msg, const uint64_t len); -extern uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -extern uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -extern uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -extern uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -extern uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -extern uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - extern void snow3g_f9_1_buffer_vaes_avx512(const snow3g_key_schedule_t *pHandle, const void *pIV, const void *pBufferIn, diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h new file mode 100644 index 00000000..2ecd15ba --- /dev/null +++ b/lib/include/arch_avx_type1.h @@ -0,0 +1,230 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX + AESNI + PCLMULQDQ */ + +#ifndef IMB_ASM_AVX_T1_H +#define IMB_ASM_AVX_T1_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + + +/* AES-CBC */ +void aes_cbc_enc_128_x8(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_192_x8(AES_ARGS *args, uint64_t len_in_bytes); +void aes_cbc_enc_256_x8(AES_ARGS *args, uint64_t len_in_bytes); + +void aes_cbc_dec_128_avx(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); +void aes_cbc_dec_192_avx(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); +void aes_cbc_dec_256_avx(const void *in, const uint8_t *IV, const void *keys, + void *out, uint64_t len_bytes); + +/* AES-CTR */ +void aes_cntr_256_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_192_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); +void aes_cntr_128_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bytes, uint64_t IV_len); + +/* AES-CTR-BITLEN */ +void aes_cntr_bit_256_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_192_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); +void aes_cntr_bit_128_avx(const void *in, const void *IV, const void *keys, + void *out, uint64_t len_bits, uint64_t IV_len); + +/* AES-CCM */ +IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); +IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); + +/* AES-ECB */ +void aes_ecb_enc_256_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_avx(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +/* AES128-ECBENC */ +void aes128_ecbenc_x3_avx(const void *in, void *keys, + void *out1, void *out2, void *out3); + +/* AES-CBCS */ +void aes_cbcs_1_9_dec_128_avx(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes, void *next_iv); + +/* stitched AES128-CNTR, CRC32 and BIP */ +IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); + +IMB_JOB *submit_job_pon_enc_no_ctr_avx(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_no_ctr_avx(IMB_JOB *job); + +/* moved from MB MGR */ +IMB_JOB *submit_job_aes128_enc_avx(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_enc_avx(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes192_enc_avx(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes192_enc_avx(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes256_enc_avx(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes256_enc_avx(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes_xcbc_avx(MB_MGR_AES_XCBC_OOO *state); + +IMB_JOB *submit_job_aes_cntr_avx(IMB_JOB *job); + +IMB_JOB *submit_job_aes_cntr_bit_avx(IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_avx(MB_MGR_ZUC_OOO *state); + +IMB_JOB *flush_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_avx(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_avx(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_avx(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); + +uint32_t hec_32_avx(const uint8_t *in); +uint64_t hec_64_avx(const uint8_t *in); + +IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_chacha20_enc_dec_avx(IMB_JOB *job); + +void *poly1305_mac_scalar(IMB_JOB *job); + +IMB_JOB *snow_v_avx(IMB_JOB *job); +IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); + +IMB_JOB *submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state); + +IMB_JOB *submit_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state); + +IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); + +void aes_cmac_256_subkey_gen_avx(const void *key_exp, + void *key1, void *key2); + +void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); + +uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); +uint32_t crc16_x25_avx(const void *msg, const uint64_t len); +uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); +uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); +uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); +uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); +uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); +uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); +uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); +uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); +uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); +uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); + +#endif /* IMB_ASM_AVX_T1_H */ + + diff --git a/lib/include/asm.h b/lib/include/asm.h index fa61031c..2bcd20fb 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -36,17 +36,6 @@ /* Define interface to base asm code */ /* AES-CBC */ -void aes_cbc_enc_128_x8(AES_ARGS *args, uint64_t len_in_bytes); -void aes_cbc_enc_192_x8(AES_ARGS *args, uint64_t len_in_bytes); -void aes_cbc_enc_256_x8(AES_ARGS *args, uint64_t len_in_bytes); - -void aes_cbc_dec_128_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_192_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); -void aes_cbc_dec_256_avx(const void *in, const uint8_t *IV, const void *keys, - void *out, uint64_t len_bytes); - void aes_cbc_dec_128_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes); @@ -58,41 +47,16 @@ void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, uint64_t len_bytes); /* AES-CTR */ -void aes_cntr_256_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_192_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); -void aes_cntr_128_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bytes, uint64_t IV_len); - void aes_cntr_128_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); /* AES-CTR-BITLEN */ -void aes_cntr_bit_256_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_192_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); -void aes_cntr_bit_128_avx(const void *in, const void *IV, const void *keys, - void *out, uint64_t len_bits, uint64_t IV_len); - void aes_cntr_bit_128_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); -/* AES-CCM */ -IMB_JOB *aes_cntr_ccm_128_avx(IMB_JOB *job); -IMB_JOB *aes_cntr_ccm_256_avx(IMB_JOB *job); - /* AES-ECB */ -void aes_ecb_enc_256_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_enc_256_vaes_avx2(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_192_vaes_avx2(const void *in, const void *keys, @@ -107,13 +71,6 @@ void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_avx(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_dec_256_vaes_avx2(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_vaes_avx2(const void *in, const void *keys, @@ -128,28 +85,15 @@ void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -/* AES128-ECBENC */ -void aes128_ecbenc_x3_avx(const void *in, void *keys, - void *out1, void *out2, void *out3); - /* AES-CBCS */ -void aes_cbcs_1_9_dec_128_avx(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, const void *keys, void *out, uint64_t len_bytes, void *next_iv); /* stitched AES128-CNTR, CRC32 and BIP */ -IMB_JOB *submit_job_pon_enc_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_avx(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_pon_enc_no_ctr_avx(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_avx(IMB_JOB *job); - IMB_JOB *submit_job_pon_enc_no_ctr_vaes_avx512(IMB_JOB *job); IMB_JOB *submit_job_pon_dec_no_ctr_vaes_avx512(IMB_JOB *job); diff --git a/lib/x86_64/aes_xcbc_expand_key.c b/lib/x86_64/aes_xcbc_expand_key.c index af9d4f1a..ccfdc4d4 100644 --- a/lib/x86_64/aes_xcbc_expand_key.c +++ b/lib/x86_64/aes_xcbc_expand_key.c @@ -36,6 +36,7 @@ #include "include/error.h" #include "include/arch_noaesni.h" #include "include/arch_sse_type1.h" +#include "include/arch_avx_type1.h" static uint32_t in[4*3] = { 0x01010101, 0x01010101, 0x01010101, 0x01010101, -- GitLab From 65df601871f8cd7c6357ee2ea93be6db983dd06b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 25 May 2022 12:58:26 +0000 Subject: [PATCH 150/369] lib: [avx2] function prototypes moved from mb_mgr_avx2.c into separate header files --- lib/avx2/mb_mgr_avx2.c | 91 +---------------------------------- lib/avx512/mb_mgr_avx512.c | 7 +-- lib/include/arch_avx2_type1.h | 90 ++++++++++++++++++++++++++++++++++ lib/include/arch_avx2_type2.h | 51 ++++++++++++++++++++ lib/include/asm.h | 14 ------ 5 files changed, 145 insertions(+), 108 deletions(-) create mode 100644 lib/include/arch_avx2_type1.h create mode 100644 lib/include/arch_avx2_type2.h diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index eda22e6f..023c6621 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -50,35 +50,8 @@ #include "include/arch_sse_type1.h" /* poly1305 */ #include "include/arch_avx_type1.h" - -IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *flush_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); - -IMB_JOB *submit_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); - -void aes_cmac_256_subkey_gen_avx2(const void *key_exp, - void *key1, void *key2); - -IMB_JOB *submit_job_chacha20_enc_dec_avx2(IMB_JOB *job); +#include "include/arch_avx2_type1.h" +#include "include/arch_avx2_type2.h" #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -177,51 +150,6 @@ IMB_JOB *submit_job_chacha20_enc_dec_avx2(IMB_JOB *job); #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx -IMB_JOB *submit_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_avx(MB_MGR_CMAC_OOO *state); - - -IMB_JOB *submit_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_avx(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); - #define SUBMIT_JOB_HMAC submit_job_hmac_avx2 #define FLUSH_JOB_HMAC flush_job_hmac_avx2 #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 @@ -260,8 +188,6 @@ IMB_JOB *flush_job_aes256_ccm_auth_avx(MB_MGR_CCM_OOO *state); #define AES_CFB_128_ONE aes_cfb_128_one_avx2 #define AES_CFB_256_ONE aes_cfb_256_one_avx2 -void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); - #define AES128_CBC_MAC aes128_cbc_mac_x8 #define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx @@ -283,19 +209,6 @@ uint32_t ethernet_fcs_avx_local(const void *msg, const uint64_t len, #define ETHERNET_FCS ethernet_fcs_avx_local -uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); -uint32_t crc16_x25_avx(const void *msg, const uint64_t len); -uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); -uint32_t crc24_lte_b_avx(const void *msg, const uint64_t len); -uint32_t crc16_fp_data_avx(const void *msg, const uint64_t len); -uint32_t crc11_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc7_fp_header_avx(const void *msg, const uint64_t len); -uint32_t crc10_iuup_data_avx(const void *msg, const uint64_t len); -uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); -uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); -uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); - /* ====================================================================== */ #define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index 3c27f9a5..eb6d769b 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -50,7 +50,8 @@ #include "include/aesni_emu.h" #include "include/error.h" -#include "include/arch_avx_type1.h" +#include "include/arch_avx_type1.h" /* AESNI */ +#include "include/arch_avx2_type1.h" /* MD5 */ IMB_JOB *submit_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state, IMB_JOB *job); @@ -284,10 +285,6 @@ IMB_JOB *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); -IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); - IMB_JOB *submit_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, IMB_JOB *job); diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h new file mode 100644 index 00000000..f09fe193 --- /dev/null +++ b/lib/include/arch_avx2_type1.h @@ -0,0 +1,90 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX2 + AESNI + PCLMULQDQ */ + +#ifndef IMB_ASM_AVX2_T1_H +#define IMB_ASM_AVX2_T1_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* moved from MB MGR */ +IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *flush_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); + +IMB_JOB *submit_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_avx2(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_avx2(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); + +void aes_cmac_256_subkey_gen_avx2(const void *key_exp, + void *key1, void *key2); + +IMB_JOB *submit_job_chacha20_enc_dec_avx2(IMB_JOB *job); + +IMB_JOB *submit_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_avx2(MB_MGR_HMAC_SHA_1_OOO *state); + +IMB_JOB *submit_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_224_avx2(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_256_avx2(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_384_avx2(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_512_avx2(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_md5_avx2(MB_MGR_HMAC_MD5_OOO *state); + +#endif /* IMB_ASM_AVX2_T1_H */ diff --git a/lib/include/arch_avx2_type2.h b/lib/include/arch_avx2_type2.h new file mode 100644 index 00000000..7cd8faa1 --- /dev/null +++ b/lib/include/arch_avx2_type2.h @@ -0,0 +1,51 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX2 + VAES + PCLMULQDQ */ + +#ifndef IMB_ASM_AVX2_T2_H +#define IMB_ASM_AVX2_T2_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* AES-ECB */ +void aes_ecb_enc_256_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_vaes_avx2(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +#endif /* IMB_ASM_AVX2_T2_H */ diff --git a/lib/include/asm.h b/lib/include/asm.h index 2bcd20fb..f96c9fad 100644 --- a/lib/include/asm.h +++ b/lib/include/asm.h @@ -57,13 +57,6 @@ void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); /* AES-ECB */ -void aes_ecb_enc_256_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_enc_256_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, @@ -71,13 +64,6 @@ void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); -void aes_ecb_dec_256_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_vaes_avx2(const void *in, const void *keys, - void *out, uint64_t len_bytes); - void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, void *out, uint64_t len_bytes); void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, -- GitLab From b573b218aa0424cbcf9760af35cf972e7f1d1778 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 25 May 2022 13:44:35 +0000 Subject: [PATCH 151/369] lib: [avx512] function prototypes moved from mb_mgr_avx512.c into separate header files --- lib/avx2/mb_mgr_avx2.c | 1 - lib/avx512/mb_mgr_avx512.c | 218 +------------------------------ lib/include/arch_avx512_type1.h | 137 +++++++++++++++++++ lib/include/arch_avx512_type2.h | 202 ++++++++++++++++++++++++++++ lib/include/asm.h | 88 ------------- lib/x86_64/aes_xcbc_expand_key.c | 1 - 6 files changed, 341 insertions(+), 306 deletions(-) create mode 100644 lib/include/arch_avx512_type1.h create mode 100644 lib/include/arch_avx512_type2.h delete mode 100644 lib/include/asm.h diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2/mb_mgr_avx2.c index 023c6621..f7acabdb 100644 --- a/lib/avx2/mb_mgr_avx2.c +++ b/lib/avx2/mb_mgr_avx2.c @@ -41,7 +41,6 @@ #include "include/chacha20_poly1305.h" #include "include/save_xmms.h" -#include "include/asm.h" #include "include/des.h" #include "include/cpu_feature.h" #include "include/noaesni.h" diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512/mb_mgr_avx512.c index eb6d769b..d50c319f 100644 --- a/lib/avx512/mb_mgr_avx512.c +++ b/lib/avx512/mb_mgr_avx512.c @@ -42,7 +42,6 @@ #include "include/snow3g_submit.h" #include "include/save_xmms.h" -#include "include/asm.h" #include "include/des.h" #include "include/gcm.h" #include "include/cpu_feature.h" @@ -52,125 +51,8 @@ #include "include/arch_avx_type1.h" /* AESNI */ #include "include/arch_avx2_type1.h" /* MD5 */ - -IMB_JOB *submit_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state); - -IMB_JOB *submit_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); - -IMB_JOB *submit_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *submit_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); - -IMB_JOB *aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job); - -IMB_JOB *aes_cntr_ccm_256_vaes_avx512(IMB_JOB *job); - -IMB_JOB *submit_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); -IMB_JOB *flush_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); - -IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, - IMB_JOB *job); - -void aes_cmac_256_subkey_gen_avx512(const void *key_exp, - void *key1, void *key2); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state); - -IMB_JOB *submit_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); - -IMB_JOB *submit_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); +#include "include/arch_avx512_type1.h" +#include "include/arch_avx512_type2.h" #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -265,46 +147,6 @@ IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx -IMB_JOB *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); - -IMB_JOB *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); - -IMB_JOB *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); -IMB_JOB *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); - -IMB_JOB *submit_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); - -IMB_JOB *submit_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); - -IMB_JOB *submit_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, - IMB_JOB *job); - -IMB_JOB *flush_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); - static IMB_JOB *submit_snow3g_uea2_job_vaes_avx512(IMB_MGR *state, IMB_JOB *job) { MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; @@ -364,11 +206,6 @@ static IMB_JOB *(*flush_job_snow3g_uia2_avx512_ptr) /* ====================================================================== */ -IMB_JOB *submit_job_chacha20_enc_dec_avx512(IMB_JOB *job); - -void poly1305_mac_fma_avx512(IMB_JOB *job); -void poly1305_mac_plain_avx512(IMB_JOB *job); - static void (*poly1305_mac_avx512) (IMB_JOB *) = poly1305_mac_plain_avx512; @@ -478,27 +315,6 @@ ethernet_fcs_avx_local(const void *msg, const uint64_t len, #define ETHERNET_FCS ethernet_fcs_avx_local -extern uint32_t ethernet_fcs_avx512(const void *msg, const uint64_t len); -extern uint32_t crc16_x25_avx512(const void *msg, const uint64_t len); -extern uint32_t crc32_sctp_avx512(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_a_avx512(const void *msg, const uint64_t len); -extern uint32_t crc24_lte_b_avx512(const void *msg, const uint64_t len); -extern uint32_t crc16_fp_data_avx512(const void *msg, const uint64_t len); -extern uint32_t crc11_fp_header_avx512(const void *msg, const uint64_t len); -extern uint32_t crc7_fp_header_avx512(const void *msg, const uint64_t len); -extern uint32_t crc10_iuup_data_avx512(const void *msg, const uint64_t len); -extern uint32_t crc6_iuup_header_avx512(const void *msg, const uint64_t len); -extern uint32_t -crc32_wimax_ofdma_data_avx512(const void *msg, const uint64_t len); -extern uint32_t -crc8_wimax_ofdma_hcs_avx512(const void *msg, const uint64_t len); - -extern void snow3g_f9_1_buffer_vaes_avx512(const snow3g_key_schedule_t *pHandle, - const void *pIV, - const void *pBufferIn, - const uint64_t lengthInBits, - void *pDigest); - /* ====================================================================== */ #define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx512 @@ -916,16 +732,10 @@ __forceinline IMB_JOB * SUBMIT_JOB_DOCSIS128_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); -extern void aes_docsis128_dec_crc32_avx512(IMB_JOB *job); -extern void aes_docsis128_dec_crc32_vaes_avx512(IMB_JOB *job); - __forceinline IMB_JOB * SUBMIT_JOB_DOCSIS256_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); -extern void aes_docsis256_dec_crc32_avx512(IMB_JOB *job); -extern void aes_docsis256_dec_crc32_vaes_avx512(IMB_JOB *job); - static IMB_JOB * submit_aes_docsis128_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) @@ -972,30 +782,6 @@ submit_aes_docsis256_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, return job; } -extern IMB_JOB * -submit_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); - -extern IMB_JOB * -submit_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job); -extern IMB_JOB * -flush_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); - static IMB_JOB * submit_job_docsis128_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h new file mode 100644 index 00000000..64ad312e --- /dev/null +++ b/lib/include/arch_avx512_type1.h @@ -0,0 +1,137 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX512 + AESNI + PCLMULQDQ */ + +#ifndef IMB_ASM_AVX512_T1_H +#define IMB_ASM_AVX512_T1_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +IMB_JOB *submit_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_3des_cbc_enc_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_3des_cbc_dec_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_docsis_des_enc_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_docsis_des_dec_avx512(MB_MGR_DES_OOO *state); + +IMB_JOB *submit_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, + IMB_JOB *job); + +IMB_JOB *submit_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state); + +IMB_JOB *submit_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uia2_avx512(MB_MGR_SNOW3G_OOO *state); + +void aes_cmac_256_subkey_gen_avx512(const void *key_exp, + void *key1, void *key2); + +IMB_JOB *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); + +IMB_JOB *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); + +IMB_JOB *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +IMB_JOB *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); + +void poly1305_mac_plain_avx512(IMB_JOB *job); + +IMB_JOB *submit_job_chacha20_enc_dec_avx512(IMB_JOB *job); + +void aes_docsis128_dec_crc32_avx512(IMB_JOB *job); +void aes_docsis256_dec_crc32_avx512(IMB_JOB *job); +IMB_JOB * +submit_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job); +IMB_JOB * +flush_job_aes_docsis128_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); + +IMB_JOB * +submit_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job); +IMB_JOB * +flush_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); + + +#endif /* IMB_ASM_AVX512_T1_H */ + + diff --git a/lib/include/arch_avx512_type2.h b/lib/include/arch_avx512_type2.h new file mode 100644 index 00000000..cdc4516b --- /dev/null +++ b/lib/include/arch_avx512_type2.h @@ -0,0 +1,202 @@ +/******************************************************************************* + Copyright (c) 2012-2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +/* AVX512 + VAES + VPCLMULQDQ + GFNI + FMA */ + + +#ifndef IMB_ASM_AVX512_T2_H +#define IMB_ASM_AVX512_T2_H + +#include "intel-ipsec-mb.h" +#include "ipsec_ooo_mgr.h" + +/* Define interface to base asm code */ + +/* AES-CBC */ +void aes_cbc_dec_128_vaes_avx512(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); +void aes_cbc_dec_192_vaes_avx512(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); +void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes); + +/* AES-CTR */ +void aes_cntr_128_submit_vaes_avx512(IMB_JOB *job); +void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); +void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); + +/* AES-CTR-BITLEN */ +void aes_cntr_bit_128_submit_vaes_avx512(IMB_JOB *job); +void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); +void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); + +/* AES-ECB */ +void aes_ecb_enc_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); +void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, + void *out, uint64_t len_bytes); + +/* AES-CBCS */ +void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes, void *next_iv); + +/* moved from MB MGR */ + +IMB_JOB *submit_job_pon_enc_vaes_avx512(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); + +IMB_JOB *submit_job_pon_enc_no_ctr_vaes_avx512(IMB_JOB *job); +IMB_JOB *submit_job_pon_dec_no_ctr_vaes_avx512(IMB_JOB *job); + +IMB_JOB *submit_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes_xcbc_vaes_avx512(MB_MGR_AES_XCBC_OOO *state); + +IMB_JOB *submit_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes192_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *submit_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_zuc256_eea3_gfni_avx512(MB_MGR_ZUC_OOO *state); + +IMB_JOB *aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job); + +IMB_JOB *aes_cntr_ccm_256_vaes_avx512(IMB_JOB *job); + +IMB_JOB *submit_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, + IMB_JOB *job, + const uint64_t tag_sz); +IMB_JOB *flush_job_zuc256_eia3_gfni_avx512(MB_MGR_ZUC_OOO *state, + const uint64_t tag_sz); + +IMB_JOB *submit_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_aes128_cbcs_1_9_enc_vaes_avx512(MB_MGR_AES_OOO *state); + +IMB_JOB *submit_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uea2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); + +IMB_JOB *submit_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_snow3g_uia2_vaes_avx512(MB_MGR_SNOW3G_OOO *state); + +IMB_JOB *submit_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state); + +IMB_JOB *submit_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes128_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); + +IMB_JOB *submit_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state, + IMB_JOB *job); + +IMB_JOB *flush_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); + +void poly1305_mac_fma_avx512(IMB_JOB *job); + +uint32_t ethernet_fcs_avx512(const void *msg, const uint64_t len); +uint32_t crc16_x25_avx512(const void *msg, const uint64_t len); +uint32_t crc32_sctp_avx512(const void *msg, const uint64_t len); +uint32_t crc24_lte_a_avx512(const void *msg, const uint64_t len); +uint32_t crc24_lte_b_avx512(const void *msg, const uint64_t len); +uint32_t crc16_fp_data_avx512(const void *msg, const uint64_t len); +uint32_t crc11_fp_header_avx512(const void *msg, const uint64_t len); +uint32_t crc7_fp_header_avx512(const void *msg, const uint64_t len); +uint32_t crc10_iuup_data_avx512(const void *msg, const uint64_t len); +uint32_t crc6_iuup_header_avx512(const void *msg, const uint64_t len); +uint32_t crc32_wimax_ofdma_data_avx512(const void *msg, const uint64_t len); +uint32_t crc8_wimax_ofdma_hcs_avx512(const void *msg, const uint64_t len); + +void snow3g_f9_1_buffer_vaes_avx512(const snow3g_key_schedule_t *pHandle, + const void *pIV, + const void *pBufferIn, + const uint64_t lengthInBits, + void *pDigest); + + +void aes_docsis128_dec_crc32_vaes_avx512(IMB_JOB *job); +void aes_docsis256_dec_crc32_vaes_avx512(IMB_JOB *job); + +IMB_JOB * +submit_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job); +IMB_JOB * +flush_job_aes_docsis128_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); + +IMB_JOB * +submit_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job); +IMB_JOB * +flush_job_aes_docsis256_enc_crc32_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state); + + +#endif /* IMB_ASM_AVX512_T2_H */ + diff --git a/lib/include/asm.h b/lib/include/asm.h deleted file mode 100644 index f96c9fad..00000000 --- a/lib/include/asm.h +++ /dev/null @@ -1,88 +0,0 @@ -/******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Intel Corporation nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*******************************************************************************/ - -/* interface to asm routines */ - -#ifndef IMB_ASM_H -#define IMB_ASM_H - -#include "intel-ipsec-mb.h" -#include "ipsec_ooo_mgr.h" - -/* Define interface to base asm code */ - -/* AES-CBC */ -void aes_cbc_dec_128_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_192_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); -void aes_cbc_dec_256_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes); - -/* AES-CTR */ -void aes_cntr_128_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_192_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_256_submit_vaes_avx512(IMB_JOB *job); - -/* AES-CTR-BITLEN */ -void aes_cntr_bit_128_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_bit_192_submit_vaes_avx512(IMB_JOB *job); -void aes_cntr_bit_256_submit_vaes_avx512(IMB_JOB *job); - -/* AES-ECB */ -void aes_ecb_enc_256_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_192_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_enc_128_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -void aes_ecb_dec_256_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_192_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); -void aes_ecb_dec_128_vaes_avx512(const void *in, const void *keys, - void *out, uint64_t len_bytes); - -/* AES-CBCS */ -void aes_cbcs_1_9_dec_128_vaes_avx512(const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, void *next_iv); - -/* stitched AES128-CNTR, CRC32 and BIP */ -IMB_JOB *submit_job_pon_enc_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_vaes_avx512(IMB_JOB *job); - -IMB_JOB *submit_job_pon_enc_no_ctr_vaes_avx512(IMB_JOB *job); -IMB_JOB *submit_job_pon_dec_no_ctr_vaes_avx512(IMB_JOB *job); - -#endif /* IMB_ASM_H */ - - diff --git a/lib/x86_64/aes_xcbc_expand_key.c b/lib/x86_64/aes_xcbc_expand_key.c index ccfdc4d4..50219e52 100644 --- a/lib/x86_64/aes_xcbc_expand_key.c +++ b/lib/x86_64/aes_xcbc_expand_key.c @@ -31,7 +31,6 @@ #ifdef AESNI_EMU #include "include/noaesni.h" #endif -#include "asm.h" #include "include/clear_regs_mem.h" #include "include/error.h" #include "include/arch_noaesni.h" -- GitLab From 33a77b412e05624521049004dc1e0a7585a0a094 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 25 May 2022 13:55:37 +0000 Subject: [PATCH 152/369] sse: [zuc] removed exec attribute from two ZUC source files --- lib/sse/zuc_top_sse.c | 0 lib/sse/zuc_x4_sse.asm | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/sse/zuc_top_sse.c mode change 100755 => 100644 lib/sse/zuc_x4_sse.asm diff --git a/lib/sse/zuc_top_sse.c b/lib/sse/zuc_top_sse.c old mode 100755 new mode 100644 diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse/zuc_x4_sse.asm old mode 100755 new mode 100644 -- GitLab From 74c76a4f255ee7996d2fad9e49264f5f6c556245 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 27 May 2022 09:04:45 +0000 Subject: [PATCH 153/369] lib: [sse] split SSE implementations into three types --- lib/Makefile | 32 +++++++++++++++++-- .../aes128_cbc_dec_by4_sse_no_aesni.asm | 2 +- .../aes128_cbc_enc_x4_sse_no_aesni.asm | 2 +- .../aes128_cbc_mac_x4_sse_no_aesni.asm | 2 +- .../aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm | 2 +- .../aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm | 2 +- lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm | 2 +- .../aes128_cntr_ccm_by8_sse_no_aesni.asm | 2 +- .../aes128_xcbc_mac_x4_sse_no_aesni.asm | 2 +- .../aes192_cbc_dec_by4_sse_no_aesni.asm | 2 +- .../aes192_cbc_enc_x4_sse_no_aesni.asm | 2 +- lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm | 2 +- .../aes256_cbc_dec_by4_sse_no_aesni.asm | 2 +- .../aes256_cbc_enc_x4_sse_no_aesni.asm | 2 +- .../aes256_cbc_mac_x4_sse_no_aesni.asm | 2 +- lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm | 2 +- .../aes256_cntr_ccm_by8_sse_no_aesni.asm | 2 +- lib/no-aesni/aes_cfb_sse_no_aesni.asm | 2 +- lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm | 2 +- lib/no-aesni/crc16_x25_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_by8_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_fp_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_iuup_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_lte_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_sctp_sse_no_aesni.asm | 2 +- lib/no-aesni/crc32_wimax_sse_no_aesni.asm | 2 +- lib/no-aesni/ethernet_fcs_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm128_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm192_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm256_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm | 2 +- lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm | 2 +- ..._mgr_aes128_cbc_enc_flush_sse_no_aesni.asm | 2 +- ...mgr_aes128_cbc_enc_submit_sse_no_aesni.asm | 2 +- ...mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm | 2 +- ...gr_aes128_cbcs_1_9_submit_sse_no_aesni.asm | 2 +- ...128_ccm_auth_submit_flush_sse_no_aesni.asm | 2 +- ..._aes128_cmac_submit_flush_sse_no_aesni.asm | 2 +- .../mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm | 2 +- ...mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm | 2 +- ..._mgr_aes192_cbc_enc_flush_sse_no_aesni.asm | 2 +- ...mgr_aes192_cbc_enc_submit_sse_no_aesni.asm | 2 +- ..._mgr_aes256_cbc_enc_flush_sse_no_aesni.asm | 2 +- ...mgr_aes256_cbc_enc_submit_sse_no_aesni.asm | 2 +- ...256_ccm_auth_submit_flush_sse_no_aesni.asm | 2 +- ..._aes256_cmac_submit_flush_sse_no_aesni.asm | 2 +- .../mb_mgr_zuc_submit_flush_sse_no_aesni.asm | 2 +- lib/no-aesni/pon_sse_no_aesni.asm | 2 +- lib/no-aesni/snow3g_uia2_sse_no_aesni.asm | 2 +- lib/no-aesni/snow_v_sse_no_aesni.asm | 2 +- lib/no-aesni/zuc_sse_no_aesni.asm | 2 +- .../aes128_cbc_dec_by4_sse.asm | 0 lib/{sse => sse_t1}/aes128_cbc_enc_x4_sse.asm | 0 lib/{sse => sse_t1}/aes128_cbc_mac_x4_sse.asm | 2 +- .../aes128_cbcs_1_9_dec_by4_sse.asm | 2 +- .../aes128_cbcs_1_9_enc_x4_sse.asm | 2 +- lib/{sse => sse_t1}/aes128_cntr_by8_sse.asm | 0 .../aes128_cntr_ccm_by8_sse.asm | 2 +- lib/{sse => sse_t1}/aes128_ecb_by4_sse.asm | 0 .../aes128_xcbc_mac_x4_sse.asm | 2 +- .../aes192_cbc_dec_by4_sse.asm | 0 lib/{sse => sse_t1}/aes192_cbc_enc_x4_sse.asm | 0 lib/{sse => sse_t1}/aes192_cntr_by8_sse.asm | 0 lib/{sse => sse_t1}/aes192_ecb_by4_sse.asm | 2 +- .../aes256_cbc_dec_by4_sse.asm | 0 lib/{sse => sse_t1}/aes256_cbc_enc_x4_sse.asm | 0 lib/{sse => sse_t1}/aes256_cbc_mac_x4_sse.asm | 2 +- lib/{sse => sse_t1}/aes256_cntr_by8_sse.asm | 0 .../aes256_cntr_ccm_by8_sse.asm | 2 +- lib/{sse => sse_t1}/aes256_ecb_by4_sse.asm | 2 +- lib/{sse => sse_t1}/aes_cfb_sse.asm | 0 lib/{sse => sse_t1}/chacha20_sse.asm | 0 lib/{sse => sse_t1}/crc16_x25_sse.asm | 0 lib/{sse => sse_t1}/crc32_by8_sse.asm | 0 lib/{sse => sse_t1}/crc32_fp_sse.asm | 0 lib/{sse => sse_t1}/crc32_iuup_sse.asm | 0 lib/{sse => sse_t1}/crc32_lte_sse.asm | 0 lib/{sse => sse_t1}/crc32_refl_by8_sse.asm | 0 lib/{sse => sse_t1}/crc32_sctp_sse.asm | 0 lib/{sse => sse_t1}/crc32_wimax_sse.asm | 0 lib/{sse => sse_t1}/ethernet_fcs_sse.asm | 0 lib/{sse => sse_t1}/gcm128_api_by8_sse.asm | 2 +- .../gcm128_gmac_api_by8_sse.asm | 2 +- .../gcm128_sgl_api_by8_sse.asm | 2 +- lib/{sse => sse_t1}/gcm192_api_by8_sse.asm | 2 +- .../gcm192_gmac_api_by8_sse.asm | 2 +- .../gcm192_sgl_api_by8_sse.asm | 2 +- lib/{sse => sse_t1}/gcm256_api_by8_sse.asm | 2 +- .../gcm256_gmac_api_by8_sse.asm | 2 +- .../gcm256_sgl_api_by8_sse.asm | 2 +- lib/{sse => sse_t1}/gcm_api_sse.inc | 2 +- lib/{sse => sse_t1}/gcm_gmac_api_sse.inc | 2 +- lib/{sse => sse_t1}/gcm_sgl_api_sse.inc | 2 +- lib/{sse => sse_t1}/gcm_sse.inc | 0 lib/{sse => sse_t1}/kasumi_sse.c | 0 .../mb_mgr_aes128_cbc_enc_flush_x4_sse.asm | 0 .../mb_mgr_aes128_cbc_enc_submit_x4_sse.asm | 0 .../mb_mgr_aes128_cbcs_1_9_flush_sse.asm | 0 .../mb_mgr_aes128_cbcs_1_9_submit_sse.asm | 0 ...gr_aes128_ccm_auth_submit_flush_x4_sse.asm | 0 ...mb_mgr_aes128_cmac_submit_flush_x4_sse.asm | 0 .../mb_mgr_aes128_xcbc_flush_x4_sse.asm | 0 .../mb_mgr_aes128_xcbc_submit_x4_sse.asm | 0 .../mb_mgr_aes192_cbc_enc_flush_x4_sse.asm | 2 +- .../mb_mgr_aes192_cbc_enc_submit_x4_sse.asm | 2 +- .../mb_mgr_aes256_cbc_enc_flush_x4_sse.asm | 2 +- .../mb_mgr_aes256_cbc_enc_submit_x4_sse.asm | 2 +- ...gr_aes256_ccm_auth_submit_flush_x4_sse.asm | 2 +- ...mb_mgr_aes256_cmac_submit_flush_x4_sse.asm | 2 +- .../mb_mgr_hmac_md5_flush_sse.asm | 0 .../mb_mgr_hmac_md5_submit_sse.asm | 0 .../mb_mgr_hmac_sha1_flush_sse.asm | 0 .../mb_mgr_hmac_sha1_submit_sse.asm | 0 .../mb_mgr_hmac_sha224_flush_sse.asm | 2 +- .../mb_mgr_hmac_sha224_submit_sse.asm | 2 +- .../mb_mgr_hmac_sha256_flush_sse.asm | 0 .../mb_mgr_hmac_sha256_submit_sse.asm | 0 .../mb_mgr_hmac_sha384_flush_sse.asm | 2 +- .../mb_mgr_hmac_sha384_submit_sse.asm | 2 +- .../mb_mgr_hmac_sha512_flush_sse.asm | 0 .../mb_mgr_hmac_sha512_submit_sse.asm | 0 lib/{sse => sse_t1}/mb_mgr_sse.c | 0 .../mb_mgr_zuc_submit_flush_sse.asm | 0 lib/{sse => sse_t1}/md5_x4x2_sse.asm | 0 lib/{sse => sse_t1}/memcpy_sse.asm | 0 lib/{sse => sse_t1}/pon_by8_sse.asm | 0 lib/{sse => sse_t1}/sha1_one_block_sse.asm | 0 lib/{sse => sse_t1}/sha1_x4_sse.asm | 0 lib/{sse => sse_t1}/sha224_one_block_sse.asm | 2 +- lib/{sse => sse_t1}/sha256_mult_sse.asm | 0 lib/{sse => sse_t1}/sha256_one_block_sse.asm | 0 lib/{sse => sse_t1}/sha384_one_block_sse.asm | 2 +- lib/{sse => sse_t1}/sha512_one_block_sse.asm | 0 lib/{sse => sse_t1}/sha512_x2_sse.asm | 0 lib/{sse => sse_t1}/sha_mb_sse.c | 0 lib/{sse => sse_t1}/sha_sse.c | 0 lib/{sse => sse_t1}/snow3g_sse.c | 0 lib/{sse => sse_t1}/snow3g_uia2_by4_sse.asm | 0 lib/{sse => sse_t1}/snow_v_sse.asm | 0 lib/{sse => sse_t1}/zuc_top_sse.c | 0 lib/{sse => sse_t1}/zuc_x4_sse.asm | 0 .../mb_mgr_hmac_sha1_flush_ni_sse.asm | 0 .../mb_mgr_hmac_sha1_submit_ni_sse.asm | 0 .../mb_mgr_hmac_sha224_flush_ni_sse.asm | 2 +- .../mb_mgr_hmac_sha224_submit_ni_sse.asm | 2 +- .../mb_mgr_hmac_sha256_flush_ni_sse.asm | 0 .../mb_mgr_hmac_sha256_submit_ni_sse.asm | 0 lib/{sse => sse_t2}/sha1_ni_x2_sse.asm | 0 lib/{sse => sse_t2}/sha256_ni_x2_sse.asm | 0 .../aes128_cbc_dec_by8_sse.asm | 0 lib/{sse => sse_t3}/aes128_cbc_enc_x8_sse.asm | 0 lib/{sse => sse_t3}/aes128_cbc_mac_x8_sse.asm | 2 +- lib/{sse => sse_t3}/aes128_ecb_by8_sse.asm | 0 .../aes192_cbc_dec_by8_sse.asm | 0 lib/{sse => sse_t3}/aes192_cbc_enc_x8_sse.asm | 0 lib/{sse => sse_t3}/aes192_ecb_by8_sse.asm | 2 +- .../aes256_cbc_dec_by8_sse.asm | 0 lib/{sse => sse_t3}/aes256_cbc_enc_x8_sse.asm | 0 lib/{sse => sse_t3}/aes256_cbc_mac_x8_sse.asm | 2 +- lib/{sse => sse_t3}/aes256_ecb_by8_sse.asm | 2 +- .../mb_mgr_aes128_cbc_enc_flush_x8_sse.asm | 2 +- .../mb_mgr_aes128_cbc_enc_submit_x8_sse.asm | 2 +- ...gr_aes128_ccm_auth_submit_flush_x8_sse.asm | 2 +- ...mb_mgr_aes128_cmac_submit_flush_x8_sse.asm | 2 +- .../mb_mgr_aes192_cbc_enc_flush_x8_sse.asm | 2 +- .../mb_mgr_aes192_cbc_enc_submit_x8_sse.asm | 2 +- .../mb_mgr_aes256_cbc_enc_flush_x8_sse.asm | 2 +- .../mb_mgr_aes256_cbc_enc_submit_x8_sse.asm | 2 +- ...gr_aes256_ccm_auth_submit_flush_x8_sse.asm | 2 +- ...mb_mgr_aes256_cmac_submit_flush_x8_sse.asm | 2 +- .../mb_mgr_zuc_submit_flush_gfni_sse.asm | 2 +- lib/{sse => sse_t3}/zuc_x4_gfni_sse.asm | 2 +- lib/win_x64.mak | 18 +++++++++-- 178 files changed, 152 insertions(+), 110 deletions(-) rename lib/{sse => sse_t1}/aes128_cbc_dec_by4_sse.asm (100%) rename lib/{sse => sse_t1}/aes128_cbc_enc_x4_sse.asm (100%) rename lib/{sse => sse_t1}/aes128_cbc_mac_x4_sse.asm (97%) rename lib/{sse => sse_t1}/aes128_cbcs_1_9_dec_by4_sse.asm (97%) rename lib/{sse => sse_t1}/aes128_cbcs_1_9_enc_x4_sse.asm (98%) rename lib/{sse => sse_t1}/aes128_cntr_by8_sse.asm (100%) rename lib/{sse => sse_t1}/aes128_cntr_ccm_by8_sse.asm (97%) rename lib/{sse => sse_t1}/aes128_ecb_by4_sse.asm (100%) rename lib/{sse => sse_t1}/aes128_xcbc_mac_x4_sse.asm (98%) rename lib/{sse => sse_t1}/aes192_cbc_dec_by4_sse.asm (100%) rename lib/{sse => sse_t1}/aes192_cbc_enc_x4_sse.asm (100%) rename lib/{sse => sse_t1}/aes192_cntr_by8_sse.asm (100%) rename lib/{sse => sse_t1}/aes192_ecb_by4_sse.asm (97%) rename lib/{sse => sse_t1}/aes256_cbc_dec_by4_sse.asm (100%) rename lib/{sse => sse_t1}/aes256_cbc_enc_x4_sse.asm (100%) rename lib/{sse => sse_t1}/aes256_cbc_mac_x4_sse.asm (97%) rename lib/{sse => sse_t1}/aes256_cntr_by8_sse.asm (100%) rename lib/{sse => sse_t1}/aes256_cntr_ccm_by8_sse.asm (97%) rename lib/{sse => sse_t1}/aes256_ecb_by4_sse.asm (97%) rename lib/{sse => sse_t1}/aes_cfb_sse.asm (100%) rename lib/{sse => sse_t1}/chacha20_sse.asm (100%) rename lib/{sse => sse_t1}/crc16_x25_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_by8_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_fp_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_iuup_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_lte_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_refl_by8_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_sctp_sse.asm (100%) rename lib/{sse => sse_t1}/crc32_wimax_sse.asm (100%) rename lib/{sse => sse_t1}/ethernet_fcs_sse.asm (100%) rename lib/{sse => sse_t1}/gcm128_api_by8_sse.asm (98%) rename lib/{sse => sse_t1}/gcm128_gmac_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm128_sgl_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm192_api_by8_sse.asm (98%) rename lib/{sse => sse_t1}/gcm192_gmac_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm192_sgl_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm256_api_by8_sse.asm (98%) rename lib/{sse => sse_t1}/gcm256_gmac_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm256_sgl_api_by8_sse.asm (97%) rename lib/{sse => sse_t1}/gcm_api_sse.inc (99%) rename lib/{sse => sse_t1}/gcm_gmac_api_sse.inc (99%) rename lib/{sse => sse_t1}/gcm_sgl_api_sse.inc (99%) rename lib/{sse => sse_t1}/gcm_sse.inc (100%) rename lib/{sse => sse_t1}/kasumi_sse.c (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_cbcs_1_9_flush_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_cbcs_1_9_submit_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_xcbc_flush_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes128_xcbc_submit_x4_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_hmac_md5_flush_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_md5_submit_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha1_flush_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha1_submit_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha224_flush_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha224_submit_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha256_flush_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha256_submit_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha384_flush_sse.asm (97%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha384_submit_sse.asm (96%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha512_flush_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_hmac_sha512_submit_sse.asm (100%) rename lib/{sse => sse_t1}/mb_mgr_sse.c (100%) rename lib/{sse => sse_t1}/mb_mgr_zuc_submit_flush_sse.asm (100%) rename lib/{sse => sse_t1}/md5_x4x2_sse.asm (100%) rename lib/{sse => sse_t1}/memcpy_sse.asm (100%) rename lib/{sse => sse_t1}/pon_by8_sse.asm (100%) rename lib/{sse => sse_t1}/sha1_one_block_sse.asm (100%) rename lib/{sse => sse_t1}/sha1_x4_sse.asm (100%) rename lib/{sse => sse_t1}/sha224_one_block_sse.asm (97%) rename lib/{sse => sse_t1}/sha256_mult_sse.asm (100%) rename lib/{sse => sse_t1}/sha256_one_block_sse.asm (100%) rename lib/{sse => sse_t1}/sha384_one_block_sse.asm (97%) rename lib/{sse => sse_t1}/sha512_one_block_sse.asm (100%) rename lib/{sse => sse_t1}/sha512_x2_sse.asm (100%) rename lib/{sse => sse_t1}/sha_mb_sse.c (100%) rename lib/{sse => sse_t1}/sha_sse.c (100%) rename lib/{sse => sse_t1}/snow3g_sse.c (100%) rename lib/{sse => sse_t1}/snow3g_uia2_by4_sse.asm (100%) rename lib/{sse => sse_t1}/snow_v_sse.asm (100%) rename lib/{sse => sse_t1}/zuc_top_sse.c (100%) rename lib/{sse => sse_t1}/zuc_x4_sse.asm (100%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha1_flush_ni_sse.asm (100%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha1_submit_ni_sse.asm (100%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha224_flush_ni_sse.asm (96%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha224_submit_ni_sse.asm (96%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha256_flush_ni_sse.asm (100%) rename lib/{sse => sse_t2}/mb_mgr_hmac_sha256_submit_ni_sse.asm (100%) rename lib/{sse => sse_t2}/sha1_ni_x2_sse.asm (100%) rename lib/{sse => sse_t2}/sha256_ni_x2_sse.asm (100%) rename lib/{sse => sse_t3}/aes128_cbc_dec_by8_sse.asm (100%) rename lib/{sse => sse_t3}/aes128_cbc_enc_x8_sse.asm (100%) rename lib/{sse => sse_t3}/aes128_cbc_mac_x8_sse.asm (97%) rename lib/{sse => sse_t3}/aes128_ecb_by8_sse.asm (100%) rename lib/{sse => sse_t3}/aes192_cbc_dec_by8_sse.asm (100%) rename lib/{sse => sse_t3}/aes192_cbc_enc_x8_sse.asm (100%) rename lib/{sse => sse_t3}/aes192_ecb_by8_sse.asm (97%) rename lib/{sse => sse_t3}/aes256_cbc_dec_by8_sse.asm (100%) rename lib/{sse => sse_t3}/aes256_cbc_enc_x8_sse.asm (100%) rename lib/{sse => sse_t3}/aes256_cbc_mac_x8_sse.asm (97%) rename lib/{sse => sse_t3}/aes256_ecb_by8_sse.asm (97%) rename lib/{sse => sse_t3}/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm (96%) rename lib/{sse => sse_t3}/mb_mgr_zuc_submit_flush_gfni_sse.asm (97%) rename lib/{sse => sse_t3}/zuc_x4_gfni_sse.asm (98%) diff --git a/lib/Makefile b/lib/Makefile index 9eea98d4..8ce31937 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -795,10 +795,38 @@ ifeq ($(CC_HAS_CET),1) mv $@.tmp $@ endif -$(OBJ_DIR)/%.o:sse/%.c +$(OBJ_DIR)/%.o:sse_t1/%.c $(CC) -MMD $(OPT_SSE) -c $(CFLAGS) $< -o $@ -$(OBJ_DIR)/%.o:sse/%.asm +$(OBJ_DIR)/%.o:sse_t1/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +$(OBJ_DIR)/%.o:sse_t2/%.c + $(CC) -MMD $(OPT_SSE) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:sse_t2/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +$(OBJ_DIR)/%.o:sse_t3/%.c + $(CC) -MMD $(OPT_SSE) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:sse_t3/%.asm ifeq ($(USE_YASM),y) $(YASM) $(YASM_FLAGS) $< -o $@ else diff --git a/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm index 6e710bfe..7fef27fe 100644 --- a/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_dec_by4_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_DEC_128 aes_cbc_dec_128_sse_no_aesni -%include "sse/aes128_cbc_dec_by4_sse.asm" +%include "sse_t1/aes128_cbc_dec_by4_sse.asm" diff --git a/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm index c36f1b1b..c925419a 100644 --- a/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_enc_x4_sse_no_aesni.asm @@ -35,4 +35,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm index df586a7b..999176ea 100644 --- a/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbc_mac_x4_sse_no_aesni.asm @@ -36,4 +36,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm index 2a3286a8..c30441ad 100644 --- a/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbcs_1_9_dec_by4_sse_no_aesni.asm @@ -32,4 +32,4 @@ %define CBCS %endif -%include "sse/aes128_cbc_dec_by4_sse.asm" +%include "sse_t1/aes128_cbc_dec_by4_sse.asm" diff --git a/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm index 91a001c1..ce0f0cb1 100644 --- a/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cbcs_1_9_enc_x4_sse_no_aesni.asm @@ -51,4 +51,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm index 257ba64b..5e001722 100644 --- a/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cntr_by8_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define AES_CNTR_128 aes_cntr_128_sse_no_aesni %define AES_CNTR_BIT_128 aes_cntr_bit_128_sse_no_aesni -%include "sse/aes128_cntr_by8_sse.asm" +%include "sse_t1/aes128_cntr_by8_sse.asm" diff --git a/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm b/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm index 4702c4bb..e3e5d0f4 100644 --- a/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_cntr_ccm_by8_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse_no_aesni -%include "sse/aes128_cntr_ccm_by8_sse.asm" +%include "sse_t1/aes128_cntr_ccm_by8_sse.asm" diff --git a/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm index 4c109a51..20c4ff6c 100644 --- a/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes128_xcbc_mac_x4_sse_no_aesni.asm @@ -34,4 +34,4 @@ %define ARG_KEYS _aesxcbcarg_keys %define ARG_IV _aesxcbcarg_ICV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm index 349e06bf..b4d9c7ae 100644 --- a/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cbc_dec_by4_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_DEC_192 aes_cbc_dec_192_sse_no_aesni -%include "sse/aes192_cbc_dec_by4_sse.asm" +%include "sse_t1/aes192_cbc_dec_by4_sse.asm" diff --git a/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm index 0d74ea19..720ccb50 100644 --- a/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cbc_enc_x4_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_ENC_X4 aes_cbc_enc_192_x4_no_aesni -%include "sse/aes192_cbc_enc_x4_sse.asm" +%include "sse_t1/aes192_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm index 6707acee..4f9cb9ad 100644 --- a/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes192_cntr_by8_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define AES_CNTR_192 aes_cntr_192_sse_no_aesni %define AES_CNTR_BIT_192 aes_cntr_bit_192_sse_no_aesni -%include "sse/aes192_cntr_by8_sse.asm" +%include "sse_t1/aes192_cntr_by8_sse.asm" diff --git a/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm index 57b7b5a3..a4f1d010 100644 --- a/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_dec_by4_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_DEC_256 aes_cbc_dec_256_sse_no_aesni -%include "sse/aes256_cbc_dec_by4_sse.asm" +%include "sse_t1/aes256_cbc_dec_by4_sse.asm" diff --git a/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm index cc6a9479..4c6ae009 100644 --- a/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_enc_x4_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_ENC_X4 aes_cbc_enc_256_x4_no_aesni -%include "sse/aes256_cbc_enc_x4_sse.asm" +%include "sse_t1/aes256_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm b/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm index a7bf17ad..ee31c351 100644 --- a/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cbc_mac_x4_sse_no_aesni.asm @@ -30,4 +30,4 @@ %include "include/aesni_emu.inc" %define AES_CBC_ENC_X4 %define CBC_MAC -%include "sse/aes256_cbc_enc_x4_sse.asm" +%include "sse_t1/aes256_cbc_enc_x4_sse.asm" diff --git a/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm b/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm index 403b1b82..00811ff1 100644 --- a/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cntr_by8_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define AES_CNTR_256 aes_cntr_256_sse_no_aesni %define AES_CNTR_BIT_256 aes_cntr_bit_256_sse_no_aesni -%include "sse/aes256_cntr_by8_sse.asm" +%include "sse_t1/aes256_cntr_by8_sse.asm" diff --git a/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm b/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm index 0d4f3ef8..8057968f 100644 --- a/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm +++ b/lib/no-aesni/aes256_cntr_ccm_by8_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse_no_aesni -%include "sse/aes256_cntr_ccm_by8_sse.asm" +%include "sse_t1/aes256_cntr_ccm_by8_sse.asm" diff --git a/lib/no-aesni/aes_cfb_sse_no_aesni.asm b/lib/no-aesni/aes_cfb_sse_no_aesni.asm index 9444682e..8c033ce4 100644 --- a/lib/no-aesni/aes_cfb_sse_no_aesni.asm +++ b/lib/no-aesni/aes_cfb_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define AES_CFB_128_ONE aes_cfb_128_one_sse_no_aesni %define AES_CFB_256_ONE aes_cfb_256_one_sse_no_aesni -%include "sse/aes_cfb_sse.asm" +%include "sse_t1/aes_cfb_sse.asm" diff --git a/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm b/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm index 63c6300f..f5072f67 100644 --- a/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm +++ b/lib/no-aesni/aes_ecb_by4_sse_no_aesni.asm @@ -32,4 +32,4 @@ %define AES_ECB_DEC_128 aes_ecb_dec_128_sse_no_aesni %define AES_ECB_DEC_192 aes_ecb_dec_192_sse_no_aesni %define AES_ECB_DEC_256 aes_ecb_dec_256_sse_no_aesni -%include "sse/aes128_ecb_by4_sse.asm" +%include "sse_t1/aes128_ecb_by4_sse.asm" diff --git a/lib/no-aesni/crc16_x25_sse_no_aesni.asm b/lib/no-aesni/crc16_x25_sse_no_aesni.asm index 4fff6211..9c46b805 100644 --- a/lib/no-aesni/crc16_x25_sse_no_aesni.asm +++ b/lib/no-aesni/crc16_x25_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define CRC16_X25_FN crc16_x25_sse_no_aesni %define CRC32_REFL_FN crc32_refl_by8_sse_no_aesni -%include "sse/crc16_x25_sse.asm" +%include "sse_t1/crc16_x25_sse.asm" diff --git a/lib/no-aesni/crc32_by8_sse_no_aesni.asm b/lib/no-aesni/crc32_by8_sse_no_aesni.asm index 9076e0f9..141c626b 100644 --- a/lib/no-aesni/crc32_by8_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_by8_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_by8_sse.asm" +%include "sse_t1/crc32_by8_sse.asm" diff --git a/lib/no-aesni/crc32_fp_sse_no_aesni.asm b/lib/no-aesni/crc32_fp_sse_no_aesni.asm index 20b40ad0..4dccb4e9 100644 --- a/lib/no-aesni/crc32_fp_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_fp_sse_no_aesni.asm @@ -30,4 +30,4 @@ %define CRC11_FP_HEADER_FN crc11_fp_header_sse_no_aesni %define CRC7_FP_HEADER_FN crc7_fp_header_sse_no_aesni %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_fp_sse.asm" +%include "sse_t1/crc32_fp_sse.asm" diff --git a/lib/no-aesni/crc32_iuup_sse_no_aesni.asm b/lib/no-aesni/crc32_iuup_sse_no_aesni.asm index 9aa73047..663acc2b 100644 --- a/lib/no-aesni/crc32_iuup_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_iuup_sse_no_aesni.asm @@ -29,4 +29,4 @@ %define CRC10_IUUP_DATA_FN crc10_iuup_data_sse_no_aesni %define CRC6_IUUP_HEADER_FN crc6_iuup_header_sse_no_aesni %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_iuup_sse.asm" +%include "sse_t1/crc32_iuup_sse.asm" diff --git a/lib/no-aesni/crc32_lte_sse_no_aesni.asm b/lib/no-aesni/crc32_lte_sse_no_aesni.asm index 8e7fac8c..9b21b49e 100644 --- a/lib/no-aesni/crc32_lte_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_lte_sse_no_aesni.asm @@ -29,4 +29,4 @@ %define CRC32_LTE24A_FN crc24_lte_a_sse_no_aesni %define CRC32_LTE24B_FN crc24_lte_b_sse_no_aesni %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_lte_sse.asm" +%include "sse_t1/crc32_lte_sse.asm" diff --git a/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm b/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm index 1d7c3c68..53027208 100644 --- a/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_refl_by8_sse_no_aesni.asm @@ -27,4 +27,4 @@ %include "include/aesni_emu.inc" %define CRC32_REFL_FN crc32_refl_by8_sse_no_aesni -%include "sse/crc32_refl_by8_sse.asm" +%include "sse_t1/crc32_refl_by8_sse.asm" diff --git a/lib/no-aesni/crc32_sctp_sse_no_aesni.asm b/lib/no-aesni/crc32_sctp_sse_no_aesni.asm index 3b438b55..2f7d9b6b 100644 --- a/lib/no-aesni/crc32_sctp_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_sctp_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define CRC32_SCTP_FN crc32_sctp_sse_no_aesni %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_sctp_sse.asm" +%include "sse_t1/crc32_sctp_sse.asm" diff --git a/lib/no-aesni/crc32_wimax_sse_no_aesni.asm b/lib/no-aesni/crc32_wimax_sse_no_aesni.asm index 3c503913..0c694829 100644 --- a/lib/no-aesni/crc32_wimax_sse_no_aesni.asm +++ b/lib/no-aesni/crc32_wimax_sse_no_aesni.asm @@ -29,4 +29,4 @@ %define CRC32_WIMAX_DATA_FN crc32_wimax_ofdma_data_sse_no_aesni %define CRC8_WIMAX_HCS_FN crc8_wimax_ofdma_hcs_sse_no_aesni %define CRC32_FN crc32_by8_sse_no_aesni -%include "sse/crc32_wimax_sse.asm" +%include "sse_t1/crc32_wimax_sse.asm" diff --git a/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm b/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm index 8e5bbcdb..c0f9f0fc 100644 --- a/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm +++ b/lib/no-aesni/ethernet_fcs_sse_no_aesni.asm @@ -29,4 +29,4 @@ %define ETHERNET_FCS_FN ethernet_fcs_sse_no_aesni %define ETHERNET_FCS_FN_LOCAL ethernet_fcs_sse_no_aesni_local %define CRC32_REFL_FN crc32_refl_by8_sse_no_aesni -%include "sse/ethernet_fcs_sse.asm" +%include "sse_t1/ethernet_fcs_sse.asm" diff --git a/lib/no-aesni/gcm128_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_api_sse_no_aesni.asm index 4d15cc6a..e116bcf6 100644 --- a/lib/no-aesni/gcm128_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm128_api_by8_sse.asm" +%include "sse_t1/gcm128_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm index 4ad41f80..8fa5c283 100644 --- a/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_gmac_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm128_gmac_api_by8_sse.asm" +%include "sse_t1/gcm128_gmac_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm index 40c336a2..a6750c2f 100644 --- a/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm128_sgl_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm128_sgl_api_by8_sse.asm" +%include "sse_t1/gcm128_sgl_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm192_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_api_sse_no_aesni.asm index 0be37a4d..efe16366 100644 --- a/lib/no-aesni/gcm192_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm192_api_by8_sse.asm" +%include "sse_t1/gcm192_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm index e6cce7c7..8c6345be 100644 --- a/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_gmac_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm192_gmac_api_by8_sse.asm" +%include "sse_t1/gcm192_gmac_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm index d1bf7f07..7a0a962e 100644 --- a/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm192_sgl_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm192_sgl_api_by8_sse.asm" +%include "sse_t1/gcm192_sgl_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm256_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_api_sse_no_aesni.asm index 7aeae369..320ec91e 100644 --- a/lib/no-aesni/gcm256_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm256_api_by8_sse.asm" +%include "sse_t1/gcm256_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm index a484dfd4..25074319 100644 --- a/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_gmac_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm256_gmac_api_by8_sse.asm" +%include "sse_t1/gcm256_gmac_api_by8_sse.asm" diff --git a/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm b/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm index 6d9cac6e..713924e7 100644 --- a/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm +++ b/lib/no-aesni/gcm256_sgl_api_sse_no_aesni.asm @@ -29,4 +29,4 @@ %include "include/aesni_emu.inc" %define NO_AESNI -%include "sse/gcm256_sgl_api_by8_sse.asm" +%include "sse_t1/gcm256_sgl_api_by8_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm index 7e7bc17b..f0cc5e80 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbc_enc_flush_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_128_x4_no_aesni %define FLUSH_JOB_AES_ENC flush_job_aes128_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm index 2e53cf8e..76332f37 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbc_enc_submit_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_128_x4_no_aesni %define SUBMIT_JOB_AES_ENC submit_job_aes128_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm index fe489abb..a6bca7a0 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_flush_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBCS_ENC_X4 aes_cbcs_1_9_enc_128_x4_no_aesni %define FLUSH_JOB_AES_CBCS_ENC flush_job_aes128_cbcs_1_9_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbcs_1_9_flush_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm index 341c0bfc..1793d305 100644 --- a/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cbcs_1_9_submit_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBCS_ENC_X4 aes_cbcs_1_9_enc_128_x4_no_aesni %define SUBMIT_JOB_AES_CBCS_ENC submit_job_aes128_cbcs_1_9_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbcs_1_9_submit_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm index a12ce6b5..e8862839 100644 --- a/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_ccm_auth_submit_flush_sse_no_aesni.asm @@ -29,4 +29,4 @@ %define AES_CBC_MAC aes128_cbc_mac_x4_no_aesni %define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes128_ccm_auth_sse_no_aesni %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes128_ccm_auth_sse_no_aesni -%include "sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm index 419fa21b..dfd49155 100644 --- a/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_cmac_submit_flush_sse_no_aesni.asm @@ -28,4 +28,4 @@ %define AES_CBC_MAC aes128_cbc_mac_x4_no_aesni %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes128_cmac_auth_sse_no_aesni %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes128_cmac_auth_sse_no_aesni -%include "sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm index 92db2710..116f0ab1 100644 --- a/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_xcbc_flush_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_XCBC_X4 aes_xcbc_mac_128_x4_no_aesni %define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni -%include "sse/mb_mgr_aes128_xcbc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm index 4da294b7..20b036ea 100644 --- a/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes128_xcbc_submit_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_XCBC_X4 aes_xcbc_mac_128_x4_no_aesni %define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse_no_aesni -%include "sse/mb_mgr_aes128_xcbc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm index 4a84bef5..e700ca62 100644 --- a/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes192_cbc_enc_flush_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x4_no_aesni %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm index cf864cc2..50c86fe7 100644 --- a/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes192_cbc_enc_submit_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x4_no_aesni %define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm index c784aca3..4256da5f 100644 --- a/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cbc_enc_flush_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x4_no_aesni %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm index aee87cb1..46de48da 100644 --- a/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cbc_enc_submit_sse_no_aesni.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x4_no_aesni %define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_sse_no_aesni -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm index 52e3fe85..02af37da 100644 --- a/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_ccm_auth_submit_flush_sse_no_aesni.asm @@ -30,4 +30,4 @@ %define AES_CBC_MAC aes256_cbc_mac_x4_no_aesni %define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes256_ccm_auth_sse_no_aesni %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes256_ccm_auth_sse_no_aesni -%include "sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm index 0aa7fedd..7d543d0e 100644 --- a/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_aes256_cmac_submit_flush_sse_no_aesni.asm @@ -28,4 +28,4 @@ %define AES_CBC_MAC aes256_cbc_mac_x4_no_aesni %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes256_cmac_auth_sse_no_aesni %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes256_cmac_auth_sse_no_aesni -%include "sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" diff --git a/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm b/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm index 5f408376..2b3009c2 100644 --- a/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm +++ b/lib/no-aesni/mb_mgr_zuc_submit_flush_sse_no_aesni.asm @@ -38,4 +38,4 @@ %define ZUC128_INIT_4 asm_ZucInitialization_4_sse_no_aesni %define ZUC256_INIT_4 asm_Zuc256Initialization_4_sse_no_aesni %define ZUC_CIPHER_4 asm_ZucCipher_4_sse_no_aesni -%include "sse/mb_mgr_zuc_submit_flush_sse.asm" +%include "sse_t1/mb_mgr_zuc_submit_flush_sse.asm" diff --git a/lib/no-aesni/pon_sse_no_aesni.asm b/lib/no-aesni/pon_sse_no_aesni.asm index 9211eacc..65170f1d 100644 --- a/lib/no-aesni/pon_sse_no_aesni.asm +++ b/lib/no-aesni/pon_sse_no_aesni.asm @@ -33,4 +33,4 @@ %define ENC_NO_CTR_FN_NAME submit_job_pon_enc_no_ctr_sse_no_aesni %define HEC_32 hec_32_sse_no_aesni %define HEC_64 hec_64_sse_no_aesni -%include "sse/pon_by8_sse.asm" +%include "sse_t1/pon_by8_sse.asm" diff --git a/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm b/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm index 65f1f51f..676912cf 100644 --- a/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm +++ b/lib/no-aesni/snow3g_uia2_sse_no_aesni.asm @@ -28,5 +28,5 @@ %include "include/aesni_emu.inc" %define NO_AESNI %define SNOW3G_F9_1_BUFFER_INTERNAL snow3g_f9_1_buffer_internal_sse_no_aesni -%include "sse/snow3g_uia2_by4_sse.asm" +%include "sse_t1/snow3g_uia2_by4_sse.asm" diff --git a/lib/no-aesni/snow_v_sse_no_aesni.asm b/lib/no-aesni/snow_v_sse_no_aesni.asm index 04a0ed60..65559065 100644 --- a/lib/no-aesni/snow_v_sse_no_aesni.asm +++ b/lib/no-aesni/snow_v_sse_no_aesni.asm @@ -28,4 +28,4 @@ %include "include/aesni_emu.inc" %define SNOW_V snow_v_sse_no_aesni %define SNOW_V_AEAD_INIT snow_v_aead_init_sse_no_aesni -%include "sse/snow_v_sse.asm" +%include "sse_t1/snow_v_sse.asm" diff --git a/lib/no-aesni/zuc_sse_no_aesni.asm b/lib/no-aesni/zuc_sse_no_aesni.asm index c2ccc003..d43e99d7 100644 --- a/lib/no-aesni/zuc_sse_no_aesni.asm +++ b/lib/no-aesni/zuc_sse_no_aesni.asm @@ -35,4 +35,4 @@ %define ZUC_EIA3ROUND16B asm_Eia3Round16B_sse_no_aesni %define ZUC_EIA3REMAINDER asm_Eia3Remainder_sse_no_aesni %define USE_GFNI 0 -%include "sse/zuc_x4_sse.asm" +%include "sse_t1/zuc_x4_sse.asm" diff --git a/lib/sse/aes128_cbc_dec_by4_sse.asm b/lib/sse_t1/aes128_cbc_dec_by4_sse.asm similarity index 100% rename from lib/sse/aes128_cbc_dec_by4_sse.asm rename to lib/sse_t1/aes128_cbc_dec_by4_sse.asm diff --git a/lib/sse/aes128_cbc_enc_x4_sse.asm b/lib/sse_t1/aes128_cbc_enc_x4_sse.asm similarity index 100% rename from lib/sse/aes128_cbc_enc_x4_sse.asm rename to lib/sse_t1/aes128_cbc_enc_x4_sse.asm diff --git a/lib/sse/aes128_cbc_mac_x4_sse.asm b/lib/sse_t1/aes128_cbc_mac_x4_sse.asm similarity index 97% rename from lib/sse/aes128_cbc_mac_x4_sse.asm rename to lib/sse_t1/aes128_cbc_mac_x4_sse.asm index 02d526c8..52726f1b 100644 --- a/lib/sse/aes128_cbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes128_cbc_mac_x4_sse.asm @@ -34,4 +34,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/sse/aes128_cbcs_1_9_dec_by4_sse.asm b/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm similarity index 97% rename from lib/sse/aes128_cbcs_1_9_dec_by4_sse.asm rename to lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm index 875b7fe2..fcef251c 100644 --- a/lib/sse/aes128_cbcs_1_9_dec_by4_sse.asm +++ b/lib/sse_t1/aes128_cbcs_1_9_dec_by4_sse.asm @@ -30,4 +30,4 @@ %define CBCS %endif -%include "sse/aes128_cbc_dec_by4_sse.asm" +%include "sse_t1/aes128_cbc_dec_by4_sse.asm" diff --git a/lib/sse/aes128_cbcs_1_9_enc_x4_sse.asm b/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm similarity index 98% rename from lib/sse/aes128_cbcs_1_9_enc_x4_sse.asm rename to lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm index 1181b75f..77e9a3a0 100644 --- a/lib/sse/aes128_cbcs_1_9_enc_x4_sse.asm +++ b/lib/sse_t1/aes128_cbcs_1_9_enc_x4_sse.asm @@ -49,4 +49,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/sse/aes128_cntr_by8_sse.asm b/lib/sse_t1/aes128_cntr_by8_sse.asm similarity index 100% rename from lib/sse/aes128_cntr_by8_sse.asm rename to lib/sse_t1/aes128_cntr_by8_sse.asm diff --git a/lib/sse/aes128_cntr_ccm_by8_sse.asm b/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm similarity index 97% rename from lib/sse/aes128_cntr_ccm_by8_sse.asm rename to lib/sse_t1/aes128_cntr_ccm_by8_sse.asm index bd3fc780..3b2e16c8 100644 --- a/lib/sse/aes128_cntr_ccm_by8_sse.asm +++ b/lib/sse_t1/aes128_cntr_ccm_by8_sse.asm @@ -29,4 +29,4 @@ %ifndef AES_CNTR_CCM_128 %define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse %endif -%include "sse/aes128_cntr_by8_sse.asm" +%include "sse_t1/aes128_cntr_by8_sse.asm" diff --git a/lib/sse/aes128_ecb_by4_sse.asm b/lib/sse_t1/aes128_ecb_by4_sse.asm similarity index 100% rename from lib/sse/aes128_ecb_by4_sse.asm rename to lib/sse_t1/aes128_ecb_by4_sse.asm diff --git a/lib/sse/aes128_xcbc_mac_x4_sse.asm b/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm similarity index 98% rename from lib/sse/aes128_xcbc_mac_x4_sse.asm rename to lib/sse_t1/aes128_xcbc_mac_x4_sse.asm index 60df2b34..376327c3 100644 --- a/lib/sse/aes128_xcbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes128_xcbc_mac_x4_sse.asm @@ -46,4 +46,4 @@ %define ARG_KEYS _aesxcbcarg_keys %define ARG_IV _aesxcbcarg_ICV -%include "sse/aes128_cbc_enc_x4_sse.asm" +%include "sse_t1/aes128_cbc_enc_x4_sse.asm" diff --git a/lib/sse/aes192_cbc_dec_by4_sse.asm b/lib/sse_t1/aes192_cbc_dec_by4_sse.asm similarity index 100% rename from lib/sse/aes192_cbc_dec_by4_sse.asm rename to lib/sse_t1/aes192_cbc_dec_by4_sse.asm diff --git a/lib/sse/aes192_cbc_enc_x4_sse.asm b/lib/sse_t1/aes192_cbc_enc_x4_sse.asm similarity index 100% rename from lib/sse/aes192_cbc_enc_x4_sse.asm rename to lib/sse_t1/aes192_cbc_enc_x4_sse.asm diff --git a/lib/sse/aes192_cntr_by8_sse.asm b/lib/sse_t1/aes192_cntr_by8_sse.asm similarity index 100% rename from lib/sse/aes192_cntr_by8_sse.asm rename to lib/sse_t1/aes192_cntr_by8_sse.asm diff --git a/lib/sse/aes192_ecb_by4_sse.asm b/lib/sse_t1/aes192_ecb_by4_sse.asm similarity index 97% rename from lib/sse/aes192_ecb_by4_sse.asm rename to lib/sse_t1/aes192_ecb_by4_sse.asm index a5386e1f..fcad26e5 100644 --- a/lib/sse/aes192_ecb_by4_sse.asm +++ b/lib/sse_t1/aes192_ecb_by4_sse.asm @@ -30,4 +30,4 @@ %define AES_ECB_ENC_192 aes_ecb_enc_192_by4_sse %define AES_ECB_DEC_192 aes_ecb_dec_192_by4_sse -%include "sse/aes128_ecb_by4_sse.asm" +%include "sse_t1/aes128_ecb_by4_sse.asm" diff --git a/lib/sse/aes256_cbc_dec_by4_sse.asm b/lib/sse_t1/aes256_cbc_dec_by4_sse.asm similarity index 100% rename from lib/sse/aes256_cbc_dec_by4_sse.asm rename to lib/sse_t1/aes256_cbc_dec_by4_sse.asm diff --git a/lib/sse/aes256_cbc_enc_x4_sse.asm b/lib/sse_t1/aes256_cbc_enc_x4_sse.asm similarity index 100% rename from lib/sse/aes256_cbc_enc_x4_sse.asm rename to lib/sse_t1/aes256_cbc_enc_x4_sse.asm diff --git a/lib/sse/aes256_cbc_mac_x4_sse.asm b/lib/sse_t1/aes256_cbc_mac_x4_sse.asm similarity index 97% rename from lib/sse/aes256_cbc_mac_x4_sse.asm rename to lib/sse_t1/aes256_cbc_mac_x4_sse.asm index ccff0e57..998c8434 100644 --- a/lib/sse/aes256_cbc_mac_x4_sse.asm +++ b/lib/sse_t1/aes256_cbc_mac_x4_sse.asm @@ -28,4 +28,4 @@ ;;; Routine to compute CBC-MAC based on 256 bit CBC AES encryption code %define CBC_MAC -%include "sse/aes256_cbc_enc_x4_sse.asm" +%include "sse_t1/aes256_cbc_enc_x4_sse.asm" diff --git a/lib/sse/aes256_cntr_by8_sse.asm b/lib/sse_t1/aes256_cntr_by8_sse.asm similarity index 100% rename from lib/sse/aes256_cntr_by8_sse.asm rename to lib/sse_t1/aes256_cntr_by8_sse.asm diff --git a/lib/sse/aes256_cntr_ccm_by8_sse.asm b/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm similarity index 97% rename from lib/sse/aes256_cntr_ccm_by8_sse.asm rename to lib/sse_t1/aes256_cntr_ccm_by8_sse.asm index 946b4286..94c9c2d2 100644 --- a/lib/sse/aes256_cntr_ccm_by8_sse.asm +++ b/lib/sse_t1/aes256_cntr_ccm_by8_sse.asm @@ -29,4 +29,4 @@ %ifndef AES_CNTR_CCM_256 %define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse %endif -%include "sse/aes256_cntr_by8_sse.asm" +%include "sse_t1/aes256_cntr_by8_sse.asm" diff --git a/lib/sse/aes256_ecb_by4_sse.asm b/lib/sse_t1/aes256_ecb_by4_sse.asm similarity index 97% rename from lib/sse/aes256_ecb_by4_sse.asm rename to lib/sse_t1/aes256_ecb_by4_sse.asm index 1ef5c49f..703ede73 100644 --- a/lib/sse/aes256_ecb_by4_sse.asm +++ b/lib/sse_t1/aes256_ecb_by4_sse.asm @@ -30,4 +30,4 @@ %define AES_ECB_ENC_256 aes_ecb_enc_256_by4_sse %define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse -%include "sse/aes128_ecb_by4_sse.asm" +%include "sse_t1/aes128_ecb_by4_sse.asm" diff --git a/lib/sse/aes_cfb_sse.asm b/lib/sse_t1/aes_cfb_sse.asm similarity index 100% rename from lib/sse/aes_cfb_sse.asm rename to lib/sse_t1/aes_cfb_sse.asm diff --git a/lib/sse/chacha20_sse.asm b/lib/sse_t1/chacha20_sse.asm similarity index 100% rename from lib/sse/chacha20_sse.asm rename to lib/sse_t1/chacha20_sse.asm diff --git a/lib/sse/crc16_x25_sse.asm b/lib/sse_t1/crc16_x25_sse.asm similarity index 100% rename from lib/sse/crc16_x25_sse.asm rename to lib/sse_t1/crc16_x25_sse.asm diff --git a/lib/sse/crc32_by8_sse.asm b/lib/sse_t1/crc32_by8_sse.asm similarity index 100% rename from lib/sse/crc32_by8_sse.asm rename to lib/sse_t1/crc32_by8_sse.asm diff --git a/lib/sse/crc32_fp_sse.asm b/lib/sse_t1/crc32_fp_sse.asm similarity index 100% rename from lib/sse/crc32_fp_sse.asm rename to lib/sse_t1/crc32_fp_sse.asm diff --git a/lib/sse/crc32_iuup_sse.asm b/lib/sse_t1/crc32_iuup_sse.asm similarity index 100% rename from lib/sse/crc32_iuup_sse.asm rename to lib/sse_t1/crc32_iuup_sse.asm diff --git a/lib/sse/crc32_lte_sse.asm b/lib/sse_t1/crc32_lte_sse.asm similarity index 100% rename from lib/sse/crc32_lte_sse.asm rename to lib/sse_t1/crc32_lte_sse.asm diff --git a/lib/sse/crc32_refl_by8_sse.asm b/lib/sse_t1/crc32_refl_by8_sse.asm similarity index 100% rename from lib/sse/crc32_refl_by8_sse.asm rename to lib/sse_t1/crc32_refl_by8_sse.asm diff --git a/lib/sse/crc32_sctp_sse.asm b/lib/sse_t1/crc32_sctp_sse.asm similarity index 100% rename from lib/sse/crc32_sctp_sse.asm rename to lib/sse_t1/crc32_sctp_sse.asm diff --git a/lib/sse/crc32_wimax_sse.asm b/lib/sse_t1/crc32_wimax_sse.asm similarity index 100% rename from lib/sse/crc32_wimax_sse.asm rename to lib/sse_t1/crc32_wimax_sse.asm diff --git a/lib/sse/ethernet_fcs_sse.asm b/lib/sse_t1/ethernet_fcs_sse.asm similarity index 100% rename from lib/sse/ethernet_fcs_sse.asm rename to lib/sse_t1/ethernet_fcs_sse.asm diff --git a/lib/sse/gcm128_api_by8_sse.asm b/lib/sse_t1/gcm128_api_by8_sse.asm similarity index 98% rename from lib/sse/gcm128_api_by8_sse.asm rename to lib/sse_t1/gcm128_api_by8_sse.asm index a142f498..96e7a340 100644 --- a/lib/sse/gcm128_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_api_by8_sse.asm @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse/gcm_api_sse.inc" +%include "sse_t1/gcm_api_sse.inc" diff --git a/lib/sse/gcm128_gmac_api_by8_sse.asm b/lib/sse_t1/gcm128_gmac_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm128_gmac_api_by8_sse.asm rename to lib/sse_t1/gcm128_gmac_api_by8_sse.asm index c8e1c9b3..e222cf4e 100644 --- a/lib/sse/gcm128_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_gmac_api_by8_sse.asm @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse/gcm_gmac_api_sse.inc" +%include "sse_t1/gcm_gmac_api_sse.inc" diff --git a/lib/sse/gcm128_sgl_api_by8_sse.asm b/lib/sse_t1/gcm128_sgl_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm128_sgl_api_by8_sse.asm rename to lib/sse_t1/gcm128_sgl_api_by8_sse.asm index 2b985810..f6df66f7 100644 --- a/lib/sse/gcm128_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm128_sgl_api_by8_sse.asm @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "sse/gcm_sgl_api_sse.inc" +%include "sse_t1/gcm_sgl_api_sse.inc" diff --git a/lib/sse/gcm192_api_by8_sse.asm b/lib/sse_t1/gcm192_api_by8_sse.asm similarity index 98% rename from lib/sse/gcm192_api_by8_sse.asm rename to lib/sse_t1/gcm192_api_by8_sse.asm index 5396935e..13cf48d4 100644 --- a/lib/sse/gcm192_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_api_by8_sse.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse/gcm_api_sse.inc" +%include "sse_t1/gcm_api_sse.inc" diff --git a/lib/sse/gcm192_gmac_api_by8_sse.asm b/lib/sse_t1/gcm192_gmac_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm192_gmac_api_by8_sse.asm rename to lib/sse_t1/gcm192_gmac_api_by8_sse.asm index 57a218a9..5c9967e0 100644 --- a/lib/sse/gcm192_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_gmac_api_by8_sse.asm @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse/gcm_gmac_api_sse.inc" +%include "sse_t1/gcm_gmac_api_sse.inc" diff --git a/lib/sse/gcm192_sgl_api_by8_sse.asm b/lib/sse_t1/gcm192_sgl_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm192_sgl_api_by8_sse.asm rename to lib/sse_t1/gcm192_sgl_api_by8_sse.asm index 56959bfc..6b6a8357 100644 --- a/lib/sse/gcm192_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm192_sgl_api_by8_sse.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "sse/gcm_sgl_api_sse.inc" +%include "sse_t1/gcm_sgl_api_sse.inc" diff --git a/lib/sse/gcm256_api_by8_sse.asm b/lib/sse_t1/gcm256_api_by8_sse.asm similarity index 98% rename from lib/sse/gcm256_api_by8_sse.asm rename to lib/sse_t1/gcm256_api_by8_sse.asm index d106086b..fdffe34b 100644 --- a/lib/sse/gcm256_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_api_by8_sse.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse/gcm_api_sse.inc" +%include "sse_t1/gcm_api_sse.inc" diff --git a/lib/sse/gcm256_gmac_api_by8_sse.asm b/lib/sse_t1/gcm256_gmac_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm256_gmac_api_by8_sse.asm rename to lib/sse_t1/gcm256_gmac_api_by8_sse.asm index 5d151ec5..5f8dc21e 100644 --- a/lib/sse/gcm256_gmac_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_gmac_api_by8_sse.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse/gcm_gmac_api_sse.inc" +%include "sse_t1/gcm_gmac_api_sse.inc" diff --git a/lib/sse/gcm256_sgl_api_by8_sse.asm b/lib/sse_t1/gcm256_sgl_api_by8_sse.asm similarity index 97% rename from lib/sse/gcm256_sgl_api_by8_sse.asm rename to lib/sse_t1/gcm256_sgl_api_by8_sse.asm index f0a22d3e..304b3cb1 100644 --- a/lib/sse/gcm256_sgl_api_by8_sse.asm +++ b/lib/sse_t1/gcm256_sgl_api_by8_sse.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "sse/gcm_sgl_api_sse.inc" +%include "sse_t1/gcm_sgl_api_sse.inc" diff --git a/lib/sse/gcm_api_sse.inc b/lib/sse_t1/gcm_api_sse.inc similarity index 99% rename from lib/sse/gcm_api_sse.inc rename to lib/sse_t1/gcm_api_sse.inc index d5f6e980..774286eb 100644 --- a/lib/sse/gcm_api_sse.inc +++ b/lib/sse_t1/gcm_api_sse.inc @@ -29,7 +29,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; -%include "sse/gcm_sse.inc" +%include "sse_t1/gcm_sse.inc" %ifndef GCM_API_SSE_INC %define GCM_API_SSE_INC diff --git a/lib/sse/gcm_gmac_api_sse.inc b/lib/sse_t1/gcm_gmac_api_sse.inc similarity index 99% rename from lib/sse/gcm_gmac_api_sse.inc rename to lib/sse_t1/gcm_gmac_api_sse.inc index 43bc8637..da90159d 100644 --- a/lib/sse/gcm_gmac_api_sse.inc +++ b/lib/sse_t1/gcm_gmac_api_sse.inc @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "sse/gcm_sse.inc" +%include "sse_t1/gcm_sse.inc" %include "include/gcm_common.inc" %ifndef GCM_GMAC_API_SSE_INC diff --git a/lib/sse/gcm_sgl_api_sse.inc b/lib/sse_t1/gcm_sgl_api_sse.inc similarity index 99% rename from lib/sse/gcm_sgl_api_sse.inc rename to lib/sse_t1/gcm_sgl_api_sse.inc index f3bedb58..82f7b2b7 100644 --- a/lib/sse/gcm_sgl_api_sse.inc +++ b/lib/sse_t1/gcm_sgl_api_sse.inc @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "sse/gcm_sse.inc" +%include "sse_t1/gcm_sse.inc" %ifndef GCM_SGL_API_SSE_INC %define GCM_SGL_API_SSE_INC diff --git a/lib/sse/gcm_sse.inc b/lib/sse_t1/gcm_sse.inc similarity index 100% rename from lib/sse/gcm_sse.inc rename to lib/sse_t1/gcm_sse.inc diff --git a/lib/sse/kasumi_sse.c b/lib/sse_t1/kasumi_sse.c similarity index 100% rename from lib/sse/kasumi_sse.c rename to lib/sse_t1/kasumi_sse.c diff --git a/lib/sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes128_cbcs_1_9_flush_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_cbcs_1_9_flush_sse.asm rename to lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm diff --git a/lib/sse/mb_mgr_aes128_cbcs_1_9_submit_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_cbcs_1_9_submit_sse.asm rename to lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm diff --git a/lib/sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes128_xcbc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_xcbc_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes128_xcbc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm similarity index 100% rename from lib/sse/mb_mgr_aes128_xcbc_submit_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm diff --git a/lib/sse/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm index 6f0d7042..b0e89607 100644 --- a/lib/sse/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes192_cbc_enc_flush_x4_sse.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x4 %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_sse -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm index 28770f84..4ce3d952 100644 --- a/lib/sse/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes192_cbc_enc_submit_x4_sse.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x4 %define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_sse -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm index 777b796e..5c82dd21 100644 --- a/lib/sse/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cbc_enc_flush_x4_sse.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x4 %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_sse -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm index d8e0138f..3736b445 100644 --- a/lib/sse/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cbc_enc_submit_x4_sse.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x4 %define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_sse -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm index 113b1a8b..6dce9dfa 100644 --- a/lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_ccm_auth_submit_flush_x4_sse.asm @@ -32,4 +32,4 @@ %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes256_ccm_auth_sse %endif -%include "sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm index e54c3a53..c7c33d97 100644 --- a/lib/sse/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes256_cmac_submit_flush_x4_sse.asm @@ -30,4 +30,4 @@ %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes256_cmac_auth_sse %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes256_cmac_auth_sse -%include "sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_md5_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_md5_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_md5_flush_sse.asm diff --git a/lib/sse/mb_mgr_hmac_md5_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_md5_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_md5_submit_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha1_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha1_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha1_flush_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha1_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha1_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha1_submit_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha224_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm similarity index 96% rename from lib/sse/mb_mgr_hmac_sha224_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm index 69588c78..bda20459 100644 --- a/lib/sse/mb_mgr_hmac_sha224_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha224_flush_sse.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_224_sse %define SHA224 -%include "sse/mb_mgr_hmac_sha256_flush_sse.asm" +%include "sse_t1/mb_mgr_hmac_sha256_flush_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha224_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm similarity index 96% rename from lib/sse/mb_mgr_hmac_sha224_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm index 09f423ce..bddf843e 100644 --- a/lib/sse/mb_mgr_hmac_sha224_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha224_submit_sse.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_224_sse %define SHA224 -%include "sse/mb_mgr_hmac_sha256_submit_sse.asm" +%include "sse_t1/mb_mgr_hmac_sha256_submit_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha256_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha256_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha256_flush_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha256_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha256_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha256_submit_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha384_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm similarity index 97% rename from lib/sse/mb_mgr_hmac_sha384_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm index 435cf501..8e95926a 100644 --- a/lib/sse/mb_mgr_hmac_sha384_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha384_flush_sse.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_384_sse %define SHA_X_DIGEST_SIZE 384 -%include "sse/mb_mgr_hmac_sha512_flush_sse.asm" +%include "sse_t1/mb_mgr_hmac_sha512_flush_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha384_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm similarity index 96% rename from lib/sse/mb_mgr_hmac_sha384_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm index ec2085f1..37349571 100644 --- a/lib/sse/mb_mgr_hmac_sha384_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_hmac_sha384_submit_sse.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_384_sse %define SHA_X_DIGEST_SIZE 384 -%include "sse/mb_mgr_hmac_sha512_submit_sse.asm" +%include "sse_t1/mb_mgr_hmac_sha512_submit_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha512_flush_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha512_flush_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha512_flush_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha512_submit_sse.asm b/lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha512_submit_sse.asm rename to lib/sse_t1/mb_mgr_hmac_sha512_submit_sse.asm diff --git a/lib/sse/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c similarity index 100% rename from lib/sse/mb_mgr_sse.c rename to lib/sse_t1/mb_mgr_sse.c diff --git a/lib/sse/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm similarity index 100% rename from lib/sse/mb_mgr_zuc_submit_flush_sse.asm rename to lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm diff --git a/lib/sse/md5_x4x2_sse.asm b/lib/sse_t1/md5_x4x2_sse.asm similarity index 100% rename from lib/sse/md5_x4x2_sse.asm rename to lib/sse_t1/md5_x4x2_sse.asm diff --git a/lib/sse/memcpy_sse.asm b/lib/sse_t1/memcpy_sse.asm similarity index 100% rename from lib/sse/memcpy_sse.asm rename to lib/sse_t1/memcpy_sse.asm diff --git a/lib/sse/pon_by8_sse.asm b/lib/sse_t1/pon_by8_sse.asm similarity index 100% rename from lib/sse/pon_by8_sse.asm rename to lib/sse_t1/pon_by8_sse.asm diff --git a/lib/sse/sha1_one_block_sse.asm b/lib/sse_t1/sha1_one_block_sse.asm similarity index 100% rename from lib/sse/sha1_one_block_sse.asm rename to lib/sse_t1/sha1_one_block_sse.asm diff --git a/lib/sse/sha1_x4_sse.asm b/lib/sse_t1/sha1_x4_sse.asm similarity index 100% rename from lib/sse/sha1_x4_sse.asm rename to lib/sse_t1/sha1_x4_sse.asm diff --git a/lib/sse/sha224_one_block_sse.asm b/lib/sse_t1/sha224_one_block_sse.asm similarity index 97% rename from lib/sse/sha224_one_block_sse.asm rename to lib/sse_t1/sha224_one_block_sse.asm index b943ca50..cebbe79f 100644 --- a/lib/sse/sha224_one_block_sse.asm +++ b/lib/sse_t1/sha224_one_block_sse.asm @@ -30,4 +30,4 @@ %define FUNC sha224_block_sse -%include "sse/sha256_one_block_sse.asm" +%include "sse_t1/sha256_one_block_sse.asm" diff --git a/lib/sse/sha256_mult_sse.asm b/lib/sse_t1/sha256_mult_sse.asm similarity index 100% rename from lib/sse/sha256_mult_sse.asm rename to lib/sse_t1/sha256_mult_sse.asm diff --git a/lib/sse/sha256_one_block_sse.asm b/lib/sse_t1/sha256_one_block_sse.asm similarity index 100% rename from lib/sse/sha256_one_block_sse.asm rename to lib/sse_t1/sha256_one_block_sse.asm diff --git a/lib/sse/sha384_one_block_sse.asm b/lib/sse_t1/sha384_one_block_sse.asm similarity index 97% rename from lib/sse/sha384_one_block_sse.asm rename to lib/sse_t1/sha384_one_block_sse.asm index 8fd010d3..9db43fd9 100644 --- a/lib/sse/sha384_one_block_sse.asm +++ b/lib/sse_t1/sha384_one_block_sse.asm @@ -30,4 +30,4 @@ %define FUNC sha384_block_sse -%include "sse/sha512_one_block_sse.asm" +%include "sse_t1/sha512_one_block_sse.asm" diff --git a/lib/sse/sha512_one_block_sse.asm b/lib/sse_t1/sha512_one_block_sse.asm similarity index 100% rename from lib/sse/sha512_one_block_sse.asm rename to lib/sse_t1/sha512_one_block_sse.asm diff --git a/lib/sse/sha512_x2_sse.asm b/lib/sse_t1/sha512_x2_sse.asm similarity index 100% rename from lib/sse/sha512_x2_sse.asm rename to lib/sse_t1/sha512_x2_sse.asm diff --git a/lib/sse/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c similarity index 100% rename from lib/sse/sha_mb_sse.c rename to lib/sse_t1/sha_mb_sse.c diff --git a/lib/sse/sha_sse.c b/lib/sse_t1/sha_sse.c similarity index 100% rename from lib/sse/sha_sse.c rename to lib/sse_t1/sha_sse.c diff --git a/lib/sse/snow3g_sse.c b/lib/sse_t1/snow3g_sse.c similarity index 100% rename from lib/sse/snow3g_sse.c rename to lib/sse_t1/snow3g_sse.c diff --git a/lib/sse/snow3g_uia2_by4_sse.asm b/lib/sse_t1/snow3g_uia2_by4_sse.asm similarity index 100% rename from lib/sse/snow3g_uia2_by4_sse.asm rename to lib/sse_t1/snow3g_uia2_by4_sse.asm diff --git a/lib/sse/snow_v_sse.asm b/lib/sse_t1/snow_v_sse.asm similarity index 100% rename from lib/sse/snow_v_sse.asm rename to lib/sse_t1/snow_v_sse.asm diff --git a/lib/sse/zuc_top_sse.c b/lib/sse_t1/zuc_top_sse.c similarity index 100% rename from lib/sse/zuc_top_sse.c rename to lib/sse_t1/zuc_top_sse.c diff --git a/lib/sse/zuc_x4_sse.asm b/lib/sse_t1/zuc_x4_sse.asm similarity index 100% rename from lib/sse/zuc_x4_sse.asm rename to lib/sse_t1/zuc_x4_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha1_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha1_flush_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha1_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha1_submit_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha1_submit_ni_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha224_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm similarity index 96% rename from lib/sse/mb_mgr_hmac_sha224_flush_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm index 56d07d20..213f5361 100644 --- a/lib/sse/mb_mgr_hmac_sha224_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha224_flush_ni_sse.asm @@ -25,4 +25,4 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; %define SHA224 -%include "sse/mb_mgr_hmac_sha256_flush_ni_sse.asm" +%include "sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha224_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm similarity index 96% rename from lib/sse/mb_mgr_hmac_sha224_submit_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm index b430f79b..e95a0ccd 100644 --- a/lib/sse/mb_mgr_hmac_sha224_submit_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha224_submit_ni_sse.asm @@ -25,4 +25,4 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; %define SHA224 -%include "sse/mb_mgr_hmac_sha256_submit_ni_sse.asm" +%include "sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm" diff --git a/lib/sse/mb_mgr_hmac_sha256_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha256_flush_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm diff --git a/lib/sse/mb_mgr_hmac_sha256_submit_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm similarity index 100% rename from lib/sse/mb_mgr_hmac_sha256_submit_ni_sse.asm rename to lib/sse_t2/mb_mgr_hmac_sha256_submit_ni_sse.asm diff --git a/lib/sse/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm similarity index 100% rename from lib/sse/sha1_ni_x2_sse.asm rename to lib/sse_t2/sha1_ni_x2_sse.asm diff --git a/lib/sse/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm similarity index 100% rename from lib/sse/sha256_ni_x2_sse.asm rename to lib/sse_t2/sha256_ni_x2_sse.asm diff --git a/lib/sse/aes128_cbc_dec_by8_sse.asm b/lib/sse_t3/aes128_cbc_dec_by8_sse.asm similarity index 100% rename from lib/sse/aes128_cbc_dec_by8_sse.asm rename to lib/sse_t3/aes128_cbc_dec_by8_sse.asm diff --git a/lib/sse/aes128_cbc_enc_x8_sse.asm b/lib/sse_t3/aes128_cbc_enc_x8_sse.asm similarity index 100% rename from lib/sse/aes128_cbc_enc_x8_sse.asm rename to lib/sse_t3/aes128_cbc_enc_x8_sse.asm diff --git a/lib/sse/aes128_cbc_mac_x8_sse.asm b/lib/sse_t3/aes128_cbc_mac_x8_sse.asm similarity index 97% rename from lib/sse/aes128_cbc_mac_x8_sse.asm rename to lib/sse_t3/aes128_cbc_mac_x8_sse.asm index af18d03a..119a60d5 100644 --- a/lib/sse/aes128_cbc_mac_x8_sse.asm +++ b/lib/sse_t3/aes128_cbc_mac_x8_sse.asm @@ -28,4 +28,4 @@ ;;; Routine to compute CBC-MAC based on 128 bit CBC AES encryptionk code %define CBC_MAC -%include "sse/aes128_cbc_enc_x8_sse.asm" +%include "sse_t3/aes128_cbc_enc_x8_sse.asm" diff --git a/lib/sse/aes128_ecb_by8_sse.asm b/lib/sse_t3/aes128_ecb_by8_sse.asm similarity index 100% rename from lib/sse/aes128_ecb_by8_sse.asm rename to lib/sse_t3/aes128_ecb_by8_sse.asm diff --git a/lib/sse/aes192_cbc_dec_by8_sse.asm b/lib/sse_t3/aes192_cbc_dec_by8_sse.asm similarity index 100% rename from lib/sse/aes192_cbc_dec_by8_sse.asm rename to lib/sse_t3/aes192_cbc_dec_by8_sse.asm diff --git a/lib/sse/aes192_cbc_enc_x8_sse.asm b/lib/sse_t3/aes192_cbc_enc_x8_sse.asm similarity index 100% rename from lib/sse/aes192_cbc_enc_x8_sse.asm rename to lib/sse_t3/aes192_cbc_enc_x8_sse.asm diff --git a/lib/sse/aes192_ecb_by8_sse.asm b/lib/sse_t3/aes192_ecb_by8_sse.asm similarity index 97% rename from lib/sse/aes192_ecb_by8_sse.asm rename to lib/sse_t3/aes192_ecb_by8_sse.asm index ce2794a5..8bc905b4 100644 --- a/lib/sse/aes192_ecb_by8_sse.asm +++ b/lib/sse_t3/aes192_ecb_by8_sse.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 12 %include "include/os.asm" -%include "sse/aes128_ecb_by8_sse.asm" +%include "sse_t3/aes128_ecb_by8_sse.asm" diff --git a/lib/sse/aes256_cbc_dec_by8_sse.asm b/lib/sse_t3/aes256_cbc_dec_by8_sse.asm similarity index 100% rename from lib/sse/aes256_cbc_dec_by8_sse.asm rename to lib/sse_t3/aes256_cbc_dec_by8_sse.asm diff --git a/lib/sse/aes256_cbc_enc_x8_sse.asm b/lib/sse_t3/aes256_cbc_enc_x8_sse.asm similarity index 100% rename from lib/sse/aes256_cbc_enc_x8_sse.asm rename to lib/sse_t3/aes256_cbc_enc_x8_sse.asm diff --git a/lib/sse/aes256_cbc_mac_x8_sse.asm b/lib/sse_t3/aes256_cbc_mac_x8_sse.asm similarity index 97% rename from lib/sse/aes256_cbc_mac_x8_sse.asm rename to lib/sse_t3/aes256_cbc_mac_x8_sse.asm index c6f0b8d6..75d861a0 100644 --- a/lib/sse/aes256_cbc_mac_x8_sse.asm +++ b/lib/sse_t3/aes256_cbc_mac_x8_sse.asm @@ -28,4 +28,4 @@ ;;; Routine to compute CBC-MAC based on 256 bit CBC AES encryption code %define CBC_MAC -%include "sse/aes256_cbc_enc_x8_sse.asm" +%include "sse_t3/aes256_cbc_enc_x8_sse.asm" diff --git a/lib/sse/aes256_ecb_by8_sse.asm b/lib/sse_t3/aes256_ecb_by8_sse.asm similarity index 97% rename from lib/sse/aes256_ecb_by8_sse.asm rename to lib/sse_t3/aes256_ecb_by8_sse.asm index fb7434a3..d40d245d 100644 --- a/lib/sse/aes256_ecb_by8_sse.asm +++ b/lib/sse_t3/aes256_ecb_by8_sse.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 14 %include "include/os.asm" -%include "sse/aes128_ecb_by8_sse.asm" +%include "sse_t3/aes128_ecb_by8_sse.asm" diff --git a/lib/sse/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm index b9cac85a..0fa6e860 100644 --- a/lib/sse/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cbc_enc_flush_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_128_x8_sse %define FLUSH_JOB_AES_ENC flush_job_aes128_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm index 01882486..54244bde 100644 --- a/lib/sse/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cbc_enc_submit_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_128_x8_sse %define SUBMIT_JOB_AES_ENC submit_job_aes128_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm index a228dc85..4c74848d 100644 --- a/lib/sse/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_ccm_auth_submit_flush_x8_sse.asm @@ -30,4 +30,4 @@ %define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes128_ccm_auth_x8_sse %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes128_ccm_auth_x8_sse -%include "sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm index 62e74082..e3eebcac 100644 --- a/lib/sse/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes128_cmac_submit_flush_x8_sse.asm @@ -30,4 +30,4 @@ %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes128_cmac_auth_x8_sse %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes128_cmac_auth_x8_sse -%include "sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm index b1c3b4c1..657b6c6b 100644 --- a/lib/sse/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes192_cbc_enc_flush_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x8_sse %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm index 63d4294b..28ca090a 100644 --- a/lib/sse/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes192_cbc_enc_submit_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_192_x8_sse %define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm index 0232f61c..33d72ab7 100644 --- a/lib/sse/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cbc_enc_flush_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x8_sse %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm index 184d25be..fd2d2a76 100644 --- a/lib/sse/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cbc_enc_submit_x8_sse.asm @@ -29,4 +29,4 @@ %define AES_CBC_ENC_X4 aes_cbc_enc_256_x8_sse %define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_x8_sse -%include "sse/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm index ea8af6ff..c5407637 100644 --- a/lib/sse/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_ccm_auth_submit_flush_x8_sse.asm @@ -31,4 +31,4 @@ %define SUBMIT_JOB_AES_CCM_AUTH submit_job_aes256_ccm_auth_x8_sse %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes256_ccm_auth_x8_sse -%include "sse/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm b/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm similarity index 96% rename from lib/sse/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm rename to lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm index a37744eb..62aeb6e8 100644 --- a/lib/sse/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm +++ b/lib/sse_t3/mb_mgr_aes256_cmac_submit_flush_x8_sse.asm @@ -30,4 +30,4 @@ %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes256_cmac_auth_x8_sse %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes256_cmac_auth_x8_sse -%include "sse/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" +%include "sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm" diff --git a/lib/sse/mb_mgr_zuc_submit_flush_gfni_sse.asm b/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm similarity index 97% rename from lib/sse/mb_mgr_zuc_submit_flush_gfni_sse.asm rename to lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm index d2473f22..77d26c1d 100644 --- a/lib/sse/mb_mgr_zuc_submit_flush_gfni_sse.asm +++ b/lib/sse_t3/mb_mgr_zuc_submit_flush_gfni_sse.asm @@ -37,4 +37,4 @@ %define ZUC128_INIT_4 asm_ZucInitialization_4_gfni_sse %define ZUC256_INIT_4 asm_Zuc256Initialization_4_gfni_sse %define ZUC_CIPHER_4 asm_ZucCipher_4_gfni_sse -%include "sse/mb_mgr_zuc_submit_flush_sse.asm" +%include "sse_t1/mb_mgr_zuc_submit_flush_sse.asm" diff --git a/lib/sse/zuc_x4_gfni_sse.asm b/lib/sse_t3/zuc_x4_gfni_sse.asm similarity index 98% rename from lib/sse/zuc_x4_gfni_sse.asm rename to lib/sse_t3/zuc_x4_gfni_sse.asm index 8d99a8a4..6f269bfe 100644 --- a/lib/sse/zuc_x4_gfni_sse.asm +++ b/lib/sse_t3/zuc_x4_gfni_sse.asm @@ -34,4 +34,4 @@ %define ZUC_KEYGEN4B_4 asm_ZucGenKeystream4B_4_gfni_sse %define ZUC_EIA3ROUND16B asm_Eia3Round16B_gfni_sse %define ZUC_EIA3REMAINDER asm_Eia3Remainder_gfni_sse -%include "sse/zuc_x4_sse.asm" +%include "sse_t1/zuc_x4_sse.asm" diff --git a/lib/win_x64.mak b/lib/win_x64.mak index c780763c..07be1c89 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -604,11 +604,25 @@ $(DEPALL): $(all_objs) {x86_64\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< -{sse\}.c{$(OBJ_DIR)}.obj: +{sse_t1\}.c{$(OBJ_DIR)}.obj: $(CC) /Fo$@ /c $(CFLAGS) $< $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep -{sse\}.asm{$(OBJ_DIR)}.obj: +{sse_t1\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +{sse_t2\}.c{$(OBJ_DIR)}.obj: + $(CC) /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{sse_t2\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +{sse_t3\}.c{$(OBJ_DIR)}.obj: + $(CC) /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{sse_t3\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< {avx\}.c{$(OBJ_DIR)}.obj: -- GitLab From 77803e1a999c22ed2399af35fc937b05651522c5 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 08:57:46 +0000 Subject: [PATCH 154/369] lib: [avx2] split AVX2 implementations into two types --- lib/Makefile | 18 ++++++++++++++++-- lib/{avx2 => avx2_t1}/aes128_gcm_by8_avx2.asm | 2 +- lib/{avx2 => avx2_t1}/aes192_gcm_by8_avx2.asm | 2 +- lib/{avx2 => avx2_t1}/aes256_gcm_by8_avx2.asm | 2 +- lib/{avx2 => avx2_t1}/chacha20_avx2.asm | 0 lib/{avx2 => avx2_t1}/gcm_avx_gen4.asm | 0 lib/{avx2 => avx2_t1}/mb_mgr_avx2.c | 0 .../mb_mgr_hmac_md5_flush_avx2.asm | 0 .../mb_mgr_hmac_md5_submit_avx2.asm | 0 .../mb_mgr_hmac_sha1_flush_avx2.asm | 0 .../mb_mgr_hmac_sha1_submit_avx2.asm | 0 .../mb_mgr_hmac_sha224_flush_avx2.asm | 2 +- .../mb_mgr_hmac_sha224_submit_avx2.asm | 2 +- .../mb_mgr_hmac_sha256_flush_avx2.asm | 0 .../mb_mgr_hmac_sha256_submit_avx2.asm | 0 .../mb_mgr_hmac_sha384_flush_avx2.asm | 2 +- .../mb_mgr_hmac_sha384_submit_avx2.asm | 2 +- .../mb_mgr_hmac_sha512_flush_avx2.asm | 0 .../mb_mgr_hmac_sha512_submit_avx2.asm | 0 .../mb_mgr_zuc_submit_flush_avx2.asm | 0 lib/{avx2 => avx2_t1}/md5_x8x2_avx2.asm | 0 lib/{avx2 => avx2_t1}/sha1_x8_avx2.asm | 0 lib/{avx2 => avx2_t1}/sha256_oct_avx2.asm | 0 lib/{avx2 => avx2_t1}/sha512_x4_avx2.asm | 0 lib/{avx2 => avx2_t1}/sha_mb_avx2.c | 0 lib/{avx2 => avx2_t1}/snow3g_avx2.c | 0 lib/{avx2 => avx2_t1}/zuc_top_avx2.c | 0 lib/{avx2 => avx2_t1}/zuc_x8_avx2.asm | 0 lib/{avx2 => avx2_t2}/aes128_ecb_vaes_avx2.asm | 0 lib/{avx2 => avx2_t2}/aes192_ecb_vaes_avx2.asm | 2 +- lib/{avx2 => avx2_t2}/aes256_ecb_vaes_avx2.asm | 2 +- lib/win_x64.mak | 11 +++++++++-- 32 files changed, 34 insertions(+), 13 deletions(-) rename lib/{avx2 => avx2_t1}/aes128_gcm_by8_avx2.asm (97%) rename lib/{avx2 => avx2_t1}/aes192_gcm_by8_avx2.asm (97%) rename lib/{avx2 => avx2_t1}/aes256_gcm_by8_avx2.asm (97%) rename lib/{avx2 => avx2_t1}/chacha20_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/gcm_avx_gen4.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_avx2.c (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_md5_flush_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_md5_submit_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha1_flush_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha1_submit_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha224_flush_avx2.asm (96%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha224_submit_avx2.asm (96%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha256_flush_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha256_submit_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha384_flush_avx2.asm (96%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha384_submit_avx2.asm (96%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha512_flush_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_hmac_sha512_submit_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/mb_mgr_zuc_submit_flush_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/md5_x8x2_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/sha1_x8_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/sha256_oct_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/sha512_x4_avx2.asm (100%) rename lib/{avx2 => avx2_t1}/sha_mb_avx2.c (100%) rename lib/{avx2 => avx2_t1}/snow3g_avx2.c (100%) rename lib/{avx2 => avx2_t1}/zuc_top_avx2.c (100%) mode change 100755 => 100644 rename lib/{avx2 => avx2_t1}/zuc_x8_avx2.asm (100%) mode change 100755 => 100644 rename lib/{avx2 => avx2_t2}/aes128_ecb_vaes_avx2.asm (100%) rename lib/{avx2 => avx2_t2}/aes192_ecb_vaes_avx2.asm (97%) rename lib/{avx2 => avx2_t2}/aes256_ecb_vaes_avx2.asm (97%) diff --git a/lib/Makefile b/lib/Makefile index 8ce31937..9124c098 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -851,10 +851,24 @@ ifeq ($(CC_HAS_CET),1) mv $@.tmp $@ endif -$(OBJ_DIR)/%.o:avx2/%.c +$(OBJ_DIR)/%.o:avx2_t1/%.c $(CC) -MMD $(OPT_AVX2) -c $(CFLAGS) $< -o $@ -$(OBJ_DIR)/%.o:avx2/%.asm +$(OBJ_DIR)/%.o:avx2_t1/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +$(OBJ_DIR)/%.o:avx2_t2/%.c + $(CC) -MMD $(OPT_AVX2) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:avx2_t2/%.asm ifeq ($(USE_YASM),y) $(YASM) $(YASM_FLAGS) $< -o $@ else diff --git a/lib/avx2/aes128_gcm_by8_avx2.asm b/lib/avx2_t1/aes128_gcm_by8_avx2.asm similarity index 97% rename from lib/avx2/aes128_gcm_by8_avx2.asm rename to lib/avx2_t1/aes128_gcm_by8_avx2.asm index 0d2633e2..78945318 100644 --- a/lib/avx2/aes128_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes128_gcm_by8_avx2.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx2/gcm_avx_gen4.asm" +%include "avx2_t1/gcm_avx_gen4.asm" diff --git a/lib/avx2/aes192_gcm_by8_avx2.asm b/lib/avx2_t1/aes192_gcm_by8_avx2.asm similarity index 97% rename from lib/avx2/aes192_gcm_by8_avx2.asm rename to lib/avx2_t1/aes192_gcm_by8_avx2.asm index cd9a60b6..6f3255a3 100644 --- a/lib/avx2/aes192_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes192_gcm_by8_avx2.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx2/gcm_avx_gen4.asm" +%include "avx2_t1/gcm_avx_gen4.asm" diff --git a/lib/avx2/aes256_gcm_by8_avx2.asm b/lib/avx2_t1/aes256_gcm_by8_avx2.asm similarity index 97% rename from lib/avx2/aes256_gcm_by8_avx2.asm rename to lib/avx2_t1/aes256_gcm_by8_avx2.asm index 71e90bf4..4e3bf23e 100644 --- a/lib/avx2/aes256_gcm_by8_avx2.asm +++ b/lib/avx2_t1/aes256_gcm_by8_avx2.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx2/gcm_avx_gen4.asm" +%include "avx2_t1/gcm_avx_gen4.asm" diff --git a/lib/avx2/chacha20_avx2.asm b/lib/avx2_t1/chacha20_avx2.asm similarity index 100% rename from lib/avx2/chacha20_avx2.asm rename to lib/avx2_t1/chacha20_avx2.asm diff --git a/lib/avx2/gcm_avx_gen4.asm b/lib/avx2_t1/gcm_avx_gen4.asm similarity index 100% rename from lib/avx2/gcm_avx_gen4.asm rename to lib/avx2_t1/gcm_avx_gen4.asm diff --git a/lib/avx2/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c similarity index 100% rename from lib/avx2/mb_mgr_avx2.c rename to lib/avx2_t1/mb_mgr_avx2.c diff --git a/lib/avx2/mb_mgr_hmac_md5_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_md5_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_md5_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_md5_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha1_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha1_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha1_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha1_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha224_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm similarity index 96% rename from lib/avx2/mb_mgr_hmac_sha224_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm index cc360c76..bae687b7 100644 --- a/lib/avx2/mb_mgr_hmac_sha224_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha224_flush_avx2.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_224_avx2 %define SHA224 -%include "avx2/mb_mgr_hmac_sha256_flush_avx2.asm" +%include "avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm" diff --git a/lib/avx2/mb_mgr_hmac_sha224_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm similarity index 96% rename from lib/avx2/mb_mgr_hmac_sha224_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm index 0c32dfc3..0b6c7fde 100644 --- a/lib/avx2/mb_mgr_hmac_sha224_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha224_submit_avx2.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_224_avx2 %define SHA224 -%include "avx2/mb_mgr_hmac_sha256_submit_avx2.asm" +%include "avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm" diff --git a/lib/avx2/mb_mgr_hmac_sha256_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha256_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha256_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha256_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha384_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm similarity index 96% rename from lib/avx2/mb_mgr_hmac_sha384_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm index 10787b40..9f4c13dd 100644 --- a/lib/avx2/mb_mgr_hmac_sha384_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha384_flush_avx2.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_384_avx2 %define SHA_X_DIGEST_SIZE 384 -%include "avx2/mb_mgr_hmac_sha512_flush_avx2.asm" +%include "avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm" diff --git a/lib/avx2/mb_mgr_hmac_sha384_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm similarity index 96% rename from lib/avx2/mb_mgr_hmac_sha384_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm index 8aec38c5..a7687b52 100644 --- a/lib/avx2/mb_mgr_hmac_sha384_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha384_submit_avx2.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_384_avx2 %define SHA_X_DIGEST_SIZE 384 -%include "avx2/mb_mgr_hmac_sha512_submit_avx2.asm" +%include "avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm" diff --git a/lib/avx2/mb_mgr_hmac_sha512_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha512_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm diff --git a/lib/avx2/mb_mgr_hmac_sha512_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_hmac_sha512_submit_avx2.asm rename to lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm diff --git a/lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm similarity index 100% rename from lib/avx2/mb_mgr_zuc_submit_flush_avx2.asm rename to lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm diff --git a/lib/avx2/md5_x8x2_avx2.asm b/lib/avx2_t1/md5_x8x2_avx2.asm similarity index 100% rename from lib/avx2/md5_x8x2_avx2.asm rename to lib/avx2_t1/md5_x8x2_avx2.asm diff --git a/lib/avx2/sha1_x8_avx2.asm b/lib/avx2_t1/sha1_x8_avx2.asm similarity index 100% rename from lib/avx2/sha1_x8_avx2.asm rename to lib/avx2_t1/sha1_x8_avx2.asm diff --git a/lib/avx2/sha256_oct_avx2.asm b/lib/avx2_t1/sha256_oct_avx2.asm similarity index 100% rename from lib/avx2/sha256_oct_avx2.asm rename to lib/avx2_t1/sha256_oct_avx2.asm diff --git a/lib/avx2/sha512_x4_avx2.asm b/lib/avx2_t1/sha512_x4_avx2.asm similarity index 100% rename from lib/avx2/sha512_x4_avx2.asm rename to lib/avx2_t1/sha512_x4_avx2.asm diff --git a/lib/avx2/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c similarity index 100% rename from lib/avx2/sha_mb_avx2.c rename to lib/avx2_t1/sha_mb_avx2.c diff --git a/lib/avx2/snow3g_avx2.c b/lib/avx2_t1/snow3g_avx2.c similarity index 100% rename from lib/avx2/snow3g_avx2.c rename to lib/avx2_t1/snow3g_avx2.c diff --git a/lib/avx2/zuc_top_avx2.c b/lib/avx2_t1/zuc_top_avx2.c old mode 100755 new mode 100644 similarity index 100% rename from lib/avx2/zuc_top_avx2.c rename to lib/avx2_t1/zuc_top_avx2.c diff --git a/lib/avx2/zuc_x8_avx2.asm b/lib/avx2_t1/zuc_x8_avx2.asm old mode 100755 new mode 100644 similarity index 100% rename from lib/avx2/zuc_x8_avx2.asm rename to lib/avx2_t1/zuc_x8_avx2.asm diff --git a/lib/avx2/aes128_ecb_vaes_avx2.asm b/lib/avx2_t2/aes128_ecb_vaes_avx2.asm similarity index 100% rename from lib/avx2/aes128_ecb_vaes_avx2.asm rename to lib/avx2_t2/aes128_ecb_vaes_avx2.asm diff --git a/lib/avx2/aes192_ecb_vaes_avx2.asm b/lib/avx2_t2/aes192_ecb_vaes_avx2.asm similarity index 97% rename from lib/avx2/aes192_ecb_vaes_avx2.asm rename to lib/avx2_t2/aes192_ecb_vaes_avx2.asm index 9f8dbdb5..d400c3b3 100644 --- a/lib/avx2/aes192_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes192_ecb_vaes_avx2.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 12 %include "include/os.asm" -%include "avx2/aes128_ecb_vaes_avx2.asm" +%include "avx2_t2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/avx2/aes256_ecb_vaes_avx2.asm b/lib/avx2_t2/aes256_ecb_vaes_avx2.asm similarity index 97% rename from lib/avx2/aes256_ecb_vaes_avx2.asm rename to lib/avx2_t2/aes256_ecb_vaes_avx2.asm index 448293c3..8c8a7308 100644 --- a/lib/avx2/aes256_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes256_ecb_vaes_avx2.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 14 %include "include/os.asm" -%include "avx2/aes128_ecb_vaes_avx2.asm" +%include "avx2_t2/aes128_ecb_vaes_avx2.asm" diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 07be1c89..62227390 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -632,11 +632,18 @@ $(DEPALL): $(all_objs) {avx\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< -{avx2\}.c{$(OBJ_DIR)}.obj: +{avx2_t1\}.c{$(OBJ_DIR)}.obj: $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep -{avx2\}.asm{$(OBJ_DIR)}.obj: +{avx2_t1\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +{avx2_t2\}.c{$(OBJ_DIR)}.obj: + $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{avx2_t2\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< {avx512\}.c{$(OBJ_DIR)}.obj: -- GitLab From 6b0939707fb7f9c586684e5f5ae72898b594c263 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 10:32:38 +0000 Subject: [PATCH 155/369] lib: [avx512] split AVX512 implementations into two types --- lib/Makefile | 18 ++++++++++++++++-- .../aes128_gcm_by8_avx512.asm | 2 +- .../aes192_gcm_by8_avx512.asm | 2 +- .../aes256_gcm_by8_avx512.asm | 2 +- lib/{avx512 => avx512_t1}/chacha20_avx512.asm | 0 lib/{avx512 => avx512_t1}/des_x16_avx512.asm | 0 lib/{avx512 => avx512_t1}/gcm_avx512.asm | 0 lib/{avx512 => avx512_t1}/mb_mgr_avx512.c | 0 .../mb_mgr_des_avx512.asm | 0 .../mb_mgr_hmac_sha1_flush_avx512.asm | 0 .../mb_mgr_hmac_sha1_submit_avx512.asm | 0 .../mb_mgr_hmac_sha224_flush_avx512.asm | 2 +- .../mb_mgr_hmac_sha224_submit_avx512.asm | 2 +- .../mb_mgr_hmac_sha256_flush_avx512.asm | 0 .../mb_mgr_hmac_sha256_submit_avx512.asm | 0 .../mb_mgr_hmac_sha384_flush_avx512.asm | 2 +- .../mb_mgr_hmac_sha384_submit_avx512.asm | 2 +- .../mb_mgr_hmac_sha512_flush_avx512.asm | 0 .../mb_mgr_hmac_sha512_submit_avx512.asm | 0 .../mb_mgr_zuc_submit_flush_avx512.asm | 0 lib/{avx512 => avx512_t1}/poly_avx512.asm | 0 lib/{avx512 => avx512_t1}/sha1_x16_avx512.asm | 0 .../sha256_x16_avx512.asm | 0 lib/{avx512 => avx512_t1}/sha512_x8_avx512.asm | 0 lib/{avx512 => avx512_t1}/sha_mb_avx512.c | 0 lib/{avx512 => avx512_t1}/snow3g_avx512.c | 0 lib/{avx512 => avx512_t1}/zuc_top_avx512.c | 0 lib/{avx512 => avx512_t1}/zuc_x16_avx512.asm | 0 .../aes128_gcm_by48_api_vaes_avx512.asm | 2 +- .../aes128_gcm_by48_sgl_api_vaes_avx512.asm | 2 +- .../aes128_gmac_by48_api_vaes_avx512.asm | 2 +- .../aes192_gcm_by48_api_vaes_avx512.asm | 2 +- .../aes192_gcm_by48_sgl_api_vaes_avx512.asm | 2 +- .../aes192_gmac_by48_api_vaes_avx512.asm | 2 +- .../aes256_gcm_by48_api_vaes_avx512.asm | 2 +- .../aes256_gcm_by48_sgl_api_vaes_avx512.asm | 2 +- .../aes256_gmac_by48_api_vaes_avx512.asm | 2 +- .../aes_cbc_dec_by16_vaes_avx512.asm | 0 .../aes_cbc_enc_vaes_avx512.asm | 0 .../aes_cbcs_dec_by16_vaes_avx512.asm | 2 +- .../aes_cbcs_enc_vaes_avx512.asm | 0 .../aes_cntr_api_by16_vaes_avx512.asm | 2 +- .../aes_cntr_bit_api_by16_vaes_avx512.asm | 2 +- .../aes_cntr_by16_vaes_avx512.inc | 0 .../aes_cntr_ccm_api_by16_vaes_avx512.asm | 2 +- .../aes_cntr_pon_api_by16_vaes_avx512.asm | 2 +- .../aes_docsis_dec_avx512.asm | 0 .../aes_docsis_dec_vaes_avx512.asm | 0 .../aes_docsis_enc_avx512.asm | 0 .../aes_docsis_enc_vaes_avx512.asm | 0 .../aes_ecb_vaes_avx512.asm | 0 lib/{avx512 => avx512_t2}/crc16_x25_avx512.asm | 0 .../crc32_by16_vclmul_avx512.asm | 0 lib/{avx512 => avx512_t2}/crc32_fp_avx512.asm | 0 .../crc32_iuup_avx512.asm | 0 lib/{avx512 => avx512_t2}/crc32_lte_avx512.asm | 0 .../crc32_refl_by16_vclmul_avx512.asm | 0 .../crc32_sctp_avx512.asm | 0 .../crc32_wimax_avx512.asm | 0 .../ethernet_fcs_avx512.asm | 0 .../gcm_api_vaes_avx512.inc | 2 +- .../gcm_gmac_api_vaes_avx512.inc | 2 +- .../gcm_sgl_api_vaes_avx512.inc | 2 +- lib/{avx512 => avx512_t2}/gcm_vaes_avx512.inc | 0 .../mb_mgr_aes128_cbc_enc_flush_avx512.asm | 0 .../mb_mgr_aes128_cbc_enc_submit_avx512.asm | 0 .../mb_mgr_aes128_cbcs_1_9_flush_avx512.asm | 2 +- .../mb_mgr_aes128_cbcs_1_9_submit_avx512.asm | 2 +- ...8_ccm_auth_submit_flush_x16_vaes_avx512.asm | 0 ...es128_cmac_submit_flush_x16_vaes_avx512.asm | 0 ...es128_xcbc_submit_flush_x16_vaes_avx512.asm | 0 .../mb_mgr_aes192_cbc_enc_flush_avx512.asm | 2 +- .../mb_mgr_aes192_cbc_enc_submit_avx512.asm | 2 +- .../mb_mgr_aes256_cbc_enc_flush_avx512.asm | 2 +- .../mb_mgr_aes256_cbc_enc_submit_avx512.asm | 2 +- ...6_ccm_auth_submit_flush_x16_vaes_avx512.asm | 2 +- ...es256_cmac_submit_flush_x16_vaes_avx512.asm | 2 +- ...gr_snow3g_uea2_submit_flush_vaes_avx512.asm | 2 +- ...gr_snow3g_uia2_submit_flush_vaes_avx512.asm | 2 +- .../mb_mgr_zuc_submit_flush_gfni_avx512.asm | 2 +- lib/{avx512 => avx512_t2}/poly_fma_avx512.asm | 0 lib/{avx512 => avx512_t2}/pon_vaes_avx512.asm | 0 .../snow3g_uea2_by16_vaes_avx512.asm | 0 .../snow3g_uia2_by32_vaes_avx512.asm | 0 .../zuc_x16_vaes_avx512.asm | 2 +- lib/win_x64.mak | 11 +++++++++-- 86 files changed, 61 insertions(+), 40 deletions(-) rename lib/{avx512 => avx512_t1}/aes128_gcm_by8_avx512.asm (97%) rename lib/{avx512 => avx512_t1}/aes192_gcm_by8_avx512.asm (97%) rename lib/{avx512 => avx512_t1}/aes256_gcm_by8_avx512.asm (97%) rename lib/{avx512 => avx512_t1}/chacha20_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/des_x16_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/gcm_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_avx512.c (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_des_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha1_flush_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha1_submit_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha224_flush_avx512.asm (96%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha224_submit_avx512.asm (96%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha256_flush_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha256_submit_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha384_flush_avx512.asm (96%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha384_submit_avx512.asm (96%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha512_flush_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_hmac_sha512_submit_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/mb_mgr_zuc_submit_flush_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/poly_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/sha1_x16_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/sha256_x16_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/sha512_x8_avx512.asm (100%) rename lib/{avx512 => avx512_t1}/sha_mb_avx512.c (100%) rename lib/{avx512 => avx512_t1}/snow3g_avx512.c (100%) rename lib/{avx512 => avx512_t1}/zuc_top_avx512.c (100%) mode change 100755 => 100644 rename lib/{avx512 => avx512_t1}/zuc_x16_avx512.asm (100%) mode change 100755 => 100644 rename lib/{avx512 => avx512_t2}/aes128_gcm_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes128_gcm_by48_sgl_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes128_gmac_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes192_gcm_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes192_gcm_by48_sgl_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes192_gmac_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes256_gcm_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes256_gcm_by48_sgl_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes256_gmac_by48_api_vaes_avx512.asm (97%) rename lib/{avx512 => avx512_t2}/aes_cbc_dec_by16_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_cbc_enc_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_cbcs_dec_by16_vaes_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/aes_cbcs_enc_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_cntr_api_by16_vaes_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/aes_cntr_bit_api_by16_vaes_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/aes_cntr_by16_vaes_avx512.inc (100%) rename lib/{avx512 => avx512_t2}/aes_cntr_ccm_api_by16_vaes_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/aes_cntr_pon_api_by16_vaes_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/aes_docsis_dec_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_docsis_dec_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_docsis_enc_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_docsis_enc_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/aes_ecb_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc16_x25_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_by16_vclmul_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_fp_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_iuup_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_lte_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_refl_by16_vclmul_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_sctp_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/crc32_wimax_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/ethernet_fcs_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/gcm_api_vaes_avx512.inc (99%) rename lib/{avx512 => avx512_t2}/gcm_gmac_api_vaes_avx512.inc (99%) rename lib/{avx512 => avx512_t2}/gcm_sgl_api_vaes_avx512.inc (99%) rename lib/{avx512 => avx512_t2}/gcm_vaes_avx512.inc (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_cbc_enc_flush_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_cbc_enc_submit_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm (99%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm (99%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes192_cbc_enc_flush_avx512.asm (96%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes192_cbc_enc_submit_avx512.asm (96%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes256_cbc_enc_flush_avx512.asm (96%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes256_cbc_enc_submit_avx512.asm (96%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm (95%) rename lib/{avx512 => avx512_t2}/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm (96%) rename lib/{avx512 => avx512_t2}/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm (99%) rename lib/{avx512 => avx512_t2}/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm (99%) rename lib/{avx512 => avx512_t2}/mb_mgr_zuc_submit_flush_gfni_avx512.asm (98%) rename lib/{avx512 => avx512_t2}/poly_fma_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/pon_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/snow3g_uea2_by16_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/snow3g_uia2_by32_vaes_avx512.asm (100%) rename lib/{avx512 => avx512_t2}/zuc_x16_vaes_avx512.asm (98%) diff --git a/lib/Makefile b/lib/Makefile index 9124c098..f9cddad8 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -879,10 +879,24 @@ ifeq ($(CC_HAS_CET),1) mv $@.tmp $@ endif -$(OBJ_DIR)/%.o:avx512/%.c +$(OBJ_DIR)/%.o:avx512_t1/%.c $(CC) -MMD $(OPT_AVX512) -c $(CFLAGS) $< -o $@ -$(OBJ_DIR)/%.o:avx512/%.asm +$(OBJ_DIR)/%.o:avx512_t1/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +$(OBJ_DIR)/%.o:avx512_t2/%.c + $(CC) -MMD $(OPT_AVX512) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:avx512_t2/%.asm ifeq ($(USE_YASM),y) $(YASM) $(YASM_FLAGS) $< -o $@ else diff --git a/lib/avx512/aes128_gcm_by8_avx512.asm b/lib/avx512_t1/aes128_gcm_by8_avx512.asm similarity index 97% rename from lib/avx512/aes128_gcm_by8_avx512.asm rename to lib/avx512_t1/aes128_gcm_by8_avx512.asm index 79d86495..4415ca93 100644 --- a/lib/avx512/aes128_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes128_gcm_by8_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512/gcm_avx512.asm" +%include "avx512_t1/gcm_avx512.asm" diff --git a/lib/avx512/aes192_gcm_by8_avx512.asm b/lib/avx512_t1/aes192_gcm_by8_avx512.asm similarity index 97% rename from lib/avx512/aes192_gcm_by8_avx512.asm rename to lib/avx512_t1/aes192_gcm_by8_avx512.asm index fee431cc..96ae8046 100644 --- a/lib/avx512/aes192_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes192_gcm_by8_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512/gcm_avx512.asm" +%include "avx512_t1/gcm_avx512.asm" diff --git a/lib/avx512/aes256_gcm_by8_avx512.asm b/lib/avx512_t1/aes256_gcm_by8_avx512.asm similarity index 97% rename from lib/avx512/aes256_gcm_by8_avx512.asm rename to lib/avx512_t1/aes256_gcm_by8_avx512.asm index d25dbd35..5083081a 100644 --- a/lib/avx512/aes256_gcm_by8_avx512.asm +++ b/lib/avx512_t1/aes256_gcm_by8_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512/gcm_avx512.asm" +%include "avx512_t1/gcm_avx512.asm" diff --git a/lib/avx512/chacha20_avx512.asm b/lib/avx512_t1/chacha20_avx512.asm similarity index 100% rename from lib/avx512/chacha20_avx512.asm rename to lib/avx512_t1/chacha20_avx512.asm diff --git a/lib/avx512/des_x16_avx512.asm b/lib/avx512_t1/des_x16_avx512.asm similarity index 100% rename from lib/avx512/des_x16_avx512.asm rename to lib/avx512_t1/des_x16_avx512.asm diff --git a/lib/avx512/gcm_avx512.asm b/lib/avx512_t1/gcm_avx512.asm similarity index 100% rename from lib/avx512/gcm_avx512.asm rename to lib/avx512_t1/gcm_avx512.asm diff --git a/lib/avx512/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c similarity index 100% rename from lib/avx512/mb_mgr_avx512.c rename to lib/avx512_t1/mb_mgr_avx512.c diff --git a/lib/avx512/mb_mgr_des_avx512.asm b/lib/avx512_t1/mb_mgr_des_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_des_avx512.asm rename to lib/avx512_t1/mb_mgr_des_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha1_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha1_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha1_submit_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha224_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_hmac_sha224_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm index 582d83cc..b7467c31 100644 --- a/lib/avx512/mb_mgr_hmac_sha224_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha224_flush_avx512.asm @@ -25,4 +25,4 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; %define SHA224 -%include "avx512/mb_mgr_hmac_sha256_flush_avx512.asm" +%include "avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm" diff --git a/lib/avx512/mb_mgr_hmac_sha224_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_hmac_sha224_submit_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm index 775bdbb4..d8828b45 100644 --- a/lib/avx512/mb_mgr_hmac_sha224_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha224_submit_avx512.asm @@ -25,4 +25,4 @@ ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;; %define SHA224 -%include "avx512/mb_mgr_hmac_sha256_submit_avx512.asm" +%include "avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm" diff --git a/lib/avx512/mb_mgr_hmac_sha256_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha256_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha256_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha256_submit_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha384_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_hmac_sha384_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm index edb49f1d..70037f6d 100644 --- a/lib/avx512/mb_mgr_hmac_sha384_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha384_flush_avx512.asm @@ -26,4 +26,4 @@ ;; %define SHA384 -%include "avx512/mb_mgr_hmac_sha512_flush_avx512.asm" +%include "avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm" diff --git a/lib/avx512/mb_mgr_hmac_sha384_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_hmac_sha384_submit_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm index f2e0f240..2fc8063c 100644 --- a/lib/avx512/mb_mgr_hmac_sha384_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha384_submit_avx512.asm @@ -26,4 +26,4 @@ ;; %define SHA384 -%include "avx512/mb_mgr_hmac_sha512_submit_avx512.asm" +%include "avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm" diff --git a/lib/avx512/mb_mgr_hmac_sha512_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha512_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm diff --git a/lib/avx512/mb_mgr_hmac_sha512_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_hmac_sha512_submit_avx512.asm rename to lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_zuc_submit_flush_avx512.asm rename to lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm diff --git a/lib/avx512/poly_avx512.asm b/lib/avx512_t1/poly_avx512.asm similarity index 100% rename from lib/avx512/poly_avx512.asm rename to lib/avx512_t1/poly_avx512.asm diff --git a/lib/avx512/sha1_x16_avx512.asm b/lib/avx512_t1/sha1_x16_avx512.asm similarity index 100% rename from lib/avx512/sha1_x16_avx512.asm rename to lib/avx512_t1/sha1_x16_avx512.asm diff --git a/lib/avx512/sha256_x16_avx512.asm b/lib/avx512_t1/sha256_x16_avx512.asm similarity index 100% rename from lib/avx512/sha256_x16_avx512.asm rename to lib/avx512_t1/sha256_x16_avx512.asm diff --git a/lib/avx512/sha512_x8_avx512.asm b/lib/avx512_t1/sha512_x8_avx512.asm similarity index 100% rename from lib/avx512/sha512_x8_avx512.asm rename to lib/avx512_t1/sha512_x8_avx512.asm diff --git a/lib/avx512/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c similarity index 100% rename from lib/avx512/sha_mb_avx512.c rename to lib/avx512_t1/sha_mb_avx512.c diff --git a/lib/avx512/snow3g_avx512.c b/lib/avx512_t1/snow3g_avx512.c similarity index 100% rename from lib/avx512/snow3g_avx512.c rename to lib/avx512_t1/snow3g_avx512.c diff --git a/lib/avx512/zuc_top_avx512.c b/lib/avx512_t1/zuc_top_avx512.c old mode 100755 new mode 100644 similarity index 100% rename from lib/avx512/zuc_top_avx512.c rename to lib/avx512_t1/zuc_top_avx512.c diff --git a/lib/avx512/zuc_x16_avx512.asm b/lib/avx512_t1/zuc_x16_avx512.asm old mode 100755 new mode 100644 similarity index 100% rename from lib/avx512/zuc_x16_avx512.asm rename to lib/avx512_t1/zuc_x16_avx512.asm diff --git a/lib/avx512/aes128_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes128_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm index dbf1da3c..90dbca2a 100644 --- a/lib/avx512/aes128_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gcm_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512/gcm_api_vaes_avx512.inc" +%include "avx512_t2/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512/aes128_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes128_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm index 6c291613..cd55ad0c 100644 --- a/lib/avx512/aes128_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gcm_by48_sgl_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512/gcm_sgl_api_vaes_avx512.inc" +%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512/aes128_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes128_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm index 6e050422..7915728b 100644 --- a/lib/avx512/aes128_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes128_gmac_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx512/gcm_gmac_api_vaes_avx512.inc" +%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512/aes192_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes192_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm index 8e7f4c6f..05693ed9 100644 --- a/lib/avx512/aes192_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gcm_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512/gcm_api_vaes_avx512.inc" +%include "avx512_t2/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512/aes192_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes192_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm index cff7c9a7..f48c2aaa 100644 --- a/lib/avx512/aes192_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gcm_by48_sgl_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512/gcm_sgl_api_vaes_avx512.inc" +%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512/aes192_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes192_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm index 8b5bcea6..71d785fd 100644 --- a/lib/avx512/aes192_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes192_gmac_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx512/gcm_gmac_api_vaes_avx512.inc" +%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512/aes256_gcm_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes256_gcm_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm index 5661f5be..b302958c 100644 --- a/lib/avx512/aes256_gcm_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gcm_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512/gcm_api_vaes_avx512.inc" +%include "avx512_t2/gcm_api_vaes_avx512.inc" diff --git a/lib/avx512/aes256_gcm_by48_sgl_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes256_gcm_by48_sgl_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm index 1f49cfa3..bf705eb8 100644 --- a/lib/avx512/aes256_gcm_by48_sgl_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gcm_by48_sgl_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512/gcm_sgl_api_vaes_avx512.inc" +%include "avx512_t2/gcm_sgl_api_vaes_avx512.inc" diff --git a/lib/avx512/aes256_gmac_by48_api_vaes_avx512.asm b/lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm similarity index 97% rename from lib/avx512/aes256_gmac_by48_api_vaes_avx512.asm rename to lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm index 43b7d830..bf648ffb 100644 --- a/lib/avx512/aes256_gmac_by48_api_vaes_avx512.asm +++ b/lib/avx512_t2/aes256_gmac_by48_api_vaes_avx512.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx512/gcm_gmac_api_vaes_avx512.inc" +%include "avx512_t2/gcm_gmac_api_vaes_avx512.inc" diff --git a/lib/avx512/aes_cbc_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_cbc_dec_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm diff --git a/lib/avx512/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_cbc_enc_vaes_avx512.asm rename to lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm diff --git a/lib/avx512/aes_cbcs_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm similarity index 98% rename from lib/avx512/aes_cbcs_dec_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm index efea7a85..c0a2fe35 100644 --- a/lib/avx512/aes_cbcs_dec_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm @@ -26,7 +26,7 @@ ;; %define CBCS -%include "avx512/aes_cbc_dec_by16_vaes_avx512.asm" +%include "avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm" %include "include/cet.inc" %define len rax diff --git a/lib/avx512/aes_cbcs_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_cbcs_enc_vaes_avx512.asm rename to lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm diff --git a/lib/avx512/aes_cntr_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm similarity index 98% rename from lib/avx512/aes_cntr_api_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm index f00625cb..1f551e02 100644 --- a/lib/avx512/aes_cntr_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/aes_cntr_by16_vaes_avx512.inc" +%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_128_submit_vaes_avx512 (IMB_JOB *job) diff --git a/lib/avx512/aes_cntr_bit_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm similarity index 98% rename from lib/avx512/aes_cntr_bit_api_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm index a7b47706..ef1df535 100644 --- a/lib/avx512/aes_cntr_bit_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/aes_cntr_by16_vaes_avx512.inc" +%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_bit_128_submit_vaes_avx512 (IMB_JOB *job) diff --git a/lib/avx512/aes_cntr_by16_vaes_avx512.inc b/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc similarity index 100% rename from lib/avx512/aes_cntr_by16_vaes_avx512.inc rename to lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc diff --git a/lib/avx512/aes_cntr_ccm_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm similarity index 98% rename from lib/avx512/aes_cntr_ccm_api_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm index 5f57046d..f28dcf72 100644 --- a/lib/avx512/aes_cntr_ccm_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/aes_cntr_by16_vaes_avx512.inc" +%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;IMB_JOB * aes_cntr_ccm_128_vaes_avx512(IMB_JOB *job) diff --git a/lib/avx512/aes_cntr_pon_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm similarity index 98% rename from lib/avx512/aes_cntr_pon_api_by16_vaes_avx512.asm rename to lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm index fdc7596f..78754018 100644 --- a/lib/avx512/aes_cntr_pon_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm @@ -28,7 +28,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/aes_cntr_by16_vaes_avx512.inc" +%include "avx512_t2/aes_cntr_by16_vaes_avx512.inc" ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;void aes_cntr_pon_enc_128_vaes_avx512 (void *src, void *dst, void *iv, void *keys, uint64_t length, uint32_t *bip) diff --git a/lib/avx512/aes_docsis_dec_avx512.asm b/lib/avx512_t2/aes_docsis_dec_avx512.asm similarity index 100% rename from lib/avx512/aes_docsis_dec_avx512.asm rename to lib/avx512_t2/aes_docsis_dec_avx512.asm diff --git a/lib/avx512/aes_docsis_dec_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_docsis_dec_vaes_avx512.asm rename to lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm diff --git a/lib/avx512/aes_docsis_enc_avx512.asm b/lib/avx512_t2/aes_docsis_enc_avx512.asm similarity index 100% rename from lib/avx512/aes_docsis_enc_avx512.asm rename to lib/avx512_t2/aes_docsis_enc_avx512.asm diff --git a/lib/avx512/aes_docsis_enc_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_docsis_enc_vaes_avx512.asm rename to lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm diff --git a/lib/avx512/aes_ecb_vaes_avx512.asm b/lib/avx512_t2/aes_ecb_vaes_avx512.asm similarity index 100% rename from lib/avx512/aes_ecb_vaes_avx512.asm rename to lib/avx512_t2/aes_ecb_vaes_avx512.asm diff --git a/lib/avx512/crc16_x25_avx512.asm b/lib/avx512_t2/crc16_x25_avx512.asm similarity index 100% rename from lib/avx512/crc16_x25_avx512.asm rename to lib/avx512_t2/crc16_x25_avx512.asm diff --git a/lib/avx512/crc32_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm similarity index 100% rename from lib/avx512/crc32_by16_vclmul_avx512.asm rename to lib/avx512_t2/crc32_by16_vclmul_avx512.asm diff --git a/lib/avx512/crc32_fp_avx512.asm b/lib/avx512_t2/crc32_fp_avx512.asm similarity index 100% rename from lib/avx512/crc32_fp_avx512.asm rename to lib/avx512_t2/crc32_fp_avx512.asm diff --git a/lib/avx512/crc32_iuup_avx512.asm b/lib/avx512_t2/crc32_iuup_avx512.asm similarity index 100% rename from lib/avx512/crc32_iuup_avx512.asm rename to lib/avx512_t2/crc32_iuup_avx512.asm diff --git a/lib/avx512/crc32_lte_avx512.asm b/lib/avx512_t2/crc32_lte_avx512.asm similarity index 100% rename from lib/avx512/crc32_lte_avx512.asm rename to lib/avx512_t2/crc32_lte_avx512.asm diff --git a/lib/avx512/crc32_refl_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm similarity index 100% rename from lib/avx512/crc32_refl_by16_vclmul_avx512.asm rename to lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm diff --git a/lib/avx512/crc32_sctp_avx512.asm b/lib/avx512_t2/crc32_sctp_avx512.asm similarity index 100% rename from lib/avx512/crc32_sctp_avx512.asm rename to lib/avx512_t2/crc32_sctp_avx512.asm diff --git a/lib/avx512/crc32_wimax_avx512.asm b/lib/avx512_t2/crc32_wimax_avx512.asm similarity index 100% rename from lib/avx512/crc32_wimax_avx512.asm rename to lib/avx512_t2/crc32_wimax_avx512.asm diff --git a/lib/avx512/ethernet_fcs_avx512.asm b/lib/avx512_t2/ethernet_fcs_avx512.asm similarity index 100% rename from lib/avx512/ethernet_fcs_avx512.asm rename to lib/avx512_t2/ethernet_fcs_avx512.asm diff --git a/lib/avx512/gcm_api_vaes_avx512.inc b/lib/avx512_t2/gcm_api_vaes_avx512.inc similarity index 99% rename from lib/avx512/gcm_api_vaes_avx512.inc rename to lib/avx512_t2/gcm_api_vaes_avx512.inc index 53a33d5c..1b76667d 100644 --- a/lib/avx512/gcm_api_vaes_avx512.inc +++ b/lib/avx512_t2/gcm_api_vaes_avx512.inc @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/gcm_vaes_avx512.inc" +%include "avx512_t2/gcm_vaes_avx512.inc" %include "include/error.inc" %include "include/os.asm" diff --git a/lib/avx512/gcm_gmac_api_vaes_avx512.inc b/lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc similarity index 99% rename from lib/avx512/gcm_gmac_api_vaes_avx512.inc rename to lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc index 8b174d3a..8ca8df8d 100644 --- a/lib/avx512/gcm_gmac_api_vaes_avx512.inc +++ b/lib/avx512_t2/gcm_gmac_api_vaes_avx512.inc @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/gcm_vaes_avx512.inc" +%include "avx512_t2/gcm_vaes_avx512.inc" %include "include/error.inc" %ifndef GCM_GMAC_API_VAES_AVX512_INC diff --git a/lib/avx512/gcm_sgl_api_vaes_avx512.inc b/lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc similarity index 99% rename from lib/avx512/gcm_sgl_api_vaes_avx512.inc rename to lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc index 20acc9ac..1ecc7ba1 100644 --- a/lib/avx512/gcm_sgl_api_vaes_avx512.inc +++ b/lib/avx512_t2/gcm_sgl_api_vaes_avx512.inc @@ -27,7 +27,7 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%include "avx512/gcm_vaes_avx512.inc" +%include "avx512_t2/gcm_vaes_avx512.inc" %include "include/error.inc" %include "include/os.asm" diff --git a/lib/avx512/gcm_vaes_avx512.inc b/lib/avx512_t2/gcm_vaes_avx512.inc similarity index 100% rename from lib/avx512/gcm_vaes_avx512.inc rename to lib/avx512_t2/gcm_vaes_avx512.inc diff --git a/lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm diff --git a/lib/avx512/mb_mgr_aes128_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_aes128_cbc_enc_submit_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm diff --git a/lib/avx512/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm similarity index 99% rename from lib/avx512/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm index 67f90f36..ab709c2e 100644 --- a/lib/avx512/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm @@ -26,7 +26,7 @@ ;; %define CBCS -%include "avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm" %include "include/cet.inc" %define AES_CBCS_ENC_X16 aes_cbcs_1_9_enc_128_vaes_avx512 %define FLUSH_JOB_AES_CBCS_ENC flush_job_aes128_cbcs_1_9_enc_vaes_avx512 diff --git a/lib/avx512/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm similarity index 99% rename from lib/avx512/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm index ae6569ae..ec25e86c 100644 --- a/lib/avx512/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm @@ -26,7 +26,7 @@ ;; %define CBCS -%include "avx512/mb_mgr_aes128_cbc_enc_submit_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm" %include "include/cet.inc" %define AES_CBCS_ENC_X16 aes_cbcs_1_9_enc_128_vaes_avx512 diff --git a/lib/avx512/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm diff --git a/lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm diff --git a/lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm similarity index 100% rename from lib/avx512/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm diff --git a/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm rename to lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm index 79a9998b..8a0416c9 100644 --- a/lib/avx512/mb_mgr_aes192_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_flush_avx512.asm @@ -28,4 +28,4 @@ %define AES_CBC_ENC_X16 aes_cbc_enc_192_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_vaes_avx512 %define NUM_KEYS 13 -%include "avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes192_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_aes192_cbc_enc_submit_avx512.asm rename to lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm index 8ec9458f..1bf43590 100644 --- a/lib/avx512/mb_mgr_aes192_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes192_cbc_enc_submit_avx512.asm @@ -28,4 +28,4 @@ %define AES_CBC_ENC_X16 aes_cbc_enc_192_vaes_avx512 %define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_vaes_avx512 %define NUM_KEYS 13 -%include "avx512/mb_mgr_aes128_cbc_enc_submit_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm rename to lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm index 7425217c..a3ec6b0c 100644 --- a/lib/avx512/mb_mgr_aes256_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_flush_avx512.asm @@ -28,4 +28,4 @@ %define AES_CBC_ENC_X16 aes_cbc_enc_256_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_vaes_avx512 %define NUM_KEYS 15 -%include "avx512/mb_mgr_aes128_cbc_enc_flush_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes256_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_aes256_cbc_enc_submit_avx512.asm rename to lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm index afb7f1d2..3e001baf 100644 --- a/lib/avx512/mb_mgr_aes256_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cbc_enc_submit_avx512.asm @@ -28,4 +28,4 @@ %define AES_CBC_ENC_X16 aes_cbc_enc_256_vaes_avx512 %define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_vaes_avx512 %define NUM_KEYS 15 -%include "avx512/mb_mgr_aes128_cbc_enc_submit_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm similarity index 95% rename from lib/avx512/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm index 8356e854..6bfdf6ea 100644 --- a/lib/avx512/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -32,4 +32,4 @@ %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes256_ccm_auth_vaes_avx512 %endif -%include "avx512/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm" diff --git a/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm similarity index 96% rename from lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm index 119fb6fc..d0224cf0 100644 --- a/lib/avx512/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes256_cmac_submit_flush_x16_vaes_avx512.asm @@ -33,4 +33,4 @@ %define NUM_KEYS 15 %endif -%include "avx512/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm" +%include "avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm" diff --git a/lib/avx512/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm similarity index 99% rename from lib/avx512/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm index 94f3c10d..7a0da197 100644 --- a/lib/avx512/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm @@ -33,7 +33,7 @@ %include "include/clear_regs.asm" %include "include/cet.inc" -%include "avx512/snow3g_uea2_by16_vaes_avx512.asm" +%include "avx512_t2/snow3g_uea2_by16_vaes_avx512.asm" %ifndef SUBMIT_JOB_SNOW3G_UEA2 %define SUBMIT_JOB_SNOW3G_UEA2_GEN2 submit_job_snow3g_uea2_vaes_avx512 diff --git a/lib/avx512/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm similarity index 99% rename from lib/avx512/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm rename to lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm index 34932ea6..9b3264c9 100644 --- a/lib/avx512/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm @@ -33,7 +33,7 @@ %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" -%include "avx512/snow3g_uea2_by16_vaes_avx512.asm" +%include "avx512_t2/snow3g_uea2_by16_vaes_avx512.asm" %ifndef SUBMIT_JOB_SNOW3G_UIA2 %define SUBMIT_JOB_SNOW3G_UIA2_GEN2 submit_job_snow3g_uia2_vaes_avx512 diff --git a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm b/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm similarity index 98% rename from lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm rename to lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm index 50a6ccad..76649d36 100644 --- a/lib/avx512/mb_mgr_zuc_submit_flush_gfni_avx512.asm +++ b/lib/avx512_t2/mb_mgr_zuc_submit_flush_gfni_avx512.asm @@ -47,4 +47,4 @@ %define ZUC_KEYGEN64B_16 asm_ZucGenKeystream64B_16_gfni_avx512 %define ZUC_ROUND64B asm_Eia3Round64B_16_VPCLMUL %define ZUC_EIA3_N64B asm_Eia3_Nx64B_AVX512_16_VPCLMUL -%include "avx512/mb_mgr_zuc_submit_flush_avx512.asm" +%include "avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm" diff --git a/lib/avx512/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm similarity index 100% rename from lib/avx512/poly_fma_avx512.asm rename to lib/avx512_t2/poly_fma_avx512.asm diff --git a/lib/avx512/pon_vaes_avx512.asm b/lib/avx512_t2/pon_vaes_avx512.asm similarity index 100% rename from lib/avx512/pon_vaes_avx512.asm rename to lib/avx512_t2/pon_vaes_avx512.asm diff --git a/lib/avx512/snow3g_uea2_by16_vaes_avx512.asm b/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm similarity index 100% rename from lib/avx512/snow3g_uea2_by16_vaes_avx512.asm rename to lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm diff --git a/lib/avx512/snow3g_uia2_by32_vaes_avx512.asm b/lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm similarity index 100% rename from lib/avx512/snow3g_uia2_by32_vaes_avx512.asm rename to lib/avx512_t2/snow3g_uia2_by32_vaes_avx512.asm diff --git a/lib/avx512/zuc_x16_vaes_avx512.asm b/lib/avx512_t2/zuc_x16_vaes_avx512.asm similarity index 98% rename from lib/avx512/zuc_x16_vaes_avx512.asm rename to lib/avx512_t2/zuc_x16_vaes_avx512.asm index 551fe808..42446d4f 100644 --- a/lib/avx512/zuc_x16_vaes_avx512.asm +++ b/lib/avx512_t2/zuc_x16_vaes_avx512.asm @@ -44,4 +44,4 @@ %define ZUC_KEYGEN_SKIP4_16 asm_ZucGenKeystream_16_skip4_gfni_avx512 %define ZUC_ROUND64B_16 asm_Eia3Round64B_16_VPCLMUL %define ZUC_EIA3_N64B asm_Eia3_Nx64B_AVX512_16_VPCLMUL -%include "avx512/zuc_x16_avx512.asm" +%include "avx512_t1/zuc_x16_avx512.asm" diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 62227390..192abb2c 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -646,11 +646,18 @@ $(DEPALL): $(all_objs) {avx2_t2\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< -{avx512\}.c{$(OBJ_DIR)}.obj: +{avx512_t1\}.c{$(OBJ_DIR)}.obj: $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep -{avx512\}.asm{$(OBJ_DIR)}.obj: +{avx512_t1\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +{avx512_t2\}.c{$(OBJ_DIR)}.obj: + $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{avx512_t2\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< {no-aesni\}.c{$(OBJ_DIR)}.obj: -- GitLab From b8298dce111d5dbf79e71bcdfe8ea61e5e0efaf9 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 10:38:37 +0000 Subject: [PATCH 156/369] lib: [zuc] remove exec mode from ZUC files --- lib/avx/zuc_top_avx.c | 0 lib/avx/zuc_x4_avx.asm | 0 lib/no-aesni/zuc_top_sse_no_aesni.c | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/avx/zuc_top_avx.c mode change 100755 => 100644 lib/avx/zuc_x4_avx.asm mode change 100755 => 100644 lib/no-aesni/zuc_top_sse_no_aesni.c diff --git a/lib/avx/zuc_top_avx.c b/lib/avx/zuc_top_avx.c old mode 100755 new mode 100644 diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm old mode 100755 new mode 100644 diff --git a/lib/no-aesni/zuc_top_sse_no_aesni.c b/lib/no-aesni/zuc_top_sse_no_aesni.c old mode 100755 new mode 100644 -- GitLab From 65f20d67bc079e94a7a9e3cda67fd4b6ab7f5e95 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 12:02:26 +0000 Subject: [PATCH 157/369] lib: [doc] added README file to each of the architecture type directories --- lib/avx/README | 3 +++ lib/avx2_t1/README | 3 +++ lib/avx2_t2/README | 3 +++ lib/avx512_t1/README | 3 +++ lib/avx512_t2/README | 3 +++ lib/sse_t1/README | 2 ++ lib/sse_t2/README | 3 +++ lib/sse_t3/README | 3 +++ 8 files changed, 23 insertions(+) create mode 100644 lib/avx/README create mode 100644 lib/avx2_t1/README create mode 100644 lib/avx2_t2/README create mode 100644 lib/avx512_t1/README create mode 100644 lib/avx512_t2/README create mode 100644 lib/sse_t1/README create mode 100644 lib/sse_t2/README create mode 100644 lib/sse_t3/README diff --git a/lib/avx/README b/lib/avx/README new file mode 100644 index 00000000..7302263d --- /dev/null +++ b/lib/avx/README @@ -0,0 +1,3 @@ +AVX: +- SSE TYPE1 +- AVX diff --git a/lib/avx2_t1/README b/lib/avx2_t1/README new file mode 100644 index 00000000..82bf0bcd --- /dev/null +++ b/lib/avx2_t1/README @@ -0,0 +1,3 @@ +AVX2 TYPE1: +- AVX +- AVX2, BMI2, AESNI, PCLMULQDQ diff --git a/lib/avx2_t2/README b/lib/avx2_t2/README new file mode 100644 index 00000000..e3fa7d2b --- /dev/null +++ b/lib/avx2_t2/README @@ -0,0 +1,3 @@ +AVX2 TYPE2: +- AVX2 TYPE1: AVX2, BMI2, AESNI, PCLMULQDQ, CMOV +- VAES, VPCLMULQDQ, SHANI, GFNI diff --git a/lib/avx512_t1/README b/lib/avx512_t1/README new file mode 100644 index 00000000..629d6232 --- /dev/null +++ b/lib/avx512_t1/README @@ -0,0 +1,3 @@ +AVX512 TYPE1: +- AVX2 TYPE1 +- AVX512F, AVX512VL, AVX512DQ, AVX512BW, AVX512CD, AESNI, PCLMULQDQ diff --git a/lib/avx512_t2/README b/lib/avx512_t2/README new file mode 100644 index 00000000..48aac0db --- /dev/null +++ b/lib/avx512_t2/README @@ -0,0 +1,3 @@ +AVX512 TYPE2: +- AVX512 TYPE1: AVX512F, AVX512VL, AVX512DQ, AVX512BW, AVX512CD, AESNI, PCLMULQDQ +- AVX512IFMA, AVX512VBMI, AVX512VBMI2, AVX512BITALG, AVX512VPOPCNTDQ, VAES, VPCLMULQDQ, GFNI, SHANI diff --git a/lib/sse_t1/README b/lib/sse_t1/README new file mode 100644 index 00000000..b0ea1cff --- /dev/null +++ b/lib/sse_t1/README @@ -0,0 +1,2 @@ +SSE TYPE1: +- SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP diff --git a/lib/sse_t2/README b/lib/sse_t2/README new file mode 100644 index 00000000..c296fc11 --- /dev/null +++ b/lib/sse_t2/README @@ -0,0 +1,3 @@ +SSE TYPE2: +- SSE TYPE1: SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP +- SHANI diff --git a/lib/sse_t3/README b/lib/sse_t3/README new file mode 100644 index 00000000..8854263e --- /dev/null +++ b/lib/sse_t3/README @@ -0,0 +1,3 @@ +SSE TYPE3: +- SSE TYPE2: SSE4.2, AESNI, PCLMULQDQ, CMOV, BSWAP, SHANI +- GFNI -- GitLab From b06352623aae0356bfd3181f78c737313a742ec8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 13:34:57 +0000 Subject: [PATCH 158/369] lib: moved gcm, gmac, docsis, snow-v and kasumi specific code out of mb_mgr_code.h --- lib/include/job_api_docsis.h | 127 ++++++++++++ lib/include/job_api_gcm.h | 221 ++++++++++++++++++++ lib/include/job_api_kasumi.h | 66 ++++++ lib/include/job_api_snowv.h | 93 +++++++++ lib/include/mb_mgr_code.h | 383 +---------------------------------- 5 files changed, 513 insertions(+), 377 deletions(-) create mode 100644 lib/include/job_api_docsis.h create mode 100644 lib/include/job_api_gcm.h create mode 100644 lib/include/job_api_kasumi.h create mode 100644 lib/include/job_api_snowv.h diff --git a/lib/include/job_api_docsis.h b/lib/include/job_api_docsis.h new file mode 100644 index 00000000..5df40d1f --- /dev/null +++ b/lib/include/job_api_docsis.h @@ -0,0 +1,127 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" +#include "include/docsis_common.h" + +#ifndef JOB_API_DOCSIS_H +#define JOB_API_DOCSIS_H + +__forceinline +IMB_JOB * +submit_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) { + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_crc32_sec_ooo; + + return SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC(p_ooo, job); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_sec_ooo; + + return SUBMIT_JOB_DOCSIS128_SEC_ENC(p_ooo, job); + } + } else { /* 32 */ + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_crc32_sec_ooo; + + return SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC(p_ooo, job); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_sec_ooo; + + return SUBMIT_JOB_DOCSIS256_SEC_ENC(p_ooo, job); + } + } +} + +__forceinline +IMB_JOB * +flush_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) { + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_crc32_sec_ooo; + + return FLUSH_JOB_DOCSIS128_SEC_CRC_ENC(p_ooo); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_sec_ooo; + + return FLUSH_JOB_DOCSIS128_SEC_ENC(p_ooo); + } + } else { /* 32 */ + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_crc32_sec_ooo; + + return FLUSH_JOB_DOCSIS256_SEC_CRC_ENC(p_ooo); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_sec_ooo; + + return FLUSH_JOB_DOCSIS256_SEC_ENC(p_ooo); + } + } +} + +__forceinline +IMB_JOB * +submit_docsis_dec_job(IMB_MGR *state, IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) { + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_crc32_sec_ooo; + + return SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC(p_ooo, job); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis128_sec_ooo; + + return SUBMIT_JOB_DOCSIS128_SEC_DEC(p_ooo, job); + } + } else { /* 32 */ + if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_crc32_sec_ooo; + + return SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC(p_ooo, job); + } else { + MB_MGR_DOCSIS_AES_OOO *p_ooo = + state->docsis256_sec_ooo; + + return SUBMIT_JOB_DOCSIS256_SEC_DEC(p_ooo, job); + } + } +} + +#endif /* JOB_API_DOCSIS_H */ diff --git a/lib/include/job_api_gcm.h b/lib/include/job_api_gcm.h new file mode 100644 index 00000000..6e8180f7 --- /dev/null +++ b/lib/include/job_api_gcm.h @@ -0,0 +1,221 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" + +#ifndef JOB_API_GCM_H +#define JOB_API_GCM_H + +__forceinline +IMB_JOB * +submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) +{ + switch (job->key_len_in_bytes) { + case IMB_KEY_128_BYTES: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES128_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES128_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + case IMB_KEY_192_BYTES: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES192_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES192_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + case IMB_KEY_256_BYTES: + default: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES256_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES256_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + } + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + + return job; +} + +__forceinline +IMB_JOB * +submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) +{ + switch (job->key_len_in_bytes) { + case IMB_KEY_128_BYTES: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES128_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES128_GCM_DEC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + case IMB_KEY_192_BYTES: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES192_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES192_GCM_DEC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + case IMB_KEY_256_BYTES: + default: + if (job->sgl_state == IMB_SGL_INIT) + IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + else if (job->sgl_state == IMB_SGL_UPDATE) + IMB_AES256_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->dst, job->src, + job->msg_len_to_cipher_in_bytes); + else /* FINALIZE */ + IMB_AES256_GCM_DEC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + break; + } + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + + return job; +} + +__forceinline +void +process_gmac(IMB_MGR *state, IMB_JOB *job, const IMB_KEY_SIZE_BYTES key_size) +{ + struct gcm_context_data ctx; + const struct gcm_key_data *key = job->u.GMAC._key; + const uint8_t *iv = job->u.GMAC._iv; + const uint64_t iv_len = job->u.GMAC.iv_len_in_bytes; + const uint8_t *src = job->src + job->hash_start_src_offset_in_bytes; + const uint64_t src_len = job->msg_len_to_hash_in_bytes; + + if (key_size == IMB_KEY_128_BYTES) { + IMB_AES128_GMAC_INIT(state, key, &ctx, iv, iv_len); + IMB_AES128_GMAC_UPDATE(state, key, &ctx, src, src_len); + IMB_AES128_GMAC_FINALIZE(state, key, &ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (key_size == IMB_KEY_192_BYTES) { + IMB_AES192_GMAC_INIT(state, key, &ctx, iv, iv_len); + IMB_AES192_GMAC_UPDATE(state, key, &ctx, src, src_len); + IMB_AES192_GMAC_FINALIZE(state, key, &ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* key_size == 256 */ + IMB_AES256_GMAC_INIT(state, key, &ctx, iv, iv_len); + IMB_AES256_GMAC_UPDATE(state, key, &ctx, src, src_len); + IMB_AES256_GMAC_FINALIZE(state, key, &ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } +} + +__forceinline IMB_JOB *process_ghash(IMB_MGR *state, IMB_JOB *job) +{ + /* copy initial tag value to the destination */ + memcpy(job->auth_tag_output, job->u.GHASH._init_tag, + job->auth_tag_output_len_in_bytes); + + /* compute new tag value */ + IMB_GHASH(state, job->u.GHASH._key, + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bytes, + job->auth_tag_output, job->auth_tag_output_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; +} + +#endif /* JOB_API_GCM_H */ diff --git a/lib/include/job_api_kasumi.h b/lib/include/job_api_kasumi.h new file mode 100644 index 00000000..f29df9a3 --- /dev/null +++ b/lib/include/job_api_kasumi.h @@ -0,0 +1,66 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" + +#ifndef JOB_API_KASUMI_H +#define JOB_API_KASUMI_H + +__forceinline +IMB_JOB * +submit_kasumi_uea1_job(IMB_MGR *state, IMB_JOB *job) +{ + const kasumi_key_sched_t *key = job->enc_keys; + const uint64_t iv = *(const uint64_t *)job->iv; + const uint32_t msg_bitlen = + (const uint32_t)job->msg_len_to_cipher_in_bits; + const uint32_t msg_bitoff = + (const uint32_t)job->cipher_start_src_offset_in_bits; + + /* Use bit length API if + * - msg length is not a multiple of bytes + * - bit offset is not a multiple of bytes + */ + if ((msg_bitlen & 0x07) || (msg_bitoff & 0x07)) { + IMB_KASUMI_F8_1_BUFFER_BIT(state, key, iv, job->src, job->dst, + msg_bitlen, msg_bitoff); + + } else { + const uint32_t msg_bytelen = msg_bitlen >> 3; + const uint32_t msg_byteoff = msg_bitoff >> 3; + const void *src = job->src + msg_byteoff; + void *dst = job->dst + msg_byteoff; + + IMB_KASUMI_F8_1_BUFFER(state, key, iv, src, dst, + msg_bytelen); + } + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +#endif /* JOB_API_KASUMI_H */ diff --git a/lib/include/job_api_snowv.h b/lib/include/job_api_snowv.h new file mode 100644 index 00000000..826eabfe --- /dev/null +++ b/lib/include/job_api_snowv.h @@ -0,0 +1,93 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" + +#ifndef JOB_API_SNOWV_H +#define JOB_API_SNOWV_H + +__forceinline +IMB_JOB * +submit_snow_v_aead_job(IMB_MGR *state, IMB_JOB *job) +{ + struct gcm_key_data gdata_key; + imb_uint128_t *auth = (imb_uint128_t *) job->auth_tag_output; + imb_uint128_t temp; + imb_uint128_t hkey_endpad[2]; + + temp.low = BSWAP64((job->u.SNOW_V_AEAD.aad_len_in_bytes << 3)); + temp.high = BSWAP64((job->msg_len_to_cipher_in_bytes << 3)); + + /* if hkey_endpad[1].high == 0: + * SUBMIT_JOB_SNOW_V_AEAD does enc/decrypt operation + * and fills hkey_endpad with first 2 keystreams + * else + * SUBMIT_JOB_SNOW_V_AEAD fills hkey_endpad with first + * 2 keystreams (no operations on src vector are done) + */ + if(job->cipher_direction == IMB_DIR_ENCRYPT) + hkey_endpad[1].high = 0; + else + hkey_endpad[1].high = 1; + + job->u.SNOW_V_AEAD.reserved = hkey_endpad; + job = SUBMIT_JOB_SNOW_V_AEAD(job); + + memset(auth, 0, sizeof(imb_uint128_t)); + + /* GHASH key H */ + IMB_GHASH_PRE(state, (void *)hkey_endpad, &gdata_key); + + /* push AAD into GHASH */ + IMB_GHASH(state, &gdata_key, job->u.SNOW_V_AEAD.aad, + job->u.SNOW_V_AEAD.aad_len_in_bytes, + (void *)auth, sizeof(imb_uint128_t)); + + if (job->cipher_direction == IMB_DIR_ENCRYPT) + IMB_GHASH(state, &gdata_key, job->dst, + job->msg_len_to_cipher_in_bytes, + (void *)auth, sizeof(imb_uint128_t)); + else + IMB_GHASH(state, &gdata_key, job->src, + job->msg_len_to_cipher_in_bytes, + (void *)auth, sizeof(imb_uint128_t)); + + IMB_GHASH(state, &gdata_key, (void *)&temp, sizeof(temp), + (void *)auth, sizeof(imb_uint128_t)); + + /* The resulting AuthTag */ + auth->low = auth->low ^ hkey_endpad[1].low; + auth->high = auth->high ^ hkey_endpad[1].high; + + if (job->cipher_direction == IMB_DIR_DECRYPT) { + hkey_endpad[1].high = 0; + job = SUBMIT_JOB_SNOW_V_AEAD(job); + } + return job; +} + +#endif /* JOB_API_SNOWV_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 45e612ae..fac4ee2d 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -44,8 +44,11 @@ #include "include/clear_regs_mem.h" #include "include/des.h" #include "intel-ipsec-mb.h" -#include "error.h" +#include "include/error.h" #include "include/snow3g_submit.h" +#include "include/job_api_gcm.h" +#include "include/job_api_snowv.h" +#include "include/job_api_kasumi.h" #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -207,10 +210,10 @@ SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) } /* ========================================================================= */ -/* DOCSIS functions */ +/* DOCSIS - it has to be below AES DEC */ /* ========================================================================= */ -#include "include/docsis_common.h" +#include "include/job_api_docsis.h" /* ========================================================================= */ /* Custom hash / cipher */ @@ -270,332 +273,6 @@ FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) return JOB_CUSTOM_HASH(job); } -__forceinline -IMB_JOB * -submit_kasumi_uea1_job(IMB_MGR *state, IMB_JOB *job) -{ - const kasumi_key_sched_t *key = job->enc_keys; - const uint64_t iv = *(const uint64_t *)job->iv; - const uint32_t msg_bitlen = - (const uint32_t)job->msg_len_to_cipher_in_bits; - const uint32_t msg_bitoff = - (const uint32_t)job->cipher_start_src_offset_in_bits; - - /* Use bit length API if - * - msg length is not a multiple of bytes - * - bit offset is not a multiple of bytes - */ - if ((msg_bitlen & 0x07) || (msg_bitoff & 0x07)) { - IMB_KASUMI_F8_1_BUFFER_BIT(state, key, iv, job->src, job->dst, - msg_bitlen, msg_bitoff); - - } else { - const uint32_t msg_bytelen = msg_bitlen >> 3; - const uint32_t msg_byteoff = msg_bitoff >> 3; - const void *src = job->src + msg_byteoff; - void *dst = job->dst + msg_byteoff; - - IMB_KASUMI_F8_1_BUFFER(state, key, iv, src, dst, - msg_bytelen); - } - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -submit_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) { - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_crc32_sec_ooo; - - return SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC(p_ooo, job); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_sec_ooo; - - return SUBMIT_JOB_DOCSIS128_SEC_ENC(p_ooo, job); - } - } else { /* 32 */ - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_crc32_sec_ooo; - - return SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC(p_ooo, job); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_sec_ooo; - - return SUBMIT_JOB_DOCSIS256_SEC_ENC(p_ooo, job); - } - } -} - -__forceinline -IMB_JOB * -flush_docsis_enc_job(IMB_MGR *state, IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) { - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_crc32_sec_ooo; - - return FLUSH_JOB_DOCSIS128_SEC_CRC_ENC(p_ooo); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_sec_ooo; - - return FLUSH_JOB_DOCSIS128_SEC_ENC(p_ooo); - } - } else { /* 32 */ - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_crc32_sec_ooo; - - return FLUSH_JOB_DOCSIS256_SEC_CRC_ENC(p_ooo); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_sec_ooo; - - return FLUSH_JOB_DOCSIS256_SEC_ENC(p_ooo); - } - } -} - -__forceinline -IMB_JOB * -submit_docsis_dec_job(IMB_MGR *state, IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) { - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_crc32_sec_ooo; - - return SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC(p_ooo, job); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis128_sec_ooo; - - return SUBMIT_JOB_DOCSIS128_SEC_DEC(p_ooo, job); - } - } else { /* 32 */ - if (job->hash_alg == IMB_AUTH_DOCSIS_CRC32) { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_crc32_sec_ooo; - - return SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC(p_ooo, job); - } else { - MB_MGR_DOCSIS_AES_OOO *p_ooo = - state->docsis256_sec_ooo; - - return SUBMIT_JOB_DOCSIS256_SEC_DEC(p_ooo, job); - } - } -} - -__forceinline -IMB_JOB * -submit_snow_v_aead_job(IMB_MGR *state, IMB_JOB *job) -{ - struct gcm_key_data gdata_key; - imb_uint128_t *auth = (imb_uint128_t *) job->auth_tag_output; - imb_uint128_t temp; - imb_uint128_t hkey_endpad[2]; - - temp.low = BSWAP64((job->u.SNOW_V_AEAD.aad_len_in_bytes << 3)); - temp.high = BSWAP64((job->msg_len_to_cipher_in_bytes << 3)); - - /* if hkey_endpad[1].high == 0: - * SUBMIT_JOB_SNOW_V_AEAD does enc/decrypt operation - * and fills hkey_endpad with first 2 keystreams - * else - * SUBMIT_JOB_SNOW_V_AEAD fills hkey_endpad with first - * 2 keystreams (no operations on src vector are done) - */ - if(job->cipher_direction == IMB_DIR_ENCRYPT) - hkey_endpad[1].high = 0; - else - hkey_endpad[1].high = 1; - - job->u.SNOW_V_AEAD.reserved = hkey_endpad; - job = SUBMIT_JOB_SNOW_V_AEAD(job); - - memset(auth, 0, sizeof(imb_uint128_t)); - - /* GHASH key H */ - IMB_GHASH_PRE(state, (void *)hkey_endpad, &gdata_key); - - /* push AAD into GHASH */ - IMB_GHASH(state, &gdata_key, job->u.SNOW_V_AEAD.aad, - job->u.SNOW_V_AEAD.aad_len_in_bytes, - (void *)auth, sizeof(imb_uint128_t)); - - if (job->cipher_direction == IMB_DIR_ENCRYPT) - IMB_GHASH(state, &gdata_key, job->dst, - job->msg_len_to_cipher_in_bytes, - (void *)auth, sizeof(imb_uint128_t)); - else - IMB_GHASH(state, &gdata_key, job->src, - job->msg_len_to_cipher_in_bytes, - (void *)auth, sizeof(imb_uint128_t)); - - IMB_GHASH(state, &gdata_key, (void *)&temp, sizeof(temp), - (void *)auth, sizeof(imb_uint128_t)); - - /* The resulting AuthTag */ - auth->low = auth->low ^ hkey_endpad[1].low; - auth->high = auth->high ^ hkey_endpad[1].high; - - if (job->cipher_direction == IMB_DIR_DECRYPT) { - hkey_endpad[1].high = 0; - job = SUBMIT_JOB_SNOW_V_AEAD(job); - } - return job; -} - -__forceinline -IMB_JOB * -submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) -{ - switch (job->key_len_in_bytes) { - case IMB_KEY_128_BYTES: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES128_GCM_ENC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES128_GCM_ENC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - case IMB_KEY_192_BYTES: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES192_GCM_ENC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES192_GCM_ENC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - case IMB_KEY_256_BYTES: - default: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES256_GCM_ENC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES256_GCM_ENC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - } - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - - return job; -} - -__forceinline -IMB_JOB * -submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) -{ - switch (job->key_len_in_bytes) { - case IMB_KEY_128_BYTES: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES128_GCM_DEC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES128_GCM_DEC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - case IMB_KEY_192_BYTES: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES192_GCM_DEC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES192_GCM_DEC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - case IMB_KEY_256_BYTES: - default: - if (job->sgl_state == IMB_SGL_INIT) - IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, - job->u.GCM.ctx, - job->iv, - job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes); - else if (job->sgl_state == IMB_SGL_UPDATE) - IMB_AES256_GCM_DEC_UPDATE(state, job->enc_keys, - job->u.GCM.ctx, - job->dst, job->src, - job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ - IMB_AES256_GCM_DEC_FINALIZE(state, job->enc_keys, - job->u.GCM.ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - break; - } - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - - return job; -} - /* ========================================================================= */ /* Cipher submit & flush functions */ /* ========================================================================= */ @@ -941,54 +618,6 @@ FLUSH_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) /* Hash submit & flush functions */ /* ========================================================================= */ -__forceinline -void -process_gmac(IMB_MGR *state, IMB_JOB *job, const IMB_KEY_SIZE_BYTES key_size) -{ - struct gcm_context_data ctx; - const struct gcm_key_data *key = job->u.GMAC._key; - const uint8_t *iv = job->u.GMAC._iv; - const uint64_t iv_len = job->u.GMAC.iv_len_in_bytes; - const uint8_t *src = job->src + job->hash_start_src_offset_in_bytes; - const uint64_t src_len = job->msg_len_to_hash_in_bytes; - - if (key_size == IMB_KEY_128_BYTES) { - IMB_AES128_GMAC_INIT(state, key, &ctx, iv, iv_len); - IMB_AES128_GMAC_UPDATE(state, key, &ctx, src, src_len); - IMB_AES128_GMAC_FINALIZE(state, key, &ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (key_size == IMB_KEY_192_BYTES) { - IMB_AES192_GMAC_INIT(state, key, &ctx, iv, iv_len); - IMB_AES192_GMAC_UPDATE(state, key, &ctx, src, src_len); - IMB_AES192_GMAC_FINALIZE(state, key, &ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* key_size == 256 */ - IMB_AES256_GMAC_INIT(state, key, &ctx, iv, iv_len); - IMB_AES256_GMAC_UPDATE(state, key, &ctx, src, src_len); - IMB_AES256_GMAC_FINALIZE(state, key, &ctx, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } -} - -__forceinline IMB_JOB *process_ghash(IMB_MGR *state, IMB_JOB *job) -{ - /* copy initial tag value to the destination */ - memcpy(job->auth_tag_output, job->u.GHASH._init_tag, - job->auth_tag_output_len_in_bytes); - - /* compute new tag value */ - IMB_GHASH(state, job->u.GHASH._key, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, - job->auth_tag_output, job->auth_tag_output_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -} - __forceinline IMB_JOB * SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) -- GitLab From 3d7454a194b72d1e1d63fd1129a2a784472d6bdf Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 15:29:32 +0000 Subject: [PATCH 159/369] lib: add new module with functions for resetting OOO MGR structures - start with AES and DOCSIS AES out-of-order manager structures --- lib/Makefile | 3 +- lib/include/ooo_mgr_reset.h | 37 +++++++++++++++++++++ lib/win_x64.mak | 3 +- lib/x86_64/ooo_mgr_reset.c | 65 +++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 lib/include/ooo_mgr_reset.h create mode 100644 lib/x86_64/ooo_mgr_reset.c diff --git a/lib/Makefile b/lib/Makefile index f9cddad8..f364b619 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -280,7 +280,8 @@ c_lib_objs := \ snow3g_tables.o \ snow3g_iv.o \ mb_mgr_auto.o \ - error.o + error.o \ + ooo_mgr_reset.o ifeq ($(AESNI_EMU), y) c_lib_objs := $(c_lib_objs) \ diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h new file mode 100644 index 00000000..9f9febdb --- /dev/null +++ b/lib/include/ooo_mgr_reset.h @@ -0,0 +1,37 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef OOO_MGR_RESET_H +#define OOO_MGR_RESET_H + +IMB_DLL_LOCAL void +ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); + +IMB_DLL_LOCAL void +ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); + +#endif /* OOO_MGR_RESET_H */ diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 192abb2c..17c58f72 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -352,7 +352,8 @@ lib_objs1 = \ $(OBJ_DIR)\mb_mgr_aes128_cbcs_1_9_flush_avx.obj \ $(OBJ_DIR)\error.obj \ $(OBJ_DIR)\memcpy_sse.obj \ - $(OBJ_DIR)\memcpy_avx.obj + $(OBJ_DIR)\memcpy_avx.obj \ + $(OBJ_DIR)\ooo_mgr_reset.obj lib_objs2 = \ $(OBJ_DIR)\mb_mgr_aes192_cbc_enc_flush_avx.obj \ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c new file mode 100644 index 00000000..f8adaa10 --- /dev/null +++ b/lib/x86_64/ooo_mgr_reset.c @@ -0,0 +1,65 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/ooo_mgr_reset.h" + +IMB_DLL_LOCAL +void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_AES_OOO *p_mgr = (MB_MGR_AES_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; + +} + +IMB_DLL_LOCAL +void ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_DOCSIS_AES_OOO *p_mgr = (MB_MGR_DOCSIS_AES_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; + +} + -- GitLab From 0d452f1efe55d02b7de2dd9555378f43f08957f8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 15:30:36 +0000 Subject: [PATCH 160/369] lib: use new API's to reset AES and DOCSIS AES OOO MGR structures --- lib/avx/mb_mgr_avx.c | 78 ++----------------- lib/avx2_t1/mb_mgr_avx2.c | 79 +++----------------- lib/avx512_t1/mb_mgr_avx512.c | 137 ++++------------------------------ lib/sse_t1/mb_mgr_sse.c | 85 +++++---------------- 4 files changed, 52 insertions(+), 327 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 2914aeaf..319fe224 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -45,6 +45,7 @@ #include "include/aesni_emu.h" #include "include/error.h" #include "include/arch_avx_type1.h" +#include "include/ooo_mgr_reset.h" #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -374,15 +375,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; @@ -402,71 +394,17 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(&aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) * uses same settings as AES CBC. */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); /* Init ZUC out-of-order fields */ memset(zuc_eea3_ooo->lens, 0, diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index f7acabdb..8d1bd56a 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -52,6 +52,8 @@ #include "include/arch_avx2_type1.h" #include "include/arch_avx2_type2.h" +#include "include/ooo_mgr_reset.h" + #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -353,15 +355,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; @@ -381,71 +374,17 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(&aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) * uses same settings as AES CBC. */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); /* Init ZUC out-of-order fields */ memset(zuc_eea3_ooo->lens, 0, diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index d50c319f..321d28cb 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -54,6 +54,8 @@ #include "include/arch_avx512_type1.h" #include "include/arch_avx512_type2.h" +#include "include/ooo_mgr_reset.h" + #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx @@ -874,15 +876,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; @@ -912,54 +905,14 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { /* init 16 lanes */ - memset(aes128_ooo->lens, 0, - sizeof(aes128_ooo->lens)); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xFEDCBA9876543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0, - sizeof(aes192_ooo->lens)); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xFEDCBA9876543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(aes256_ooo->lens, 0, - sizeof(aes256_ooo->lens)); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_ooo, 16); + ooo_mgr_aes_reset(state->aes192_ooo, 16); + ooo_mgr_aes_reset(state->aes256_ooo, 16); } else { /* init 8 lanes */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 8); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF76543210; - aes128_ooo->num_lanes_inuse = 0; - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 8); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF76543210; - aes192_ooo->num_lanes_inuse = 0; - - memset(aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 8); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF76543210; - aes256_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); } @@ -968,74 +921,16 @@ reset_ooo_mgrs(IMB_MGR *state) */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { /* init 16 lanes */ - memset(docsis128_sec_ooo->lens, 0, - sizeof(docsis128_sec_ooo->lens)); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0, - sizeof(docsis256_sec_ooo->lens)); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - } else { - /* init 8 lanes */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 8); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF76543210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 8); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF76543210; - docsis256_sec_ooo->num_lanes_inuse = 0; - } - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - memset(docsis128_crc32_sec_ooo->lens, 0, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 16); } else { /* init 8 lanes */ - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 8); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 8); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); } /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 56e5d836..fccbe6e8 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -52,6 +52,8 @@ #include "include/arch_sse_type2.h" #include "include/arch_sse_type3.h" +#include "include/ooo_mgr_reset.h" + #define SAVE_XMMS save_xmms #define RESTORE_XMMS restore_xmms @@ -547,15 +549,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; @@ -576,40 +569,28 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, sizeof(aes128_ooo->lens)); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->num_lanes_inuse = 0; if (state->features & IMB_FEATURE_GFNI) { - aes128_ooo->unused_lanes = 0xF76543210; + ooo_mgr_aes_reset(state->aes128_ooo, 8); submit_job_aes128_enc_ptr = submit_job_aes128_enc_x8_sse; flush_job_aes128_enc_ptr = flush_job_aes128_enc_x8_sse; } else { - aes128_ooo->unused_lanes = 0xF3210; + ooo_mgr_aes_reset(state->aes128_ooo, 4); } - memset(aes192_ooo->lens, 0xFF, sizeof(aes192_ooo->lens)); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->num_lanes_inuse = 0; if (state->features & IMB_FEATURE_GFNI) { - aes192_ooo->unused_lanes = 0xF76543210; + ooo_mgr_aes_reset(state->aes192_ooo, 8); submit_job_aes192_enc_ptr = submit_job_aes192_enc_x8_sse; flush_job_aes192_enc_ptr = flush_job_aes192_enc_x8_sse; } else { - aes192_ooo->unused_lanes = 0xF3210; + ooo_mgr_aes_reset(state->aes192_ooo, 4); } - memset(aes256_ooo->lens, 0xFF, sizeof(aes256_ooo->lens)); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->num_lanes_inuse = 0; if (state->features & IMB_FEATURE_GFNI) { - aes256_ooo->unused_lanes = 0xF76543210; + ooo_mgr_aes_reset(state->aes256_ooo, 8); submit_job_aes256_enc_ptr = submit_job_aes256_enc_x8_sse; flush_job_aes256_enc_ptr = flush_job_aes256_enc_x8_sse; } else { - aes256_ooo->unused_lanes = 0xF3210; + ooo_mgr_aes_reset(state->aes256_ooo, 4); } if (state->features & IMB_FEATURE_GFNI) { @@ -631,45 +612,17 @@ reset_ooo_mgrs(IMB_MGR *state) } /* DOCSIS SEC BPI uses same settings as AES CBC */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis128_sec_ooo->unused_lanes = 0xF76543210; - else - docsis128_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis128_crc32_sec_ooo->unused_lanes = 0xF76543210; - else - docsis128_crc32_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis256_sec_ooo->unused_lanes = 0xF76543210; - else - docsis256_sec_ooo->unused_lanes = 0xF3210; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; - if (state->features & IMB_FEATURE_GFNI) - docsis256_crc32_sec_ooo->unused_lanes = 0xF76543210; - else - docsis256_crc32_sec_ooo->unused_lanes = 0xF3210; + if (state->features & IMB_FEATURE_GFNI) { + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + } else { + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 4); + } /* Init ZUC out-of-order fields */ memset(zuc_eea3_ooo->lens, 0, -- GitLab From 4b854e471545c3045b128329bcc676c64dc0c4d9 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 15:37:42 +0000 Subject: [PATCH 161/369] lib: use new API to reset CBCS OOO MGR structures --- lib/avx/mb_mgr_avx.c | 10 +--------- lib/avx2_t1/mb_mgr_avx2.c | 10 +--------- lib/avx512_t1/mb_mgr_avx512.c | 26 ++++---------------------- lib/sse_t1/mb_mgr_sse.c | 7 +------ lib/x86_64/ooo_mgr_reset.c | 9 ++++++--- 5 files changed, 13 insertions(+), 49 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 319fe224..0809d6a3 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -389,7 +389,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; @@ -687,14 +686,7 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_cmac_ooo->num_lanes_inuse = 0; /* Init AES CBC-S out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ sha_1_ooo->lens[0] = 0; diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 8d1bd56a..b76630c4 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -370,7 +370,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ @@ -667,14 +666,7 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_cmac_ooo->num_lanes_inuse = 0; /* Init AES CBC-S out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ sha_1_ooo->unused_lanes = 0xF76543210; diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 321d28cb..1fe7476c 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -897,7 +897,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; @@ -1365,27 +1364,10 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES CBC-S out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 12 lanes */ - memset(aes128_cbcs_ooo->lens64, 0xFF, - sizeof(aes128_cbcs_ooo->lens64)); - memset(&aes128_cbcs_ooo->lens64[0], 0, - sizeof(aes128_cbcs_ooo->lens64[0]) * 12); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xBA9876543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; - - } else { - memset(aes128_cbcs_ooo->lens, 0xFF, - sizeof(aes128_cbcs_ooo->lens)); - memset(&aes128_cbcs_ooo->lens[0], 0, - sizeof(aes128_cbcs_ooo->lens[0]) * 8); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->unused_lanes = 0xF76543210; - aes128_cbcs_ooo->num_lanes_inuse = 0; - } + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 12); + else + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SNOW3G out-of-order fields */ memset(&snow3g_uea2_ooo->args, 0, diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index fccbe6e8..e330737c 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -563,7 +563,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; @@ -976,11 +975,7 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES-CBCS out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, sizeof(aes128_cbcs_ooo->lens)); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->num_lanes_inuse = 0; - aes128_cbcs_ooo->unused_lanes = 0xF3210; + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); /* Init SHA1 out-of-order fields */ sha_1_ooo->lens[0] = 0; diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index f8adaa10..ed3a64f6 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -42,9 +42,13 @@ void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF3210; else if (num_lanes == 8) p_mgr->unused_lanes = 0xF76543210; - else if (num_lanes == 16) + else if (num_lanes == 12) { + /* CBCS only */ + p_mgr->unused_lanes = 0xBA9876543210; + memset(&p_mgr->lens64[12], 0xFF, + sizeof(p_mgr->lens64) - sizeof(p_mgr->lens64[0] * 12)); + } else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; - } IMB_DLL_LOCAL @@ -60,6 +64,5 @@ void ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF76543210; else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; - } -- GitLab From 0f4100f3a22239b483729b169063fa87f4eff0c0 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 15:58:01 +0000 Subject: [PATCH 162/369] lib: add new API and use it to reset CMAC OOO MGR structures --- lib/avx/mb_mgr_avx.c | 19 ++------------- lib/avx2_t1/mb_mgr_avx2.c | 19 ++------------- lib/avx512_t1/mb_mgr_avx512.c | 44 ++++------------------------------- lib/include/ooo_mgr_reset.h | 3 +++ lib/sse_t1/mb_mgr_sse.c | 24 ++++--------------- lib/x86_64/ooo_mgr_reset.c | 14 +++++++++++ 6 files changed, 29 insertions(+), 94 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 0809d6a3..f16e6f54 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -384,8 +384,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -669,21 +667,8 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_ccm_ooo->num_lanes_inuse = 0; /* Init AES-CMAC auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_cmac_ooo->init_done[j] = 0; - aes_cmac_ooo->lens[j] = 0; - aes_cmac_ooo->job_in_lane[j] = NULL; - } - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_cmac_ooo->init_done[j] = 0; - aes256_cmac_ooo->lens[j] = 0; - aes256_cmac_ooo->job_in_lane[j] = NULL; - } - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); /* Init AES CBC-S out-of-order fields */ ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index b76630c4..69fbb009 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -364,8 +364,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -649,21 +647,8 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_ccm_ooo->num_lanes_inuse = 0; /* Init AES-CMAC auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_cmac_ooo->init_done[j] = 0; - aes_cmac_ooo->lens[j] = 0; - aes_cmac_ooo->job_in_lane[j] = NULL; - } - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_cmac_ooo->init_done[j] = 0; - aes256_cmac_ooo->lens[j] = 0; - aes256_cmac_ooo->job_in_lane[j] = NULL; - } - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); /* Init AES CBC-S out-of-order fields */ ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 1fe7476c..8e9f3469 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -891,8 +891,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -1321,46 +1319,12 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init AES-CMAC auth out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { /* init 16 lanes */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0, - sizeof(aes_cmac_ooo->lens)); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0, - sizeof(aes256_cmac_ooo->lens)); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_cmac_ooo->num_lanes_inuse = 0; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 16); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 16); } else { /* init 8 lanes */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0xFF, - sizeof(aes_cmac_ooo->lens)); - memset(&aes_cmac_ooo->lens[0], 0, - sizeof(aes_cmac_ooo->lens[0]) * 8); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->unused_lanes = 0xF76543210; - aes_cmac_ooo->num_lanes_inuse = 0; - - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0xFF, - sizeof(aes256_cmac_ooo->lens)); - memset(&aes256_cmac_ooo->lens[0], 0, - sizeof(aes256_cmac_ooo->lens[0]) * 8); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->unused_lanes = 0xF76543210; - aes256_cmac_ooo->num_lanes_inuse = 0; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); } /* Init AES CBC-S out-of-order fields */ diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 9f9febdb..f5f61480 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -34,4 +34,7 @@ ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL void +ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index e330737c..4a77d743 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -558,8 +558,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -940,38 +938,24 @@ reset_ooo_mgrs(IMB_MGR *state) aes256_ccm_ooo->num_lanes_inuse = 0; /* Init AES-CMAC auth out-of-order fields */ - memset(aes_cmac_ooo->init_done, 0, - sizeof(aes_cmac_ooo->init_done)); - memset(aes_cmac_ooo->lens, 0xff, - sizeof(aes_cmac_ooo->lens)); - memset(aes_cmac_ooo->job_in_lane, 0, - sizeof(aes_cmac_ooo->job_in_lane)); - aes_cmac_ooo->num_lanes_inuse = 0; if (state->features & IMB_FEATURE_GFNI) { submit_job_aes128_cmac_auth_ptr = submit_job_aes128_cmac_auth_x8_sse; flush_job_aes128_cmac_auth_ptr = flush_job_aes128_cmac_auth_x8_sse; - aes_cmac_ooo->unused_lanes = 0xF76543210; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); } else { - aes_cmac_ooo->unused_lanes = 0xF3210; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 4); } - memset(aes256_cmac_ooo->init_done, 0, - sizeof(aes256_cmac_ooo->init_done)); - memset(aes256_cmac_ooo->lens, 0xff, - sizeof(aes256_cmac_ooo->lens)); - memset(aes256_cmac_ooo->job_in_lane, 0, - sizeof(aes256_cmac_ooo->job_in_lane)); - aes256_cmac_ooo->num_lanes_inuse = 0; if (state->features & IMB_FEATURE_GFNI) { submit_job_aes256_cmac_auth_ptr = submit_job_aes256_cmac_auth_x8_sse; flush_job_aes256_cmac_auth_ptr = flush_job_aes256_cmac_auth_x8_sse; - aes256_cmac_ooo->unused_lanes = 0xF76543210; + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); } else { - aes256_cmac_ooo->unused_lanes = 0xF3210; + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 4); } /* Init AES-CBCS out-of-order fields */ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index ed3a64f6..6f3a61ad 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -66,3 +66,17 @@ void ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xFEDCBA9876543210; } +IMB_DLL_LOCAL +void ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_CMAC_OOO *p_mgr = (MB_MGR_CMAC_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From 51c1e7cb53f2330ed49a9bdcc31e18ddb82a9a97 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 16:58:27 +0000 Subject: [PATCH 163/369] lib: add new API and use it to reset CCM OOO MGR structures --- lib/avx/mb_mgr_avx.c | 25 ++----------------- lib/avx2_t1/mb_mgr_avx2.c | 25 ++----------------- lib/avx512_t1/mb_mgr_avx512.c | 45 ++++------------------------------- lib/include/ooo_mgr_reset.h | 3 +++ lib/sse_t1/mb_mgr_sse.c | 22 ++++------------- lib/x86_64/ooo_mgr_reset.c | 15 ++++++++++++ 6 files changed, 30 insertions(+), 105 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index f16e6f54..a763edc7 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -382,8 +382,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -644,27 +642,8 @@ reset_ooo_mgrs(IMB_MGR *state) aes_xcbc_ooo->num_lanes_inuse = 0; /* Init AES-CCM auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); /* Init AES-CMAC auth out-of-order fields */ ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 69fbb009..54c68adc 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -362,8 +362,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -624,27 +622,8 @@ reset_ooo_mgrs(IMB_MGR *state) aes_xcbc_ooo->num_lanes_inuse = 0; /* Init AES-CCM auth out-of-order fields */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); /* Init AES-CMAC auth out-of-order fields */ ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 8e9f3469..e9f70f01 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -889,8 +889,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -1273,47 +1271,12 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init AES-CCM auth out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { /* init 16 lanes */ - memset(aes_ccm_ooo->init_done, 0, - sizeof(aes_ccm_ooo->init_done)); - memset(aes_ccm_ooo->lens, 0, - sizeof(aes_ccm_ooo->lens)); - memset(aes_ccm_ooo->job_in_lane, 0, - sizeof(aes_ccm_ooo->job_in_lane)); - aes_ccm_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - memset(aes256_ccm_ooo->init_done, 0, - sizeof(aes256_ccm_ooo->init_done)); - memset(aes256_ccm_ooo->lens, 0, - sizeof(aes256_ccm_ooo->lens)); - memset(aes256_ccm_ooo->job_in_lane, 0, - sizeof(aes256_ccm_ooo->job_in_lane)); - aes256_ccm_ooo->unused_lanes = 0xFEDCBA9876543210; - aes256_ccm_ooo->num_lanes_inuse = 0; - + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 16); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 16); } else { /* init 8 lanes */ - for (j = 0; j < 8; j++) { - aes_ccm_ooo->init_done[j] = 0; - aes_ccm_ooo->lens[j] = 0; - aes_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - - aes_ccm_ooo->unused_lanes = 0xF76543210; - aes_ccm_ooo->num_lanes_inuse = 0; - - for (j = 0; j < 8; j++) { - aes256_ccm_ooo->init_done[j] = 0; - aes256_ccm_ooo->lens[j] = 0; - aes256_ccm_ooo->job_in_lane[j] = NULL; - } - for (; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - - aes256_ccm_ooo->unused_lanes = 0xF76543210; - aes256_ccm_ooo->num_lanes_inuse = 0; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); } /* Init AES-CMAC auth out-of-order fields */ diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index f5f61480..c4724f77 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -37,4 +37,7 @@ ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL void +ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 4a77d743..f8420e67 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -556,8 +556,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -905,37 +903,25 @@ reset_ooo_mgrs(IMB_MGR *state) aes_xcbc_ooo->num_lanes_inuse = 0; /* Init AES-CCM auth out-of-order fields */ - memset(aes_ccm_ooo->init_done, 0, - sizeof(aes_ccm_ooo->init_done)); - memset(aes_ccm_ooo->lens, 0xff, sizeof(aes_ccm_ooo->lens)); - memset(aes_ccm_ooo->job_in_lane, 0, - sizeof(aes_ccm_ooo->job_in_lane)); if (state->features & IMB_FEATURE_GFNI) { submit_job_aes128_ccm_auth_ptr = submit_job_aes128_ccm_auth_x8_sse; flush_job_aes128_ccm_auth_ptr = flush_job_aes128_ccm_auth_x8_sse; - aes_ccm_ooo->unused_lanes = 0xF76543210; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); } else { - aes_ccm_ooo->unused_lanes = 0xF3210; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 4); } - aes_ccm_ooo->num_lanes_inuse = 0; - memset(aes256_ccm_ooo->init_done, 0, - sizeof(aes256_ccm_ooo->init_done)); - memset(aes256_ccm_ooo->lens, 0xff, sizeof(aes256_ccm_ooo->lens)); - memset(aes256_ccm_ooo->job_in_lane, 0, - sizeof(aes256_ccm_ooo->job_in_lane)); if (state->features & IMB_FEATURE_GFNI) { submit_job_aes256_ccm_auth_ptr = submit_job_aes256_ccm_auth_x8_sse; flush_job_aes256_ccm_auth_ptr = flush_job_aes256_ccm_auth_x8_sse; - aes256_ccm_ooo->unused_lanes = 0xF76543210; + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); } else { - aes256_ccm_ooo->unused_lanes = 0xF3210; + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 4); } - aes256_ccm_ooo->num_lanes_inuse = 0; /* Init AES-CMAC auth out-of-order fields */ if (state->features & IMB_FEATURE_GFNI) { diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 6f3a61ad..2dbbbd44 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -80,3 +80,18 @@ void ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_CCM_OOO *p_mgr = (MB_MGR_CCM_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From de776f5b3dbdfebf766c871ce67bb22ec29b0a51 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 19:04:01 +0000 Subject: [PATCH 164/369] lib: add new API and use it to reset XCBC OOO MGR structures --- lib/avx/mb_mgr_avx.c | 12 +----------- lib/avx2_t1/mb_mgr_avx2.c | 11 +---------- lib/avx512_t1/mb_mgr_avx512.c | 25 ++++--------------------- lib/include/ooo_mgr_reset.h | 3 +++ lib/sse_t1/mb_mgr_sse.c | 12 +----------- lib/x86_64/ooo_mgr_reset.c | 21 +++++++++++++++++++++ 6 files changed, 31 insertions(+), 53 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index a763edc7..523b37ae 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -381,7 +381,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -630,16 +629,7 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].job_in_lane = NULL; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); /* Init AES-CCM auth out-of-order fields */ ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 54c68adc..a376e6d7 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -361,7 +361,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -611,15 +610,7 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); /* Init AES-CCM auth out-of-order fields */ ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index e9f70f01..25cbc05a 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -1246,27 +1246,10 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES/XCBC OOO fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - aes_xcbc_ooo->unused_lanes = 0xFEDCBA9876543210; - aes_xcbc_ooo->num_lanes_inuse = 0; - for (j = 0; j < 16; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, - 0x00, 15); - } - } else { - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xF76543210; - aes_xcbc_ooo->num_lanes_inuse = 0; - for (j = 0; j < 8; j++) { - aes_xcbc_ooo->lens[j] = 0; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, - 0x00, 15); - } - } + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 16); + else + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); /* Init AES-CCM auth out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index c4724f77..c4abc9bd 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -40,4 +40,7 @@ ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index f8420e67..88910860 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -555,7 +555,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -891,16 +890,7 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < 4; j++) { - aes_xcbc_ooo->lens[j] = 0xFFFF; - aes_xcbc_ooo->ldata[j].job_in_lane = NULL; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); /* Init AES-CCM auth out-of-order fields */ if (state->features & IMB_FEATURE_GFNI) { diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 2dbbbd44..c9f5d245 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -95,3 +95,24 @@ void ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_AES_XCBC_OOO *p_mgr = (MB_MGR_AES_XCBC_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) + p_mgr->ldata[i].final_block[16] = 0x80; + + if (num_lanes == 4) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; + +} -- GitLab From 9f1982dd26096648319390b83bc9a7bfb2d6a9bb Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 19:24:34 +0000 Subject: [PATCH 165/369] lib: add new API and use it to reset HMAC-SHA1 OOO MGR structures --- lib/avx/mb_mgr_avx.c | 26 ++------------------------ lib/avx2_t1/mb_mgr_avx2.c | 26 ++------------------------ lib/avx512_t1/mb_mgr_avx512.c | 33 +-------------------------------- lib/include/ooo_mgr_reset.h | 4 ++++ lib/sse_t1/mb_mgr_sse.c | 35 ++--------------------------------- lib/x86_64/ooo_mgr_reset.c | 29 +++++++++++++++++++++++++++++ 6 files changed, 40 insertions(+), 113 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 523b37ae..354502a2 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -375,7 +375,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; @@ -446,29 +445,8 @@ reset_ooo_mgrs(IMB_MGR *state) zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5*4] = 0x80; - p[64-2] = 0x02; - p[64-1] = 0xA0; - } + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX_NUM_SHA1_LANES); + /* Init HMAC/SHA224 out-of-order fields */ hmac_sha_224_ooo->lens[0] = 0; hmac_sha_224_ooo->lens[1] = 0; diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index a376e6d7..24c11f2d 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -355,7 +355,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; @@ -426,29 +425,8 @@ reset_ooo_mgrs(IMB_MGR *state) zuc256_eia3_ooo->unused_lane_bitmask = 0xff; /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0; - hmac_sha_1_ooo->lens[5] = 0; - hmac_sha_1_ooo->lens[6] = 0; - hmac_sha_1_ooo->lens[7] = 0; - hmac_sha_1_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX2_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5 * 4] = 0x80; - p[64 - 2] = 0x02; - p[64 - 1] = 0xA0; - } + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); + /* Init HMAC/SHA224 out-of-order fields */ hmac_sha_224_ooo->lens[0] = 0; hmac_sha_224_ooo->lens[1] = 0; diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 25cbc05a..2ec3a43c 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -1034,38 +1034,7 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(zuc256_eia3_ooo->args.digest)); /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0; - hmac_sha_1_ooo->lens[5] = 0; - hmac_sha_1_ooo->lens[6] = 0; - hmac_sha_1_ooo->lens[7] = 0; - hmac_sha_1_ooo->lens[8] = 0; - hmac_sha_1_ooo->lens[9] = 0; - hmac_sha_1_ooo->lens[10] = 0; - hmac_sha_1_ooo->lens[11] = 0; - hmac_sha_1_ooo->lens[12] = 0; - hmac_sha_1_ooo->lens[13] = 0; - hmac_sha_1_ooo->lens[14] = 0; - hmac_sha_1_ooo->lens[15] = 0; - hmac_sha_1_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5 * 4] = 0x80; - p[64 - 2] = 0x02; - p[64 - 1] = 0xA0; - } + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ hmac_sha_224_ooo->lens[0] = 0; diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index c4abc9bd..ddbc036f 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -43,4 +43,8 @@ ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); + + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 88910860..b2396a65 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -549,7 +549,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; @@ -664,42 +663,12 @@ reset_ooo_mgrs(IMB_MGR *state) zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5*4] = 0x80; - p[64-2] = 0x02; - p[64-1] = 0xA0; - } + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, SSE_NUM_SHA1_LANES); #ifdef HASH_USE_SHAEXT if (state->features & IMB_FEATURE_SHANI) { /* Init HMAC/SHA1 NI out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0xFFFF; - hmac_sha_1_ooo->lens[3] = 0xFFFF; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF0100; + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, 2); } #endif /* HASH_USE_SHAEXT */ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index c9f5d245..b38c7ff5 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -114,5 +114,34 @@ void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF76543210; else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; +} + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_SHA_1_OOO *p_mgr = (MB_MGR_HMAC_SHA_1_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[64] = 0x80; + + /* 5 dwords of digest, add padding */ + p_mgr->ldata[i].outer_block[5 * 4] = 0x80; + p_mgr->ldata[i].outer_block[64 - 2] = 0x02; + p_mgr->ldata[i].outer_block[64 - 1] = 0xa0; + } + + IMB_ASSERT(AVX_NUM_SHA1_LANES == SSE_NUM_SHA1_LANES); + if (num_lanes == 2) + p_mgr->unused_lanes = 0xFF0100; /* SHANI */ + else if (num_lanes == AVX_NUM_SHA1_LANES) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == AVX2_NUM_SHA1_LANES) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == AVX512_NUM_SHA1_LANES) + p_mgr->unused_lanes = 0xFEDCBA9876543210; } -- GitLab From b1a670442256966921351536a9ba680c553930be Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 20:53:33 +0000 Subject: [PATCH 166/369] lib: add new API and use it to reset HMAC-SHA224/256 OOO MGR structures --- lib/avx/mb_mgr_avx.c | 54 ++---------------------- lib/avx2_t1/mb_mgr_avx2.c | 57 ++------------------------ lib/avx512_t1/mb_mgr_avx512.c | 77 ++--------------------------------- lib/include/ooo_mgr_reset.h | 5 +++ lib/sse_t1/mb_mgr_sse.c | 74 ++++----------------------------- lib/x86_64/ooo_mgr_reset.c | 60 +++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 243 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 354502a2..3b2d0b63 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -375,8 +375,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; @@ -448,56 +446,12 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX_NUM_SHA256_LANES); /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } - + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ hmac_sha_384_ooo->lens[0] = 0; diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 24c11f2d..94114ead 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -355,8 +355,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; @@ -428,59 +426,12 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0; - hmac_sha_224_ooo->lens[5] = 0; - hmac_sha_224_ooo->lens[6] = 0; - hmac_sha_224_ooo->lens[7] = 0; - hmac_sha_224_ooo->unused_lanes = 0xF76543210; - /* sha256 and sha224 are very similar except for - * digest constants and output size - */ - for (j = 0; j < AVX2_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX2_NUM_SHA256_LANES); /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0; - hmac_sha_256_ooo->lens[5] = 0; - hmac_sha_256_ooo->lens[6] = 0; - hmac_sha_256_ooo->lens[7] = 0; - hmac_sha_256_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX2_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - /* hmac related */ - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX2_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ hmac_sha_384_ooo->lens[0] = 0; diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 2ec3a43c..d4b57db0 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -882,13 +882,9 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -1037,77 +1033,12 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0; - hmac_sha_224_ooo->lens[5] = 0; - hmac_sha_224_ooo->lens[6] = 0; - hmac_sha_224_ooo->lens[7] = 0; - hmac_sha_224_ooo->lens[8] = 0; - hmac_sha_224_ooo->lens[9] = 0; - hmac_sha_224_ooo->lens[10] = 0; - hmac_sha_224_ooo->lens[11] = 0; - hmac_sha_224_ooo->lens[12] = 0; - hmac_sha_224_ooo->lens[13] = 0; - hmac_sha_224_ooo->lens[14] = 0; - hmac_sha_224_ooo->lens[15] = 0; - hmac_sha_224_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_224_ooo->num_lanes_inuse = 0; - /* sha256 and sha224 are very similar except for - * digest constants and output size - */ - for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7 * 4] = 0x80; /* digest 7 words long */ - p[64 - 2] = 0x02; /* length in little endian = 0x02E0 */ - p[64 - 1] = 0xE0; - } + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX512_NUM_SHA256_LANES); /* Init HMAC/SHA256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0; - hmac_sha_256_ooo->lens[5] = 0; - hmac_sha_256_ooo->lens[6] = 0; - hmac_sha_256_ooo->lens[7] = 0; - hmac_sha_256_ooo->lens[8] = 0; - hmac_sha_256_ooo->lens[9] = 0; - hmac_sha_256_ooo->lens[10] = 0; - hmac_sha_256_ooo->lens[11] = 0; - hmac_sha_256_ooo->lens[12] = 0; - hmac_sha_256_ooo->lens[13] = 0; - hmac_sha_256_ooo->lens[14] = 0; - hmac_sha_256_ooo->lens[15] = 0; - hmac_sha_256_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_sha_256_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64 + 7); - /* hmac related */ - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); - p[8 * 4] = 0x80; /* 8 digest words */ - p[64 - 2] = 0x03; /* length */ - p[64 - 1] = 0x00; - } + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX512_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ hmac_sha_384_ooo->lens[0] = 0; diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index ddbc036f..ac707ca4 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -46,5 +46,10 @@ void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes); + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes); #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index b2396a65..6d94e9e6 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -549,8 +549,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; @@ -673,82 +671,24 @@ reset_ooo_mgrs(IMB_MGR *state) #endif /* HASH_USE_SHAEXT */ /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + SSE_NUM_SHA256_LANES); - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7*4] = 0x80; /* digest 7 words long */ - p[64-2] = 0x02; /* length in little endian = 0x02E0 */ - p[64-1] = 0xE0; - } #ifdef HASH_USE_SHAEXT if (state->features & IMB_FEATURE_SHANI) { /* Init HMAC/SHA224 NI out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0xFFFF; - hmac_sha_224_ooo->lens[3] = 0xFFFF; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF0100; + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); } #endif /* HASH_USE_SHAEXT */ /* Init HMAC/SHA_256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */ - p[8*4] = 0x80; - p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits) - * in hex is 0x300 */ - p[64-1] = 0x00; - } + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + SSE_NUM_SHA256_LANES); + #ifdef HASH_USE_SHAEXT if (state->features & IMB_FEATURE_SHANI) { /* Init HMAC/SHA256 NI out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0xFFFF; - hmac_sha_256_ooo->lens[3] = 0xFFFF; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF0100; + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); } #endif /* HASH_USE_SHAEXT */ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index b38c7ff5..bdeb5283 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -145,3 +145,63 @@ void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == AVX512_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[64] = 0x80; + + /* 7 dwords of digest, add padding */ + p_mgr->ldata[i].outer_block[7 * 4] = 0x80; + p_mgr->ldata[i].outer_block[64 - 2] = 0x02; + p_mgr->ldata[i].outer_block[64 - 1] = 0xe0; + } + + IMB_ASSERT(AVX_NUM_SHA256_LANES == SSE_NUM_SHA256_LANES); + + if (num_lanes == 2) + p_mgr->unused_lanes = 0xFF0100; /* SHANI */ + else if (num_lanes == AVX_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == AVX2_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == AVX512_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[64] = 0x80; + + /* 8 dwords of digest, add padding */ + p_mgr->ldata[i].outer_block[8 * 4] = 0x80; + p_mgr->ldata[i].outer_block[64 - 2] = 0x03; + p_mgr->ldata[i].outer_block[64 - 1] = 0x00; + } + + IMB_ASSERT(AVX_NUM_SHA256_LANES == SSE_NUM_SHA256_LANES); + + if (num_lanes == 2) + p_mgr->unused_lanes = 0xFF0100; /* SHANI */ + else if (num_lanes == AVX_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == AVX2_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == AVX512_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From 69e7a781c57f1386939dd24fa02e1f7ba500ff93 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 22:25:54 +0000 Subject: [PATCH 167/369] lib: add new API and use it to reset HMAC-SHA384/512 OOO MGR structures --- lib/avx/mb_mgr_avx.c | 75 ++----------------------- lib/avx2_t1/mb_mgr_avx2.c | 72 ++---------------------- lib/avx512_t1/mb_mgr_avx512.c | 72 ++---------------------- lib/include/ooo_mgr_reset.h | 6 ++ lib/sse_t1/mb_mgr_sse.c | 74 ++---------------------- lib/x86_64/ooo_mgr_reset.c | 102 +++++++++++++++++++++++++++++----- 6 files changed, 109 insertions(+), 292 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 3b2d0b63..a7c72017 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -375,8 +375,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -454,77 +452,12 @@ reset_ooo_mgrs(IMB_MGR *state) AVX_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0xFFFF; - hmac_sha_384_ooo->lens[3] = 0xFFFF; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < AVX_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX_NUM_SHA512_LANES); /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0xFFFF; - hmac_sha_512_ooo->lens[3] = 0xFFFF; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < AVX_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* - * hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } - + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ hmac_md5_ooo->lens[0] = 0; diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 94114ead..42e2d1dc 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -355,8 +355,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -434,74 +432,12 @@ reset_ooo_mgrs(IMB_MGR *state) AVX2_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0; - hmac_sha_384_ooo->lens[3] = 0; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX2_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX2_NUM_SHA512_LANES); /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0; - hmac_sha_512_ooo->lens[3] = 0; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < AVX2_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX2_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ hmac_md5_ooo->lens[0] = 0; diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index d4b57db0..db593cec 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -882,8 +882,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -1041,74 +1039,12 @@ reset_ooo_mgrs(IMB_MGR *state) AVX512_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0; - hmac_sha_384_ooo->lens[3] = 0; - hmac_sha_384_ooo->lens[4] = 0; - hmac_sha_384_ooo->lens[5] = 0; - hmac_sha_384_ooo->lens[6] = 0; - hmac_sha_384_ooo->lens[7] = 0; - hmac_sha_384_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 384 bits, 1408 bits == 0x0580. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX512_NUM_SHA512_LANES); /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0; - hmac_sha_512_ooo->lens[3] = 0; - hmac_sha_512_ooo->lens[4] = 0; - hmac_sha_512_ooo->lens[5] = 0; - hmac_sha_512_ooo->lens[6] = 0; - hmac_sha_512_ooo->lens[7] = 0; - hmac_sha_512_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX512_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - p = ctx->ldata[j].outer_block; - /* special end point because this length is constant */ - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - /* mark the end */ - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; - /* hmac outer block length always of fixed size, - * it is OKey length, a whole message block length, 1024 bits, - * with padding plus the length of the inner digest, - * which is 512 bits, 1536 bits == 0x600. - * The input message block needs to be converted to big endian - * within the sha implementation before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX512_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ hmac_md5_ooo->lens[0] = 0; diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index ac707ca4..b8a84a27 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -52,4 +52,10 @@ void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes); + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 6d94e9e6..ceef8935 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -549,8 +549,6 @@ reset_ooo_mgrs(IMB_MGR *state) unsigned int j; uint8_t *p; size_t size; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -693,76 +691,12 @@ reset_ooo_mgrs(IMB_MGR *state) #endif /* HASH_USE_SHAEXT */ /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0xFFFF; - hmac_sha_384_ooo->lens[3] = 0xFFFF; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 384 bits - * 1408 bits == 0x0580. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + SSE_NUM_SHA512_LANES); /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0xFFFF; - hmac_sha_512_ooo->lens[3] = 0xFFFF; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 512 bits - * 1536 bits == 0x600. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + SSE_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ hmac_md5_ooo->lens[0] = 0; diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index bdeb5283..e085bd35 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -126,12 +126,12 @@ void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { - p_mgr->ldata[i].extra_block[64] = 0x80; + p_mgr->ldata[i].extra_block[IMB_SHA1_BLOCK_SIZE] = 0x80; - /* 5 dwords of digest, add padding */ - p_mgr->ldata[i].outer_block[5 * 4] = 0x80; - p_mgr->ldata[i].outer_block[64 - 2] = 0x02; - p_mgr->ldata[i].outer_block[64 - 1] = 0xa0; + p_mgr->ldata[i].outer_block[IMB_SHA1_DIGEST_SIZE_IN_BYTES] = + 0x80; + p_mgr->ldata[i].outer_block[IMB_SHA1_BLOCK_SIZE - 2] = 0x02; + p_mgr->ldata[i].outer_block[IMB_SHA1_BLOCK_SIZE - 1] = 0xa0; } IMB_ASSERT(AVX_NUM_SHA1_LANES == SSE_NUM_SHA1_LANES); @@ -156,12 +156,12 @@ void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { - p_mgr->ldata[i].extra_block[64] = 0x80; + p_mgr->ldata[i].extra_block[IMB_SHA_256_BLOCK_SIZE] = 0x80; - /* 7 dwords of digest, add padding */ - p_mgr->ldata[i].outer_block[7 * 4] = 0x80; - p_mgr->ldata[i].outer_block[64 - 2] = 0x02; - p_mgr->ldata[i].outer_block[64 - 1] = 0xe0; + p_mgr->ldata[i].outer_block[IMB_SHA224_DIGEST_SIZE_IN_BYTES] = + 0x80; + p_mgr->ldata[i].outer_block[IMB_SHA_256_BLOCK_SIZE - 2] = 0x02; + p_mgr->ldata[i].outer_block[IMB_SHA_256_BLOCK_SIZE - 1] = 0xe0; } IMB_ASSERT(AVX_NUM_SHA256_LANES == SSE_NUM_SHA256_LANES); @@ -186,12 +186,12 @@ void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { - p_mgr->ldata[i].extra_block[64] = 0x80; + p_mgr->ldata[i].extra_block[IMB_SHA_256_BLOCK_SIZE] = 0x80; - /* 8 dwords of digest, add padding */ - p_mgr->ldata[i].outer_block[8 * 4] = 0x80; - p_mgr->ldata[i].outer_block[64 - 2] = 0x03; - p_mgr->ldata[i].outer_block[64 - 1] = 0x00; + p_mgr->ldata[i].outer_block[IMB_SHA256_DIGEST_SIZE_IN_BYTES] = + 0x80; + p_mgr->ldata[i].outer_block[IMB_SHA_256_BLOCK_SIZE - 2] = 0x03; + p_mgr->ldata[i].outer_block[IMB_SHA_256_BLOCK_SIZE - 1] = 0x00; } IMB_ASSERT(AVX_NUM_SHA256_LANES == SSE_NUM_SHA256_LANES); @@ -205,3 +205,75 @@ void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == AVX512_NUM_SHA256_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; + + p_mgr->ldata[i].outer_block[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = + 0x80; + /* + * hmac outer block length always of fixed size, it is OKey + * length, a whole message block length, 1024 bits, with padding + * plus the length of the inner digest, which is 384 bits + * 1408 bits == 0x0580. The input message block needs to be + * converted to big endian within the sha implementation + * before use. + */ + p_mgr->ldata[i].outer_block[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; + p_mgr->ldata[i].outer_block[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; + } + + IMB_ASSERT(AVX_NUM_SHA512_LANES == SSE_NUM_SHA512_LANES); + + if (num_lanes == AVX_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xFF0100; + else if (num_lanes == AVX2_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == AVX512_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xF76543210; +} + +IMB_DLL_LOCAL +void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; + + p_mgr->ldata[i].outer_block[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = + 0x80; + /* + * hmac outer block length always of fixed size, it is OKey + * length, a whole message block length, 1024 bits, with padding + * plus the length of the inner digest, which is 512 bits + * 1536 bits == 0x600. The input message block needs to be + * converted to big endian within the sha implementation + * before use. + */ + p_mgr->ldata[i].outer_block[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; + p_mgr->ldata[i].outer_block[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; + } + + IMB_ASSERT(AVX_NUM_SHA512_LANES == SSE_NUM_SHA512_LANES); + + if (num_lanes == AVX_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xFF0100; + else if (num_lanes == AVX2_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xFF03020100; + else if (num_lanes == AVX512_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xF76543210; +} -- GitLab From 6672c3c863eda40405b87e29e92dcda9629bae5c Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 22:46:07 +0000 Subject: [PATCH 168/369] lib: add new API and use it to reset HMAC-MD5 OOO MGR structures --- lib/avx/mb_mgr_avx.c | 36 +-------------------------------- lib/avx2_t1/mb_mgr_avx2.c | 37 +--------------------------------- lib/avx512_t1/mb_mgr_avx512.c | 38 +---------------------------------- lib/include/ooo_mgr_reset.h | 3 +++ lib/sse_t1/mb_mgr_sse.c | 36 +-------------------------------- lib/x86_64/ooo_mgr_reset.c | 25 +++++++++++++++++++++++ 6 files changed, 32 insertions(+), 143 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index a7c72017..85bd0621 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -373,9 +373,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -460,38 +457,7 @@ reset_ooo_mgrs(IMB_MGR *state) AVX_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0xFFFF; - hmac_md5_ooo->lens[9] = 0xFFFF; - hmac_md5_ooo->lens[10] = 0xFFFF; - hmac_md5_ooo->lens[11] = 0xFFFF; - hmac_md5_ooo->lens[12] = 0xFFFF; - hmac_md5_ooo->lens[13] = 0xFFFF; - hmac_md5_ooo->lens[14] = 0xFFFF; - hmac_md5_ooo->lens[15] = 0xFFFF; - hmac_md5_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < AVX_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX_NUM_MD5_LANES); /* Init AES/XCBC OOO fields */ ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 42e2d1dc..1f1afea3 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -353,9 +353,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; @@ -440,39 +437,7 @@ reset_ooo_mgrs(IMB_MGR *state) AVX2_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0; - hmac_md5_ooo->lens[9] = 0; - hmac_md5_ooo->lens[10] = 0; - hmac_md5_ooo->lens[11] = 0; - hmac_md5_ooo->lens[12] = 0; - hmac_md5_ooo->lens[13] = 0; - hmac_md5_ooo->lens[14] = 0; - hmac_md5_ooo->lens[15] = 0; - hmac_md5_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_md5_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX2_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); /* Init AES/XCBC OOO fields */ ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index db593cec..7edba3fd 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -874,15 +874,12 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - uint8_t *p; - size_t size; MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -904,7 +901,6 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes256_ooo, 8); } - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) * uses same settings as AES CBC. */ @@ -1047,39 +1043,7 @@ reset_ooo_mgrs(IMB_MGR *state) AVX512_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0; - hmac_md5_ooo->lens[9] = 0; - hmac_md5_ooo->lens[10] = 0; - hmac_md5_ooo->lens[11] = 0; - hmac_md5_ooo->lens[12] = 0; - hmac_md5_ooo->lens[13] = 0; - hmac_md5_ooo->lens[14] = 0; - hmac_md5_ooo->lens[15] = 0; - hmac_md5_ooo->unused_lanes = 0xFEDCBA9876543210; - hmac_md5_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4 * 4] = 0x80; - p[64 - 7] = 0x02; - p[64 - 8] = 0x80; - } + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); /* Init AES/XCBC OOO fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index b8a84a27..43281fcc 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -58,4 +58,7 @@ void ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index ceef8935..ed2d2e22 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -547,9 +547,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; @@ -699,38 +696,7 @@ reset_ooo_mgrs(IMB_MGR *state) SSE_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0xFFFF; - hmac_md5_ooo->lens[9] = 0xFFFF; - hmac_md5_ooo->lens[10] = 0xFFFF; - hmac_md5_ooo->lens[11] = 0xFFFF; - hmac_md5_ooo->lens[12] = 0xFFFF; - hmac_md5_ooo->lens[13] = 0xFFFF; - hmac_md5_ooo->lens[14] = 0xFFFF; - hmac_md5_ooo->lens[15] = 0xFFFF; - hmac_md5_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < SSE_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4*4] = 0x80; - p[64-7] = 0x02; - p[64-8] = 0x80; - } + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); /* Init AES/XCBC OOO fields */ ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index e085bd35..b3ca179e 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -277,3 +277,28 @@ void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == AVX512_NUM_SHA512_LANES) p_mgr->unused_lanes = 0xF76543210; } + +IMB_DLL_LOCAL +void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_HMAC_MD5_OOO *p_mgr = (MB_MGR_HMAC_MD5_OOO *) p_ooo_mgr; + unsigned i; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + for (i = 0; i < num_lanes; i++) { + p_mgr->ldata[i].extra_block[64] = 0x80; + + p_mgr->ldata[i].outer_block[4 * 4] = 0x80; + p_mgr->ldata[i].outer_block[64 - 7] = 0x02; + p_mgr->ldata[i].outer_block[64 - 8] = 0x80; + } + + IMB_ASSERT(AVX_NUM_MD5_LANES == SSE_NUM_MD5_LANES); + + if (num_lanes == AVX_NUM_MD5_LANES) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == AVX2_NUM_MD5_LANES) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From 4466ced368fb5d4b8563ac835f25223747c3b0f2 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 22:59:12 +0000 Subject: [PATCH 169/369] lib: add new API and use it to reset ZUC OOO MGR structures --- lib/avx/mb_mgr_avx.c | 51 +++------------------------------ lib/avx2_t1/mb_mgr_avx2.c | 51 +++------------------------------ lib/avx512_t1/mb_mgr_avx512.c | 54 +++-------------------------------- lib/include/ooo_mgr_reset.h | 3 ++ lib/sse_t1/mb_mgr_sse.c | 51 +++------------------------------ lib/x86_64/ooo_mgr_reset.c | 20 +++++++++++++ 6 files changed, 39 insertions(+), 191 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 85bd0621..ecd2ddaf 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -373,10 +373,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ @@ -393,49 +389,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); /* Init HMAC/SHA1 out-of-order fields */ ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX_NUM_SHA1_LANES); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 1f1afea3..de57b244 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -353,10 +353,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; /* Init AES out-of-order fields */ @@ -373,49 +369,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xF76543210; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xF76543210; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xF76543210; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0xff; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xF76543210; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0xff; + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 8); /* Init HMAC/SHA1 out-of-order fields */ ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 7edba3fd..a026e085 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -880,10 +880,6 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; @@ -976,52 +972,10 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(docsis_des_dec_ooo->args)); /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0xffff; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0xffff; - memset(zuc_eia3_ooo->args.digest, 0, sizeof(zuc_eia3_ooo->args.digest)); - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0xffff; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFEDCBA9876543210; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0xffff; - memset(zuc256_eia3_ooo->args.digest, 0, - sizeof(zuc256_eia3_ooo->args.digest)); + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 16); /* Init HMAC/SHA1 out-of-order fields */ ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 43281fcc..b4d64250 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -61,4 +61,7 @@ void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index ed2d2e22..b9fbe781 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -547,10 +547,6 @@ static void reset_ooo_mgrs(IMB_MGR *state) { unsigned int j; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; @@ -611,49 +607,10 @@ reset_ooo_mgrs(IMB_MGR *state) } /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); /* Init HMAC/SHA1 out-of-order fields */ ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, SSE_NUM_SHA1_LANES); diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index b3ca179e..3c8e6ca2 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -302,3 +302,23 @@ void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == AVX2_NUM_MD5_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_ZUC_OOO *p_mgr = (MB_MGR_ZUC_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + if (num_lanes == 4) { + p_mgr->unused_lanes = 0xFF03020100; + p_mgr->unused_lane_bitmask = 0x0f; + } else if (num_lanes == 8) { + p_mgr->unused_lanes = 0xF76543210; + p_mgr->unused_lane_bitmask = 0xff; + } else if (num_lanes == 16) { + p_mgr->unused_lanes = 0xFEDCBA9876543210; + p_mgr->unused_lane_bitmask = 0xffff; + } +} -- GitLab From 6eba8bda714fd24db31bdd69ef24cfe366164570 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 23:08:27 +0000 Subject: [PATCH 170/369] lib: add new API and use it to reset SHA1 OOO MGR structures --- lib/avx/mb_mgr_avx.c | 12 +----------- lib/avx2_t1/mb_mgr_avx2.c | 10 +--------- lib/avx512_t1/mb_mgr_avx512.c | 10 ++-------- lib/include/ooo_mgr_reset.h | 3 +++ .../mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 4 ++-- lib/sse_t1/mb_mgr_sse.c | 11 +---------- lib/{sse => sse_t1}/snow3g_uea2_by4_sse.asm | 0 lib/x86_64/ooo_mgr_reset.c | 15 +++++++++++++++ 8 files changed, 25 insertions(+), 40 deletions(-) rename lib/{sse => sse_t1}/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm (99%) rename lib/{sse => sse_t1}/snow3g_uea2_by4_sse.asm (100%) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index ecd2ddaf..ba82fb5f 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -372,9 +372,6 @@ submit_job_aes_cntr_bit_avx(IMB_JOB *job) static void reset_ooo_mgrs(IMB_MGR *state) { - unsigned int j; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 8); ooo_mgr_aes_reset(state->aes192_ooo, 8); @@ -431,14 +428,7 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ - sha_1_ooo->lens[0] = 0; - sha_1_ooo->lens[1] = 0; - sha_1_ooo->lens[2] = 0; - sha_1_ooo->lens[3] = 0; - sha_1_ooo->unused_lanes = 0xF3210; - sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX_NUM_SHA1_LANES; j++) - sha_1_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha1_reset(state->sha_1_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index de57b244..b0731f48 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -352,9 +352,6 @@ submit_job_aes_gcm_enc_avx2(IMB_MGR *state, IMB_JOB *job) static void reset_ooo_mgrs(IMB_MGR *state) { - unsigned int j; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 8); ooo_mgr_aes_reset(state->aes192_ooo, 8); @@ -411,12 +408,7 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ - sha_1_ooo->unused_lanes = 0xF76543210; - sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX2_NUM_SHA1_LANES; j++){ - sha_1_ooo->lens[j] = 0; - sha_1_ooo->ldata[j].job_in_lane = NULL; - } + ooo_mgr_sha1_reset(state->sha_1_ooo, 8); } IMB_DLL_LOCAL void diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index a026e085..12815895 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -873,7 +873,6 @@ static IMB_JOB * static void reset_ooo_mgrs(IMB_MGR *state) { - unsigned int j; MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; @@ -882,7 +881,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + unsigned j; /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { @@ -1058,12 +1057,7 @@ reset_ooo_mgrs(IMB_MGR *state) sizeof(snow3g_uia2_ooo->lens)); /* Init SHA1 out-of-order fields */ - sha_1_ooo->unused_lanes = 0xFEDCBA9876543210; - sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < AVX512_NUM_SHA1_LANES; j++){ - sha_1_ooo->lens[j] = 0; - sha_1_ooo->ldata[j].job_in_lane = NULL; - } + ooo_mgr_sha1_reset(state->sha_1_ooo, 16); } IMB_DLL_LOCAL void diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index b4d64250..e0017096 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -64,4 +64,7 @@ void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm similarity index 99% rename from lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm rename to lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index 38392efe..05f031f6 100644 --- a/lib/sse/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -30,7 +30,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/cet.inc" -%include "sse/snow3g_uea2_by4_sse.asm" +%include "sse_t1/snow3g_uea2_by4_sse.asm" %define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_sse %define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_sse @@ -423,4 +423,4 @@ FLUSH_JOB_SNOW3G_UEA2: xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, \ xmm14, xmm15 ret -mksection stack-noexec \ No newline at end of file +mksection stack-noexec diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index b9fbe781..de8ebbae 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -546,8 +546,6 @@ static IMB_JOB * static void reset_ooo_mgrs(IMB_MGR *state) { - unsigned int j; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; /* Init AES out-of-order fields */ @@ -704,14 +702,7 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); /* Init SHA1 out-of-order fields */ - sha_1_ooo->lens[0] = 0; - sha_1_ooo->lens[1] = 0; - sha_1_ooo->lens[2] = 0; - sha_1_ooo->lens[3] = 0; - sha_1_ooo->unused_lanes = 0xF3210; - sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < SSE_NUM_SHA1_LANES; j++) - sha_1_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha1_reset(state->sha_1_ooo, 4); /* Init SNOW3G-UEA out-of-order fields */ memset(&snow3g_uea2_ooo->args, 0, sizeof(snow3g_uea2_ooo->args)); diff --git a/lib/sse/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm similarity index 100% rename from lib/sse/snow3g_uea2_by4_sse.asm rename to lib/sse_t1/snow3g_uea2_by4_sse.asm diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 3c8e6ca2..b1d54666 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -322,3 +322,18 @@ void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lane_bitmask = 0xffff; } } + +IMB_DLL_LOCAL +void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_SHA_1_OOO *p_mgr = (MB_MGR_SHA_1_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + + if (num_lanes == 4) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == 8) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From 535bbf1126609ab76f245ce36eafc2bc12f33cb4 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 30 May 2022 23:16:25 +0000 Subject: [PATCH 171/369] lib: add new API and use it to reset DES, 3DES and DOCSIS DES OOO MGR structures --- lib/avx512_t1/mb_mgr_avx512.c | 68 ++++------------------------------- lib/include/ooo_mgr_reset.h | 3 ++ lib/x86_64/ooo_mgr_reset.c | 11 ++++++ 3 files changed, 20 insertions(+), 62 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 12815895..d890c95d 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -873,15 +873,8 @@ static IMB_JOB * static void reset_ooo_mgrs(IMB_MGR *state) { - MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; - MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; - MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; - MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; - MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; - MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - unsigned j; /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { @@ -914,61 +907,12 @@ reset_ooo_mgrs(IMB_MGR *state) } /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ - /* - separate DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des_enc_ooo->lens[j] = 0; - des_enc_ooo->job_in_lane[j] = NULL; - } - des_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - des_enc_ooo->num_lanes_inuse = 0; - memset(&des_enc_ooo->args, 0, sizeof(des_enc_ooo->args)); - - /* - separate DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des_dec_ooo->lens[j] = 0; - des_dec_ooo->job_in_lane[j] = NULL; - } - des_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - des_dec_ooo->num_lanes_inuse = 0; - memset(&des_dec_ooo->args, 0, sizeof(des_dec_ooo->args)); - - /* - separate 3DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des3_enc_ooo->lens[j] = 0; - des3_enc_ooo->job_in_lane[j] = NULL; - } - des3_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - des3_enc_ooo->num_lanes_inuse = 0; - memset(&des3_enc_ooo->args, 0, sizeof(des3_enc_ooo->args)); - - /* - separate 3DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - des3_dec_ooo->lens[j] = 0; - des3_dec_ooo->job_in_lane[j] = NULL; - } - des3_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - des3_dec_ooo->num_lanes_inuse = 0; - memset(&des3_dec_ooo->args, 0, sizeof(des3_dec_ooo->args)); - - /* - separate DOCSIS DES OOO for encryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - docsis_des_enc_ooo->lens[j] = 0; - docsis_des_enc_ooo->job_in_lane[j] = NULL; - } - docsis_des_enc_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis_des_enc_ooo->num_lanes_inuse = 0; - memset(&docsis_des_enc_ooo->args, 0, - sizeof(docsis_des_enc_ooo->args)); - - /* - separate DES OOO for decryption */ - for (j = 0; j < AVX512_NUM_DES_LANES; j++) { - docsis_des_dec_ooo->lens[j] = 0; - docsis_des_dec_ooo->job_in_lane[j] = NULL; - } - docsis_des_dec_ooo->unused_lanes = 0xFEDCBA9876543210; - docsis_des_dec_ooo->num_lanes_inuse = 0; - memset(&docsis_des_dec_ooo->args, 0, - sizeof(docsis_des_dec_ooo->args)); + ooo_mgr_des_reset(state->des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_dec_ooo, AVX512_NUM_DES_LANES); /* Init ZUC out-of-order fields */ ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 16); diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index e0017096..d0990855 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -67,4 +67,7 @@ void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index b1d54666..99282557 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -337,3 +337,14 @@ void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_DES_OOO *p_mgr = (MB_MGR_DES_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + + if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From 307886dcd26d6fca20cc0b4dc5297eb41602afc3 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 31 May 2022 08:33:13 +0000 Subject: [PATCH 172/369] lib: add new API and use it to reset SNOW3G OOO MGR structures --- lib/avx512_t1/mb_mgr_avx512.c | 27 ++------------------------- lib/include/ooo_mgr_reset.h | 3 +++ lib/sse_t1/mb_mgr_sse.c | 27 +-------------------------- lib/x86_64/ooo_mgr_reset.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 51 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index d890c95d..46f2fe65 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -873,9 +873,6 @@ static IMB_JOB * static void reset_ooo_mgrs(IMB_MGR *state) { - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; - /* Init AES out-of-order fields */ if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { /* init 16 lanes */ @@ -977,28 +974,8 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SNOW3G out-of-order fields */ - memset(&snow3g_uea2_ooo->args, 0, - sizeof(snow3g_uea2_ooo->args)); - memset(snow3g_uea2_ooo->job_in_lane, 0, - sizeof(snow3g_uea2_ooo->job_in_lane)); - snow3g_uea2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uea2_ooo->num_lanes_inuse = 0; - snow3g_uea2_ooo->init_mask = 0; - snow3g_uea2_ooo->init_done = 0; - memset(snow3g_uea2_ooo->lens, 0xff, - sizeof(snow3g_uea2_ooo->lens)); - memset(&snow3g_uia2_ooo->args, 0, - sizeof(snow3g_uia2_ooo->args)); - memset(snow3g_uia2_ooo->job_in_lane, 0, - sizeof(snow3g_uia2_ooo->job_in_lane)); - memset(snow3g_uia2_ooo->ks, 0, - sizeof(snow3g_uia2_ooo->ks)); - snow3g_uia2_ooo->unused_lanes = 0xFEDCBA9876543210; - snow3g_uia2_ooo->num_lanes_inuse = 0; - snow3g_uia2_ooo->init_mask = 0; - snow3g_uia2_ooo->init_done = 0; - memset(snow3g_uia2_ooo->lens, 0, - sizeof(snow3g_uia2_ooo->lens)); + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 16); + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 16); /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, 16); diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index d0990855..0539768d 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -70,4 +70,7 @@ void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_snow3g_reset(void *p_ooo_mgr, const unsigned num_lanes); + #endif /* OOO_MGR_RESET_H */ diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index de8ebbae..51c355ce 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -546,8 +546,6 @@ static IMB_JOB * static void reset_ooo_mgrs(IMB_MGR *state) { - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - /* Init AES out-of-order fields */ if (state->features & IMB_FEATURE_GFNI) { ooo_mgr_aes_reset(state->aes128_ooo, 8); @@ -705,30 +703,7 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_sha1_reset(state->sha_1_ooo, 4); /* Init SNOW3G-UEA out-of-order fields */ - memset(&snow3g_uea2_ooo->args, 0, sizeof(snow3g_uea2_ooo->args)); - memset(snow3g_uea2_ooo->job_in_lane, 0, - sizeof(snow3g_uea2_ooo->job_in_lane)); - snow3g_uea2_ooo->unused_lanes = 0x3210; - snow3g_uea2_ooo->num_lanes_inuse = 0; - /* - * lens[0:3] are used to indicate outstanding bytes after rounding - * up length to dwords - * lens[4] is used for common min length for all lanes in dwords - * lens[8:11] are used for keeping lengths rounded up to dwords - * lens [5:7], [12:15] are unused - */ - snow3g_uea2_ooo->lens[0]=0; - snow3g_uea2_ooo->lens[1]=0; - snow3g_uea2_ooo->lens[2]=0; - snow3g_uea2_ooo->lens[3]=0; - /* - * Dwords rounded lengths need to be initialized to max length not to - * interfere when searching for minimum length. - */ - snow3g_uea2_ooo->lens[8]=0xffffffff; - snow3g_uea2_ooo->lens[9]=0xffffffff; - snow3g_uea2_ooo->lens[10]=0xffffffff; - snow3g_uea2_ooo->lens[11]=0xffffffff; + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 99282557..1838da4a 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -348,3 +348,33 @@ void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes) if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } + +IMB_DLL_LOCAL +void ooo_mgr_snow3g_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_SNOW3G_OOO *p_mgr = (MB_MGR_SNOW3G_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); + + if (num_lanes == 4) { + /* + * lens[0:3] indicate outstanding bytes after + * rounding up length to dwords + * - initialize to 0 + * lens[4] common min length for all lanes in dwords + * - initialize to 0 + * lens[8:11] keep lengths rounded up to dwords + * - initialize to UINT32_MAX not to interfere + * when searching for minimum length + * lens[5:7] unused + * lens[12:15] unused + */ + p_mgr->lens[8] = 0xffffffff; + p_mgr->lens[9] = 0xffffffff; + p_mgr->lens[10] = 0xffffffff; + p_mgr->lens[11] = 0xffffffff; + p_mgr->unused_lanes = 0x3210; + } else if (num_lanes == 16) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} -- GitLab From a0495998917aa5d387506cac7986014e7f23d390 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 9 Jun 2022 12:45:11 +0000 Subject: [PATCH 173/369] lib: [x86_64] added new arch_x86_64.h header file Scalar poly1305, DES and 3DES function prototypes moved to it --- lib/include/arch_avx_type1.h | 4 -- lib/include/arch_noaesni.h | 2 - lib/include/arch_sse_type1.h | 2 - lib/include/arch_x86_64.h | 124 ++++++++++++++++++++++++++++++++ lib/include/chacha20_poly1305.h | 6 +- lib/include/des.h | 79 +------------------- 6 files changed, 126 insertions(+), 91 deletions(-) create mode 100644 lib/include/arch_x86_64.h diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index 2ecd15ba..7c4e888f 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -158,8 +158,6 @@ IMB_JOB *flush_job_aes128_cbcs_1_9_enc_avx(MB_MGR_AES_OOO *state); IMB_JOB *submit_job_chacha20_enc_dec_avx(IMB_JOB *job); -void *poly1305_mac_scalar(IMB_JOB *job); - IMB_JOB *snow_v_avx(IMB_JOB *job); IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); @@ -226,5 +224,3 @@ uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); #endif /* IMB_ASM_AVX_T1_H */ - - diff --git a/lib/include/arch_noaesni.h b/lib/include/arch_noaesni.h index 6ab55351..f1a1100d 100644 --- a/lib/include/arch_noaesni.h +++ b/lib/include/arch_noaesni.h @@ -163,8 +163,6 @@ IMB_JOB *submit_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_aes128_cbcs_1_9_enc_sse_no_aesni(MB_MGR_AES_OOO *state); -void *poly1305_mac_scalar(IMB_JOB *job); - IMB_JOB *snow_v_sse_no_aesni(IMB_JOB *job); IMB_JOB *snow_v_aead_init_sse_no_aesni(IMB_JOB *job); diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index f2de7321..9779b831 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -217,8 +217,6 @@ IMB_JOB *submit_job_chacha20_enc_dec_sse(IMB_JOB *job); IMB_JOB *snow_v_sse(IMB_JOB *job); IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); -void *poly1305_mac_scalar(IMB_JOB *job); - IMB_JOB *submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state); diff --git a/lib/include/arch_x86_64.h b/lib/include/arch_x86_64.h new file mode 100644 index 00000000..87b9a8f2 --- /dev/null +++ b/lib/include/arch_x86_64.h @@ -0,0 +1,124 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef IMB_ARCH_X86_64_H +#define IMB_ARCH_X86_64_H + +#include "intel-ipsec-mb.h" + +IMB_DLL_LOCAL void *poly1305_mac_scalar(IMB_JOB *job); +IMB_DLL_LOCAL void +poly1305_aead_update_scalar(const void *msg, const uint64_t msg_len, + void *hash, const void *key); +IMB_DLL_LOCAL void +poly1305_aead_complete_scalar(const void *hash, const void *key, void *tag); + +/** + * @brief DES CBC encryption + * + * @param input source buffer with plain text + * @param output destination buffer for cipher text + * @param size number of bytes to encrypt (multiple of 8) + * @param ks pointer to key schedule structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void des_enc_cbc_basic(const void *input, void *output, const int size, + const uint64_t *ks, const uint64_t *ivec); + +/** + * @brief DES CBC decryption + * + * @param input source buffer with cipher text + * @param output destination buffer for plain text + * @param size number of bytes to decrypt (multiple of 8) + * @param ks pointer to key schedule structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void des_dec_cbc_basic(const void *input, void *output, const int size, + const uint64_t *ks, const uint64_t *ivec); + +/** + * @brief 3DES CBC encryption + * + * @param input source buffer with plain text + * @param output destination buffer for cipher text + * @param size number of bytes to encrypt (multiple of 8) + * @param ks1 pointer to key schedule 1 structure + * @param ks2 pointer to key schedule 2 structure + * @param ks3 pointer to key schedule 3 structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void des3_enc_cbc_basic(const void *input, void *output, const int size, + const uint64_t *ks1, const uint64_t *ks2, + const uint64_t *ks3, const uint64_t *ivec); + +/** + * @brief 3DES CBC decryption + * + * @param input source buffer with cipher text + * @param output destination buffer for plain text + * @param size number of bytes to decrypt (multiple of 8) + * @param ks1 pointer to key schedule 1 structure + * @param ks2 pointer to key schedule 2 structure + * @param ks3 pointer to key schedule 3 structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void des3_dec_cbc_basic(const void *input, void *output, const int size, + const uint64_t *ks1, const uint64_t *ks2, + const uint64_t *ks3, const uint64_t *ivec); + +/** + * @brief DOCSIS DES encryption + * + * @param input source buffer with plain text + * @param output destination buffer for cipher text + * @param size number of bytes to encrypt + * @param ks pointer to key schedule structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void docsis_des_enc_basic(const void *input, void *output, const int size, + const uint64_t *ks, const uint64_t *ivec); + +/** + * @brief DOCSIS DES decryption + * + * @param input source buffer with cipher text + * @param output destination buffer for plain text + * @param size number of bytes to decrypt + * @param ks pointer to key schedule structure + * @param ivec pointer to initialization vector + */ +IMB_DLL_LOCAL +void docsis_des_dec_basic(const void *input, void *output, const int size, + const uint64_t *ks, const uint64_t *ivec); + +#endif /* IMB_ARCH_X86_64_H */ diff --git a/lib/include/chacha20_poly1305.h b/lib/include/chacha20_poly1305.h index 511985c9..ced3872a 100644 --- a/lib/include/chacha20_poly1305.h +++ b/lib/include/chacha20_poly1305.h @@ -29,6 +29,7 @@ #define IMB_CHACHA20POLY1305_H #include "intel-ipsec-mb.h" +#include "include/arch_x86_64.h" /* new internal API's */ IMB_JOB *aead_chacha20_poly1305_sse(IMB_MGR *mgr, IMB_JOB *job); @@ -68,11 +69,6 @@ IMB_JOB *submit_job_chacha20_poly_enc_sse(IMB_JOB *, void *poly_key); void poly1305_key_gen_sse(const void *key, const void *iv, void *poly_key); void poly1305_key_gen_avx(const void *key, const void *iv, void *poly_key); -void poly1305_aead_update_scalar(const void *msg, const uint64_t msg_len, - void *hash, const void *key); -void poly1305_aead_complete_scalar(const void *hash, const void *key, - void *tag); - void poly1305_aead_update_avx512(const void *msg, const uint64_t msg_len, void *hash, const void *key); void poly1305_aead_complete_avx512(const void *hash, const void *key, diff --git a/lib/include/des.h b/lib/include/des.h index f55509dd..f8944b3a 100644 --- a/lib/include/des.h +++ b/lib/include/des.h @@ -29,84 +29,7 @@ #define IMB_DES_H #include - -/** - * @brief DES CBC encryption - * - * @param input source buffer with plain text - * @param output destination buffer for cipher text - * @param size number of bytes to encrypt (multiple of 8) - * @param ks pointer to key schedule structure - * @param ivec pointer to initialization vector - */ -void des_enc_cbc_basic(const void *input, void *output, const int size, - const uint64_t *ks, const uint64_t *ivec); - -/** - * @brief DES CBC decryption - * - * @param input source buffer with cipher text - * @param output destination buffer for plain text - * @param size number of bytes to decrypt (multiple of 8) - * @param ks pointer to key schedule structure - * @param ivec pointer to initialization vector - */ -void des_dec_cbc_basic(const void *input, void *output, const int size, - const uint64_t *ks, const uint64_t *ivec); - -/** - * @brief 3DES CBC encryption - * - * @param input source buffer with plain text - * @param output destination buffer for cipher text - * @param size number of bytes to encrypt (multiple of 8) - * @param ks1 pointer to key schedule 1 structure - * @param ks2 pointer to key schedule 2 structure - * @param ks3 pointer to key schedule 3 structure - * @param ivec pointer to initialization vector - */ -void des3_enc_cbc_basic(const void *input, void *output, const int size, - const uint64_t *ks1, const uint64_t *ks2, - const uint64_t *ks3, const uint64_t *ivec); - -/** - * @brief 3DES CBC decryption - * - * @param input source buffer with cipher text - * @param output destination buffer for plain text - * @param size number of bytes to decrypt (multiple of 8) - * @param ks1 pointer to key schedule 1 structure - * @param ks2 pointer to key schedule 2 structure - * @param ks3 pointer to key schedule 3 structure - * @param ivec pointer to initialization vector - */ -void des3_dec_cbc_basic(const void *input, void *output, const int size, - const uint64_t *ks1, const uint64_t *ks2, - const uint64_t *ks3, const uint64_t *ivec); - -/** - * @brief DOCSIS DES encryption - * - * @param input source buffer with plain text - * @param output destination buffer for cipher text - * @param size number of bytes to encrypt - * @param ks pointer to key schedule structure - * @param ivec pointer to initialization vector - */ -void docsis_des_enc_basic(const void *input, void *output, const int size, - const uint64_t *ks, const uint64_t *ivec); - -/** - * @brief DOCSIS DES decryption - * - * @param input source buffer with cipher text - * @param output destination buffer for plain text - * @param size number of bytes to decrypt - * @param ks pointer to key schedule structure - * @param ivec pointer to initialization vector - */ -void docsis_des_dec_basic(const void *input, void *output, const int size, - const uint64_t *ks, const uint64_t *ivec); +#include "arch_x86_64.h" /* ========================================================================= */ /* DES and 3DES inline function for use in mb_mgr_code.h */ -- GitLab From c96d56f9a464058177dd5fd8c5785aeb1323036f Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 9 Jun 2022 11:11:40 +0000 Subject: [PATCH 174/369] lib: fix Doxygen comment --- lib/intel-ipsec-mb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index ff44e7ab..beefcc23 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1809,8 +1809,8 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); * KASUMI F9 key schedule init function. * * @param[in] _mgr Pointer to multi-buffer structure - * @param[in] _exp_key Integrity key (expected in LE format) - * @param[out] _ctx Key schedule context to be initialised + * @param[in] _key Integrity key (expected in LE format) + * @param[out] _exp_key Key schedule context to be initialised * @return 0 on success, -1 on failure * ******************************************************************************/ -- GitLab From 0ce55861dfe7786e03a8b77bf62c42d5d9bf5f2e Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 9 Jun 2022 09:05:00 +0000 Subject: [PATCH 175/369] lib: document SHAx functions --- lib/intel-ipsec-mb.h | 368 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 316 insertions(+), 52 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index beefcc23..b254352c 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1422,26 +1422,111 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); ((_mgr)->des_key_sched((_exp_key), (_key))) /* Hash API's */ + +/** + * Authenticate 64-byte data buffer with SHA1. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 64-byte data buffer + * @param[out] _tag Digest output (20 bytes) + */ #define IMB_SHA1_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->sha1_one_block((_src), (_tag))) + +/** + * Authenticate variable sized data with SHA1. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src Data buffer + * @param[in] _length Length of data in bytes for authentication. + * @param[out] _tag Digest output (20 bytes) + */ #define IMB_SHA1(_mgr, _src, _length, _tag) \ ((_mgr)->sha1((_src), (_length), (_tag))) +/** + * Authenticate 64-byte data buffer with SHA224. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 64-byte data buffer + * @param[out] _tag Digest output (28 bytes) + */ #define IMB_SHA224_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->sha224_one_block((_src), (_tag))) + +/** + * Authenticate variable sized data with SHA224. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src Data buffer + * @param[in] _length Length of data in bytes for authentication. + * @param[out] _tag Digest output (28 bytes) + */ #define IMB_SHA224(_mgr, _src, _length, _tag) \ ((_mgr)->sha224((_src), (_length), (_tag))) +/** + * Authenticate 64-byte data buffer with SHA256. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 64-byte data buffer + * @param[out] _tag Digest output (32 bytes) + */ #define IMB_SHA256_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->sha256_one_block((_src), (_tag))) +/** + * Authenticate variable sized data with SHA256. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src Data buffer + * @param[in] _length Length of data in bytes for authentication. + * @param[out] _tag Digest output (32 bytes) + */ #define IMB_SHA256(_mgr, _src, _length, _tag) \ ((_mgr)->sha256((_src), (_length), (_tag))) +/** + * Authenticate 128-byte data buffer with SHA384. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 128-byte data buffer + * @param[out] _tag Digest output (48 bytes) + */ #define IMB_SHA384_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->sha384_one_block((_src), (_tag))) +/** + * Authenticate variable sized data with SHA384. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src Data buffer + * @param[in] _length Length of data in bytes for authentication. + * @param[out] _tag Digest output (48 bytes) + */ #define IMB_SHA384(_mgr, _src, _length, _tag) \ ((_mgr)->sha384((_src), (_length), (_tag))) +/** + * Authenticate 128-byte data buffer with SHA512. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 128-byte data buffer + * @param[out] _tag Digest output (64 bytes) + */ #define IMB_SHA512_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->sha512_one_block((_src), (_tag))) +/** + * Authenticate variable sized data with SHA512. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src Data buffer + * @param[in] _length Length of data in bytes for authentication. + * @param[out] _tag Digest output (20 bytes) + */ #define IMB_SHA512(_mgr, _src, _length, _tag) \ ((_mgr)->sha512((_src), (_length), (_tag))) +/** + * Authenticate 64-byte data buffer with MD5. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _src 64-byte data buffer + * @param[out] _tag Digest output (16 bytes) + */ #define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->md5_one_block((_src), (_tag))) @@ -2179,23 +2264,253 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); IMB_DLL_EXPORT int des_key_schedule(uint64_t *ks, const void *key); -/* SSE */ +/** + * Authenticate variable sized data with SHA1. + * + * @param[in] data Data buffer + * @param[in] length Length of data in bytes for authentication. + * @param[out] digest Digest output (20 bytes) + */ IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length, void *digest); + +/** + * @copydoc sha1_sse + */ +IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha1_sse + */ +IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha1_sse + */ +IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, + void *digest); + +/** + * Authenticate 64-byte data buffer with SHA1. + * + * @param[in] data 64-byte data buffer + * @param[out] digest Digest output (20 bytes) + */ IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest); +/** + * @copydoc sha1_one_block_sse + */ +IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest); +/** + * @copydoc sha1_one_block_sse + */ +IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest); +/** + * @copydoc sha1_one_block_sse + */ +IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest); + +/** + * Authenticate variable sized data with SHA224. + * + * @param[in] data Data buffer + * @param[in] length Length of data in bytes for authentication. + * @param[out] digest Digest output (28 bytes) + */ IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length, void *digest); +/** + * @copydoc sha224_sse + */ +IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha224_sse + */ +IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha224_sse + */ +IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, + void *digest); + +/** + * Authenticate 64-byte data buffer with SHA224. + * + * @param[in] data 64-byte data buffer + * @param[out] digest Digest output (28 bytes) + */ IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest); +/** + * @copydoc sha224_one_block_sse + */ +IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest); +/** + * @copydoc sha224_one_block_sse + */ +IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest); +/** + * @copydoc sha224_one_block_sse + */ +IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest); + +/** + * Authenticate variable sized data with SHA256. + * + * @param[in] data Data buffer + * @param[in] length Length of data in bytes for authentication. + * @param[out] digest Digest output (32 bytes) + */ IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length, void *digest); +/** + * @copydoc sha256_sse + */ +IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha256_sse + */ +IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha256_sse + */ +IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, + void *digest); + +/** + * Authenticate 64-byte data buffer with SHA256. + * + * @param[in] data 64-byte data buffer + * @param[out] digest Digest output (32 bytes) + */ IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest); +/** + * @copydoc sha256_one_block_sse + */ +IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest); +/** + * @copydoc sha256_one_block_sse + */ +IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest); +/** + * @copydoc sha256_one_block_sse + */ +IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest); + +/** + * Authenticate variable sized data with SHA384. + * + * @param[in] data Data buffer + * @param[in] length Length of data in bytes for authentication. + * @param[out] digest Digest output (48 bytes) + */ IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length, void *digest); +/** + * @copydoc sha384_sse + */ +IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha384_sse + */ +IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha384_sse + */ +IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, + void *digest); + +/** + * Authenticate 128-byte data buffer with SHA384. + * + * @param[in] data 64-byte data buffer + * @param[out] digest Digest output (48 bytes) + */ IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest); +/** + * @copydoc sha384_one_block_sse + */ +IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest); +/** + * @copydoc sha384_one_block_sse + */ +IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest); +/** + * @copydoc sha384_one_block_sse + */ +IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest); + +/** + * Authenticate variable sized data with SHA512. + * + * @param[in] data Data buffer + * @param[in] length Length of data in bytes for authentication. + * @param[out] digest Digest output (64 bytes) + */ IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length, void *digest); +/** + * @copydoc sha512_sse + */ +IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha512_sse + */ +IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, + void *digest); +/** + * @copydoc sha512_sse + */ +IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, + void *digest); + +/** + * Authenticate 64-byte data buffer with SHA512. + * + * @param[in] data 128-byte data buffer + * @param[out] digest Digest output (64 bytes) + */ IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest); +/** + * @copydoc sha512_one_block_sse + */ +IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest); +/** + * @copydoc sha512_one_block_sse + */ +IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest); +/** + * @copydoc sha512_one_block_sse + */ +IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest); + +/** + * Authenticate 64-byte data buffer with MD5. + * + * @param[in] data 64-byte data buffer + * @param[out] digest Digest output (16 bytes) + */ IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest); +/** + * @copydoc md5_one_block_sse + */ +IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest); +/** + * @copydoc md5_one_block_sse + */ +IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest); +/** + * @copydoc md5_one_block_sse + */ +IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest); + + IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys); IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys, @@ -2215,23 +2530,6 @@ IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1, IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -/* AVX */ -IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest); -IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest); -IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest); -IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest); -IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest); -IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest); IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys); IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, @@ -2251,23 +2549,6 @@ IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -/* AVX2 */ -IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest); -IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest); -IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest); -IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest); -IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest); -IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest); IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys); IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, @@ -2288,23 +2569,6 @@ IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -/* AVX512 */ -IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest); -IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest); -IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest); -IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest); -IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, - void *digest); -IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest); -IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest); IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys); IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, -- GitLab From 2bf193470e6c7c42306f3a544cb12391338f4fd1 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 9 Jun 2022 11:04:16 +0000 Subject: [PATCH 176/369] lib: document AES key expansion functions --- lib/intel-ipsec-mb.h | 285 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 234 insertions(+), 51 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index b254352c..4ac2ac26 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1402,19 +1402,69 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash))) /* Key expansion and generation API's */ + +/** + * Generate encryption/decryption AES-128 expansion keys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _key AES-128 key + * @param[out] _enc_exp_key AES-128 encryption expansion key + * @param[out] _dec_exp_key AES-128 decryption expansion key + */ #define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \ ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key))) +/** + * Generate encryption/decryption AES-192 expansion keys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _key AES-192 key + * @param[out] _enc_exp_key AES-192 encryption expansion key + * @param[out] _dec_exp_key AES-192 decryption expansion key + */ #define IMB_AES_KEYEXP_192(_mgr, _key, _enc_exp_key, _dec_exp_key) \ ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key))) +/** + * Generate encryption/decryption AES-256 expansion keys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _key AES-256 key + * @param[out] _enc_exp_key AES-256 encryption expansion key + * @param[out] _dec_exp_key AES-256 decryption expansion key + */ #define IMB_AES_KEYEXP_256(_mgr, _key, _enc_exp_key, _dec_exp_key) \ ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key))) +/** + * Generate AES-128-CMAC subkeys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _exp_key Input expanded AES-128-CMAC key + * @param[out] _key1 Subkey 1 + * @param[out] _key2 Subkey 2 + */ #define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _exp_key, _key1, _key2) \ ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2))) +/** + * Generate AES-256-CMAC subkeys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _exp_key Input expanded AES-256-CMAC key + * @param[out] _key1 Subkey 1 + * @param[out] _key2 Subkey 2 + */ #define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _exp_key, _key1, _key2) \ ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2))) +/** + * Generate AES-128-XCBC expansion keys. + * + * @param[in] _mgr Pointer to multi-buffer structure + * @param[in] _key AES-128-XCBC key + * @param[out] _exp_key k1 expansion key + * @param[out] _exp_key2 k2 expansion key + * @param[out] _exp_key3 k3 expansion key + */ #define IMB_AES_XCBC_KEYEXP(_mgr, _key, _exp_key, _exp_key2, _exp_key3) \ ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3))) @@ -2511,80 +2561,213 @@ IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest); IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest); +/** + * Generate encryption/decryption AES-128 expansion keys. + * + * @param[in] key AES-128 key + * @param[out] enc_exp_keys AES-128 encryption expansion key + * @param[out] dec_exp_keys AES-128 decryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys); +/** + * @copydoc aes_keyexp_128_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_128_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_128_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, + void *dec_exp_keys); + +/** + * Generate encryption/decryption AES-192 expansion keys. + * + * @param[in] key AES-192 key + * @param[out] enc_exp_keys AES-192 encryption expansion key + * @param[out] dec_exp_keys AES-192 decryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, + void *dec_exp_keys); + +/** + * Generate encryption/decryption AES-256 expansion keys. + * + * @param[in] key AES-256 key + * @param[out] enc_exp_keys AES-256 encryption expansion key + * @param[out] dec_exp_keys AES-256 decryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys); -IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp, - void *k2, void *k3); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys, + void *dec_exp_keys); +/** + * @copydoc aes_keyexp_256_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys, + void *dec_exp_keys); + +/** + * Generate encryption AES-128 expansion keys. + * + * @param[in] key AES-128 key + * @param[out] enc_exp_keys AES-128 encryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key, void *enc_exp_keys); +/** + * @copydoc aes_keyexp_128_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_128_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_128_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key, + void *enc_exp_keys); + +/** + * Generate encryption AES-192 expansion keys. + * + * @param[in] key AES-192 key + * @param[out] enc_exp_keys AES-192 encryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key, void *enc_exp_keys); +/** + * @copydoc aes_keyexp_192_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_192_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_192_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key, + void *enc_exp_keys); + +/** + * Generate encryption AES-256 expansion keys. + * + * @param[in] key AES-256 key + * @param[out] enc_exp_keys AES-256 encryption expansion key + */ IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key, void *enc_exp_keys); +/** + * @copydoc aes_keyexp_256_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_256_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key, + void *enc_exp_keys); +/** + * @copydoc aes_keyexp_256_enc_sse + */ +IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key, + void *enc_exp_keys); + +/** + * Generate AES-128-XCBC expansion keys. + * + * @param[in] key Input AES-128-XCBC key + * @param[out] k1_exp k1 expansion key + * @param[out] k2 k2 key + * @param[out] k3 k3 key + */ +IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp, + void *k2, void *k3); +/** + * @copydoc aes_xcbc_expand_key_sse + */ +IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp, + void *k2, void *k3); +/** + * @copydoc aes_xcbc_expand_key_sse + */ +IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp, + void *k2, void *k3); +/** + * @copydoc aes_xcbc_expand_key_sse + */ +IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp, + void *k2, void *k3); + +/** + * Generate AES-128-CMAC subkeys. + * + * @param[in] key_exp Input expanded AES-128-CMAC key + * @param[out] key1 Subkey 1 + * @param[out] key2 Subkey 2 + */ IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1, void *key2); +/** + * @copydoc aes_cmac_subkey_gen_sse + */ +IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, + void *key2); +/** + * @copydoc aes_cmac_subkey_gen_sse + */ +IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, + void *key2); +/** + * @copydoc aes_cmac_subkey_gen_sse + */ +IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, + void *key2); + IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp, - void *k2, void *k3); -IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, - void *key2); IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp, - void *k2, void *k3); -IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, - void *key2); IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, const void *iv, const void *keys, uint64_t len); - -IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys, - void *dec_exp_keys); -IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp, - void *k2, void *k3); -IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key, - void *enc_exp_keys); -IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, - void *key2); IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -- GitLab From 7fa2edfe609150ea345911c365b453884a9195bc Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 13 Jun 2022 09:54:28 +0000 Subject: [PATCH 177/369] lib: [burst API] fixes for detected dead code CID 379081, 379080 and 379079 --- lib/include/mb_mgr_code.h | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index fac4ee2d..e20cccbf 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2703,11 +2703,6 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - if (job == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - /* validate job */ if (is_job_invalid(state, job, job->cipher_mode, job->hash_alg, @@ -2808,11 +2803,6 @@ uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - if (job == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - /* validate job */ if (is_job_invalid(state, job, IMB_CIPHER_CBC, IMB_AUTH_NULL, @@ -2878,11 +2868,6 @@ uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - if (job == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - /* validate job */ if (is_job_invalid(state, job, IMB_CIPHER_CBC, IMB_AUTH_NULL, @@ -2929,11 +2914,6 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - if (job == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - /* validate job */ if (is_job_invalid(state, job, IMB_CIPHER_CNTR, IMB_AUTH_NULL, @@ -3083,11 +3063,6 @@ uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - if (job == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - /* validate job */ if (is_job_invalid(state, job, IMB_CIPHER_NULL, -- GitLab From 29b6101e08c13146789601a0f59b4d812fccd972 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 13 Jun 2022 09:56:05 +0000 Subject: [PATCH 178/369] lib: [burst API] removed static keyword from function pointer declarations --- lib/include/mb_mgr_code.h | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index e20cccbf..cba23010 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2794,9 +2794,8 @@ uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, uint32_t i, completed_jobs = 0; MB_MGR_AES_OOO *aes_ooo = NULL; - static IMB_JOB * (*submit_fn) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; - static IMB_JOB * (*flush_fn) (MB_MGR_AES_OOO *state) = NULL; + IMB_JOB * (*submit_fn)(MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; + IMB_JOB * (*flush_fn)(MB_MGR_AES_OOO *state) = NULL; if (run_check) { /* validate jobs */ @@ -2857,10 +2856,9 @@ uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, const int run_check) { uint32_t i, completed_jobs = 0; - - static void (*submit_fn) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = NULL; + void (*submit_fn) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = NULL; (void) state; if (run_check) { @@ -2926,7 +2924,7 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, #ifdef AVX512 if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - static void (*submit_fn_vaes) (IMB_JOB *job) = NULL; + void (*submit_fn_vaes) (IMB_JOB *job) = NULL; if (key_size == 16) submit_fn_vaes = aes_cntr_128_submit_vaes_avx512; @@ -2944,10 +2942,10 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, } } else { #endif - static void (*submit_fn) (const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bytes, - uint64_t iv_len_bytes) = NULL; + void (*submit_fn) (const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bytes, + uint64_t iv_len_bytes) = NULL; if (key_size == 16) submit_fn = AES_CNTR_128; -- GitLab From b9333eb37acecfcde5aee8edff2c9c4c0d3b70bc Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 13 Jun 2022 09:59:03 +0000 Subject: [PATCH 179/369] lib: [burst API] despite SAFE_PARAM setting, no-check API's check nothing and normal API's check everything --- lib/include/mb_mgr_code.h | 88 +++++++++++++-------------------------- 1 file changed, 29 insertions(+), 59 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index cba23010..cffedb11 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2698,7 +2698,15 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, { uint32_t i, completed_jobs = 0; + /* reset error status */ + imb_set_errno(state, 0); + if (run_check) { + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } + /* validate jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -2735,7 +2743,7 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, } } - /* return if all jobs complete */ + /* return if all jobs complete */ if (completed_jobs == n_jobs) return completed_jobs; @@ -2755,32 +2763,12 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, uint32_t SUBMIT_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif - return submit_burst_and_check(state, jobs, n_jobs, 1); } uint32_t SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif - return submit_burst_and_check(state, jobs, n_jobs, 0); } @@ -2981,6 +2969,16 @@ uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const IMB_KEY_SIZE_BYTES key_size, const int run_check) { + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } + } + switch (cipher) { case IMB_CIPHER_CBC: if (dir == IMB_DIR_ENCRYPT) @@ -3009,16 +3007,6 @@ SUBMIT_CIPHER_BURST(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif - return submit_cipher_burst_and_check(state, jobs, n_jobs, cipher, dir, key_size, 1); } @@ -3030,16 +3018,6 @@ SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif - return submit_cipher_burst_and_check(state, jobs, n_jobs, cipher, dir, key_size, 0); } @@ -3194,6 +3172,16 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const IMB_HASH_ALG hash, const int run_check) { + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } + } + switch (hash) { case IMB_AUTH_HMAC_SHA_1: return submit_burst_hmac_sha_1(state, jobs, @@ -3225,15 +3213,6 @@ SUBMIT_HASH_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 1); } @@ -3242,15 +3221,6 @@ SUBMIT_HASH_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } -#endif return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 0); } -- GitLab From 78873f0d6b751df8c20ce484e185eccd1122f6b6 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 13 Jun 2022 10:30:33 +0000 Subject: [PATCH 180/369] lib: [JOB API] remove check for NULL multi-buffer manager pointer from no-check API --- lib/include/mb_mgr_code.h | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index cffedb11..1aff589f 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2480,6 +2480,17 @@ IMB_JOB * submit_job_and_check(IMB_MGR *state, const int run_check) { IMB_JOB *job = NULL; + + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return NULL; + } + } + #ifndef LINUX DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); @@ -2544,32 +2555,12 @@ exit: IMB_JOB * SUBMIT_JOB(IMB_MGR *state) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - return submit_job_and_check(state, 1); } IMB_JOB * SUBMIT_JOB_NOCHECK(IMB_MGR *state) { - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - return submit_job_and_check(state, 0); } -- GitLab From 26f6e6fccd4a0799148f2de91256c4fcee322a24 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 15 Jun 2022 10:14:09 +0000 Subject: [PATCH 181/369] lib: fix in memset size calculation in CBCS OOO manager reset --- lib/x86_64/ooo_mgr_reset.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 1838da4a..15f37fb2 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -44,9 +44,11 @@ void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xF76543210; else if (num_lanes == 12) { /* CBCS only */ + const size_t set_0xff_size = + sizeof(p_mgr->lens64) - (12 * sizeof(p_mgr->lens64[0])); + p_mgr->unused_lanes = 0xBA9876543210; - memset(&p_mgr->lens64[12], 0xFF, - sizeof(p_mgr->lens64) - sizeof(p_mgr->lens64[0] * 12)); + memset(&p_mgr->lens64[12], 0xFF, set_0xff_size); } else if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; } -- GitLab From 0355b018a15c123a3dde9e2724ee7f8297a9fe2c Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 16 Jun 2022 09:49:50 +0000 Subject: [PATCH 182/369] no-aesni: [ZUC-EIA3-256] Remove outdated comment --- lib/no-aesni/zuc_top_sse_no_aesni.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/no-aesni/zuc_top_sse_no_aesni.c b/lib/no-aesni/zuc_top_sse_no_aesni.c index 44fcfa4d..6e102811 100644 --- a/lib/no-aesni/zuc_top_sse_no_aesni.c +++ b/lib/no-aesni/zuc_top_sse_no_aesni.c @@ -972,7 +972,6 @@ zuc256_eia3_4_buffer_job_sse_no_aesni(const void * const pKey[NUM_SSE_BUFS], keys.pKeys[i] = pKey[i]; } - /* TODO: Handle 16-byte digest cases */ asm_Zuc256Initialization_4_sse_no_aesni(&keys, ivs, &state, T, tag_size); -- GitLab From 938ae87583d6377dcfe9a190808252f50422e0ad Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Mon, 13 Jun 2022 13:26:01 +0000 Subject: [PATCH 183/369] test: [ACVP] support CTR --- test/acvp_app_main.c | 87 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 0ee30cba..c2c16091 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -279,6 +279,84 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_ctr_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + fprintf(stderr, "Unsupported direction\n"); + return EXIT_FAILURE; + } + + switch (tc->key_len) { + case 128: + IMB_AES_KEYEXP_128(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 192: + IMB_AES_KEYEXP_192(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 256: + IMB_AES_KEYEXP_256(mb_mgr, tc->key, enc_keys, dec_keys); + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len >> 3; + job->cipher_mode = IMB_CIPHER_CNTR; + job->hash_alg = IMB_AUTH_NULL; + + job->iv = tc->iv; + job->iv_len_in_bytes = tc->iv_len; + job->cipher_start_src_offset_in_bytes = 0; + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + tc->ct_len = tc->pt_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } else /* DECRYPT */ { + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + tc->pt_len = tc->ct_len; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -293,8 +371,7 @@ static void usage(const char *app_name) app_name, app_name); } -int -main(int argc, char **argv) +int main(int argc, char **argv) { ACVP_RESULT acvp_ret = ACVP_SUCCESS; ACVP_CTX *ctx = NULL; @@ -375,7 +452,7 @@ main(int argc, char **argv) goto exit; } - /* Create test session and enable GCM tests */ + /* Create test session and enable supported algorithms */ acvp_ret = acvp_create_test_session(&ctx, logger, ACVP_LOG_LVL_INFO); if (acvp_ret != ACVP_SUCCESS) goto exit; @@ -388,6 +465,10 @@ main(int argc, char **argv) &aes_cbc_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CTR, + &aes_ctr_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From b9ebb131ad0ea42f7d24e10942dfc66af7855584 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 14 Jun 2022 09:34:48 +0000 Subject: [PATCH 184/369] sse: [SHA256] expose multi-buffer API --- lib/include/arch_sse_type1.h | 5 + lib/include/ipsec_ooo_mgr.h | 11 +- lib/include/mb_mgr_code.h | 14 +++ lib/include/ooo_mgr_reset.h | 3 + lib/include/sha_mb_mgr.h | 178 ++++++++++++++++++++++++++++- lib/intel-ipsec-mb.h | 1 + lib/no-aesni/mb_mgr_sse_no_aesni.c | 13 +++ lib/sse_t1/mb_mgr_sse.c | 7 +- lib/sse_t1/sha256_mult_sse.asm | 75 ++++++++++++ lib/sse_t1/sha_mb_sse.c | 23 ++++ lib/x86_64/alloc.c | 3 +- lib/x86_64/ooo_mgr_reset.c | 21 +++- perf/ipsec_perf.c | 38 +++--- 13 files changed, 361 insertions(+), 31 deletions(-) diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index 9779b831..32052805 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -202,6 +202,11 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_sse(const void *key_exp, void *key1, void *key2); uint32_t hec_32_sse(const uint8_t *in); diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index a12958ea..72c4348e 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -119,7 +119,7 @@ typedef struct { typedef struct { DECLARE_ALIGNED(uint32_t digest[SHA256_DIGEST_SZ], 32); - uint8_t *data_ptr[AVX512_NUM_SHA256_LANES]; + const uint8_t *data_ptr[AVX512_NUM_SHA256_LANES]; } SHA256_ARGS; typedef struct { @@ -356,6 +356,15 @@ typedef struct { uint64_t road_block; } MB_MGR_HMAC_SHA_256_OOO; +typedef struct { + SHA256_ARGS args; + DECLARE_ALIGNED(uint64_t lens[AVX512_NUM_SHA256_LANES], 16); + uint64_t unused_lanes; + HMAC_SHA1_LANE_DATA ldata[AVX512_NUM_SHA256_LANES]; + uint32_t num_lanes_inuse; + uint64_t road_block; +} MB_MGR_SHA_256_OOO; + typedef struct { SHA512_ARGS args; DECLARE_ALIGNED(uint16_t lens[8], 16); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 1aff589f..55018eb1 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -636,6 +636,9 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#ifdef SSE + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -701,11 +704,15 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_256: +#ifdef SSE + return SUBMIT_JOB_SHA256(sha_256_ooo, job); +#else IMB_SHA256(state, job->src + job->hash_start_src_offset_in_bytes, job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; +#endif case IMB_AUTH_SHA_384: IMB_SHA384(state, job->src + job->hash_start_src_offset_in_bytes, @@ -838,6 +845,9 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; +#ifdef SSE + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -869,6 +879,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); +#ifdef SSE + case IMB_AUTH_SHA_256: + return FLUSH_JOB_SHA256(sha_256_ooo, job); +#endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index 0539768d..c8510ad1 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -67,6 +67,9 @@ void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes); + IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index a8ef5204..15a4238c 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -41,6 +41,9 @@ extern void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, extern void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) @@ -63,28 +66,69 @@ void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) digest[lane + 4*16] = H4; } +__forceinline +void sha256_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[lane + 0*16] = SHA256_H0; + digest[lane + 1*16] = SHA256_H1; + digest[lane + 2*16] = SHA256_H2; + digest[lane + 3*16] = SHA256_H3; + digest[lane + 4*16] = SHA256_H4; + digest[lane + 5*16] = SHA256_H5; + digest[lane + 6*16] = SHA256_H6; + digest[lane + 7*16] = SHA256_H7; +} + __forceinline void sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) { if (sha_type == 1) sha1_mb_init_digest(digest, lane); + else if (sha_type == 256) + sha256_mb_init_digest(digest, lane); } __forceinline void sha_mb_generic_write_digest(void *dst, const void *src, - const int sha_type, const size_t offset, - const unsigned lane) + const int sha_type, const size_t offset, + const unsigned lane) { if (sha_type == 1) copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, offset, lane); + else if (sha_type == 256) + copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, + lane); +} + +__forceinline +void sha1_create_extra_blocks(MB_MGR_SHA_1_OOO *state, + const uint64_t blk_size, const uint64_t r, + const unsigned min_idx) +{ + HMAC_SHA1_LANE_DATA *ld = &state->ldata[min_idx]; + const uint64_t xblk_size = blk_size*state->ldata[min_idx].extra_blocks; + + memset(ld->extra_block, 0, sizeof(ld->extra_block)); + + var_memcpy(ld->extra_block, state->args.data_ptr[min_idx], r); + ld->extra_block[r] = 0x80; + + store8_be(&ld->extra_block[xblk_size - 8], + ld->job_in_lane->msg_len_to_hash_in_bytes * 8); + + state->args.data_ptr[min_idx] = &ld->extra_block[0]; + + state->lens[min_idx] = (uint16_t)xblk_size; + + state->ldata[min_idx].extra_blocks = 0; } __forceinline -void create_extra_blocks(MB_MGR_SHA_1_OOO *state, - const uint64_t blk_size, const uint64_t r, - const unsigned min_idx) +void sha256_create_extra_blocks(MB_MGR_SHA_256_OOO *state, + const uint64_t blk_size, const uint64_t r, + const unsigned min_idx) { HMAC_SHA1_LANE_DATA *ld = &state->ldata[min_idx]; const uint64_t xblk_size = blk_size*state->ldata[min_idx].extra_blocks; @@ -211,7 +255,7 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, /* create extra blocks */ if (state->ldata[min_idx].extra_blocks != 0) - create_extra_blocks(state, blk_size, r, min_idx); + sha1_create_extra_blocks(state, blk_size, r, min_idx); } while(state->lens[min_idx] != 0); @@ -227,3 +271,125 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, return ret_job; } + +__forceinline +IMB_JOB * +submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, + const unsigned max_jobs, const int is_submit, + const int sha_type, const uint64_t blk_size, + const uint64_t pad_size, + void (*fn)(SHA256_ARGS *, uint32_t)) +{ + unsigned lane, min_idx; + uint64_t min_len; + IMB_JOB *ret_job = NULL; + unsigned i; + + if (is_submit) { + /* + * SUBMIT + * - get a free lane id + */ + + lane = state->unused_lanes & 15; + state->unused_lanes >>= 4; + state->num_lanes_inuse++; + state->args.data_ptr[lane] = + job->src + job->hash_start_src_offset_in_bytes; + + sha_mb_generic_init(state->args.digest, sha_type, lane); + + /* copy job data in and set up initial blocks */ + state->ldata[lane].job_in_lane = job; + state->lens[lane] = job->msg_len_to_hash_in_bytes; + state->ldata[lane].extra_blocks = 1; + + /* enough jobs to start processing? */ + if (state->num_lanes_inuse != max_jobs) + return NULL; + } else { + /* + * FLUSH + * - find 1st non null job + */ + for (lane = 0; lane < max_jobs; lane++) + if (state->ldata[lane].job_in_lane != NULL) + break; + if (lane >= max_jobs) + return NULL; /* no not null job */ + } + + do { + if (is_submit) { + /* + * SUBMIT + * - find min common length to process + */ + min_idx = 0; + min_len = state->lens[0]; + + for (i = 1; i < max_jobs; i++) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } + } else { + /* + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ + min_idx = lane; + min_len = state->lens[lane]; + + for (i = 0; i < max_jobs; i++) { + if (i == lane) + continue; + + if (state->ldata[i].job_in_lane != NULL) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } else { + state->args.data_ptr[i] = + state->args.data_ptr[lane]; + state->lens[i] = UINT64_MAX; + } + } + } + + /* subtract min len from all lanes */ + const uint64_t min_len_blk = min_len & (~(blk_size - 1)); + + for (i = 0; i < max_jobs; i++) + state->lens[i] -= min_len_blk; + + const uint64_t r = min_len % blk_size; + + if (r >= (blk_size - pad_size)) + state->ldata[min_idx].extra_blocks = 2; + + /* run the algorithmic code on full selected blocks */ + if(min_len >= blk_size) + (*fn)(&state->args, + (uint32_t)(min_len/blk_size)); + + /* create extra blocks */ + if (state->ldata[min_idx].extra_blocks != 0) + sha256_create_extra_blocks(state, blk_size, r, min_idx); + + } while(state->lens[min_idx] != 0); + + ret_job = state->ldata[min_idx].job_in_lane; + /* put back processed packet into unused lanes, set job as complete */ + state->unused_lanes = (state->unused_lanes << 4) | min_idx; + state->num_lanes_inuse--; + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 16, min_idx); + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; + state->ldata[min_idx].job_in_lane = NULL; + return ret_job; +} diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 4ac2ac26..19c514e8 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1200,6 +1200,7 @@ typedef struct IMB_MGR { void *snow3g_uea2_ooo; void *snow3g_uia2_ooo; void *sha_1_ooo; + void *sha_256_ooo; } IMB_MGR; /** diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index ea9ae3a3..8edf8b33 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -88,6 +88,8 @@ #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni #define SUBMIT_JOB_SHA1 submit_job_sha1_sse #define FLUSH_JOB_SHA1 flush_job_sha1_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni @@ -400,6 +402,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -769,6 +772,16 @@ reset_ooo_mgrs(IMB_MGR *state) sha_1_ooo->num_lanes_inuse = 0; for (j = 0; j < SSE_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; + + /* Init SHA256 out-of-order fields */ + sha_256_ooo->lens[0] = 0; + sha_256_ooo->lens[1] = 0; + sha_256_ooo->lens[2] = 0; + sha_256_ooo->lens[3] = 0; + sha_256_ooo->unused_lanes = 0xF3210; + sha_256_ooo->num_lanes_inuse = 0; + for (j = 0; j < SSE_NUM_SHA256_LANES; j++) + sha_256_ooo->ldata[j].job_in_lane = NULL; } IMB_DLL_LOCAL void diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 51c355ce..a07db7c5 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -94,6 +94,8 @@ #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse #define SUBMIT_JOB_SHA1 submit_job_sha1_sse #define FLUSH_JOB_SHA1 flush_job_sha1_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse @@ -700,7 +702,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, 4); + ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); /* Init SNOW3G-UEA out-of-order fields */ ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); diff --git a/lib/sse_t1/sha256_mult_sse.asm b/lib/sse_t1/sha256_mult_sse.asm index b69c9b47..e77fcd27 100644 --- a/lib/sse_t1/sha256_mult_sse.asm +++ b/lib/sse_t1/sha256_mult_sse.asm @@ -311,6 +311,72 @@ endstruc ROUND_00_15 %%T1, %%i %endm +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + movdqa [rsp + 3*16], xmm6 + movdqa [rsp + 4*16], xmm7 + movdqa [rsp + 5*16], xmm8 + movdqa [rsp + 6*16], xmm9 + movdqa [rsp + 7*16], xmm10 + movdqa [rsp + 8*16], xmm11 + movdqa [rsp + 9*16], xmm12 + movdqa [rsp + 10*16], xmm13 + movdqa [rsp + 11*16], xmm14 + movdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + movdqa xmm6, [rsp + 3*16] + movdqa xmm7, [rsp + 4*16] + movdqa xmm8, [rsp + 5*16] + movdqa xmm9, [rsp + 6*16] + movdqa xmm10, [rsp + 7*16] + movdqa xmm11, [rsp + 8*16] + movdqa xmm12, [rsp + 9*16] + movdqa xmm13, [rsp + 10*16] + movdqa xmm14, [rsp + 11*16] + movdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + pxor xmm5, xmm5 + movdqa xmm5, [rsp + 3*16] + movdqa xmm5, [rsp + 4*16] + movdqa xmm5, [rsp + 5*16] + movdqa xmm5, [rsp + 6*16] + movdqa xmm5, [rsp + 7*16] + movdqa xmm5, [rsp + 8*16] + movdqa xmm5, [rsp + 9*16] + movdqa xmm5, [rsp + 10*16] + movdqa xmm5, [rsp + 11*16] + movdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + ;; SHA256_ARGS: ;; UINT128 digest[8]; // transposed digests ;; UINT8 *data_ptr[4]; @@ -320,6 +386,7 @@ endstruc ;; arg 1 : STATE : pointer args ;; arg 2 : INP_SIZE : size of data in blocks (assumed >= 1) ;; + MKGLOBAL(sha_256_mult_sse,function,internal) align 32 sha_256_mult_sse: @@ -447,4 +514,12 @@ Lrounds_16_xx: ; outer calling routine restores XMM and other GP registers ret +; void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha_256_mult_sse_from_c,function,internal) +call_sha_256_mult_sse_from_c: + FUNC_SAVE + call sha_256_mult_sse + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 1dd8f467..6905cec8 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -29,6 +29,8 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); /* ========================================================================== */ /* @@ -50,3 +52,24 @@ IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, call_sha1_mult_sse_from_c); } + +/* ========================================================================== */ +/* + * SHA256 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 1, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha_256_mult_sse_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 0, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha_256_mult_sse_from_c); +} diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 33b83663..c49999ee 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -84,7 +84,8 @@ const struct { OOO_INFO(aes256_cmac_ooo, MB_MGR_CMAC_OOO), OOO_INFO(snow3g_uea2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), - OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO) + OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO), + OOO_INFO(sha_256_ooo, MB_MGR_SHA_256_OOO) }; /** diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 15f37fb2..2daec121 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -332,11 +332,26 @@ void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, sizeof(*p_mgr)); - if (num_lanes == 4) + if (num_lanes == AVX_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xF3210; - else if (num_lanes == 8) + else if (num_lanes == AVX2_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xF76543210; - else if (num_lanes == 16) + else if (num_lanes == AVX512_NUM_SHA1_LANES) + p_mgr->unused_lanes = 0xFEDCBA9876543210; +} + +IMB_DLL_LOCAL +void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_SHA_256_OOO *p_mgr = (MB_MGR_SHA_256_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + + if (num_lanes == AVX_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == AVX2_NUM_SHA256_LANES) + p_mgr->unused_lanes = 0xF76543210; + else if (num_lanes == AVX512_NUM_SHA256_LANES) p_mgr->unused_lanes = 0xFEDCBA9876543210; } diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 3ea2cf8c..8dcead19 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -512,51 +512,51 @@ const struct str_value_mapping hash_algo_str_map[] = { } }, { - .name = "sha1", + .name = "aes-xcbc", .values.job_params = { - .hash_alg = TEST_SHA1 + .hash_alg = TEST_XCBC } }, { - .name = "sha224", + .name = "md5-hmac", .values.job_params = { - .hash_alg = TEST_SHA_224 + .hash_alg = TEST_MD5 } }, { - .name = "sha256", + .name = "aes-cmac", .values.job_params = { - .hash_alg = TEST_SHA_256 + .hash_alg = TEST_HASH_CMAC } }, { - .name = "sha384", + .name = "sha1", .values.job_params = { - .hash_alg = TEST_SHA_384 + .hash_alg = TEST_SHA1 } }, { - .name = "sha512", + .name = "sha224", .values.job_params = { - .hash_alg = TEST_SHA_512 + .hash_alg = TEST_SHA_224 } }, { - .name = "aes-xcbc", + .name = "sha256", .values.job_params = { - .hash_alg = TEST_XCBC + .hash_alg = TEST_SHA_256 } }, { - .name = "md5-hmac", + .name = "sha384", .values.job_params = { - .hash_alg = TEST_MD5 + .hash_alg = TEST_SHA_384 } }, { - .name = "aes-cmac", + .name = "sha512", .values.job_params = { - .hash_alg = TEST_HASH_CMAC + .hash_alg = TEST_SHA_512 } }, { @@ -2693,9 +2693,9 @@ print_times(struct variant_s *variant_list, struct params_s *params, }; const char *h_alg_names[TEST_NUM_HASH_TESTS - 1] = { "SHA1_HMAC", "SHA_224_HMAC", "SHA_256_HMAC", - "SHA_384_HMAC", "SHA_512_HMAC", - "SHA1", "SHA_224", "SHA_256", "SHA_384", "SHA_512", - "XCBC", "MD5", "CMAC", "CMAC_BITLEN", "CMAC_256", + "SHA_384_HMAC", "SHA_512_HMAC", "XCBC", + "MD5", "CMAC", "SHA1", "SHA_224", "SHA_256", + "SHA_384", "SHA_512", "CMAC_BITLEN", "CMAC_256", "NULL_HASH", "CRC32", "GCM", "CUSTOM", "CCM", "BIP-CRC32", "ZUC_EIA3_BITLEN", "SNOW3G_UIA2_BITLEN", "KASUMI_UIA1", "GMAC-128", "GMAC-192", "GMAC-256", -- GitLab From 970427a611f7d0003f7f56f632c14b21bf1c5663 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 16 Jun 2022 09:49:35 +0000 Subject: [PATCH 185/369] avx: [SHA256] expose multi-buffer API --- lib/avx/mb_mgr_avx.c | 7 +++- lib/avx/sha256_mult_avx.asm | 74 ++++++++++++++++++++++++++++++++++++ lib/avx/sha_mb_avx.c | 24 ++++++++++++ lib/include/arch_avx_type1.h | 5 +++ lib/include/mb_mgr_code.h | 8 ++-- lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index ba82fb5f..3c412481 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -123,6 +123,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx #define FLUSH_JOB_SHA1 flush_job_sha1_avx +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx +#define FLUSH_JOB_SHA256 flush_job_sha256_avx #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx @@ -428,7 +430,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, 4); + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX_NUM_SHA1_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX_NUM_SHA256_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx/sha256_mult_avx.asm b/lib/avx/sha256_mult_avx.asm index b6094fcb..598d4cda 100644 --- a/lib/avx/sha256_mult_avx.asm +++ b/lib/avx/sha256_mult_avx.asm @@ -252,6 +252,72 @@ PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x0c0d0e0f08090a0b0405060700010203 mksection .text +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + vmovdqa [rsp + 3*16], xmm6 + vmovdqa [rsp + 4*16], xmm7 + vmovdqa [rsp + 5*16], xmm8 + vmovdqa [rsp + 6*16], xmm9 + vmovdqa [rsp + 7*16], xmm10 + vmovdqa [rsp + 8*16], xmm11 + vmovdqa [rsp + 9*16], xmm12 + vmovdqa [rsp + 10*16], xmm13 + vmovdqa [rsp + 11*16], xmm14 + vmovdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + vmovdqa xmm6, [rsp + 3*16] + vmovdqa xmm7, [rsp + 4*16] + vmovdqa xmm8, [rsp + 5*16] + vmovdqa xmm9, [rsp + 6*16] + vmovdqa xmm10, [rsp + 7*16] + vmovdqa xmm11, [rsp + 8*16] + vmovdqa xmm12, [rsp + 9*16] + vmovdqa xmm13, [rsp + 10*16] + vmovdqa xmm14, [rsp + 11*16] + vmovdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa xmm5, [rsp + 3*16] + vmovdqa xmm5, [rsp + 4*16] + vmovdqa xmm5, [rsp + 5*16] + vmovdqa xmm5, [rsp + 6*16] + vmovdqa xmm5, [rsp + 7*16] + vmovdqa xmm5, [rsp + 8*16] + vmovdqa xmm5, [rsp + 9*16] + vmovdqa xmm5, [rsp + 10*16] + vmovdqa xmm5, [rsp + 11*16] + vmovdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + ;; SHA256_ARGS: ;; UINT128 digest[8]; // transposed digests ;; UINT8 *data_ptr[4]; @@ -384,4 +450,12 @@ Lrounds_16_xx: ; outer calling routine restores XMM and other GP registers ret +; void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha_256_mult_avx_from_c,function,internal) +call_sha_256_mult_avx_from_c: + FUNC_SAVE + call sha_256_mult_avx + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index 923eaade..b445b24b 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -50,3 +53,24 @@ IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, call_sha1_mult_avx_from_c); } + +/* ========================================================================== */ +/* + * SHA256 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 1, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha_256_mult_avx_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 0, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha_256_mult_avx_from_c); +} diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index 7c4e888f..3b81b851 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -149,6 +149,11 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + uint32_t hec_32_avx(const uint8_t *in); uint64_t hec_64_avx(const uint8_t *in); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 55018eb1..c54a2777 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -636,7 +636,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; #endif #ifdef AVX512 @@ -704,7 +704,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_256: -#ifdef SSE +#if defined(SSE) || defined(AVX) return SUBMIT_JOB_SHA256(sha_256_ooo, job); #else IMB_SHA256(state, @@ -845,7 +845,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; #endif #ifdef AVX512 @@ -879,7 +879,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#ifdef SSE +#if defined(SSE) || defined(AVX) case IMB_AUTH_SHA_256: return FLUSH_JOB_SHA256(sha_256_ooo, job); #endif diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 15a4238c..57aa7c06 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -44,6 +44,9 @@ extern void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, extern void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From 1dd2c3bcf5efc3759d6f6d8dfb53d35add2e1382 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 16 Jun 2022 10:51:24 +0000 Subject: [PATCH 186/369] avx2: [SHA256] expose multi-buffer API --- lib/avx2_t1/mb_mgr_avx2.c | 7 ++- lib/avx2_t1/sha256_oct_avx2.asm | 80 +++++++++++++++++++++++++++++++++ lib/avx2_t1/sha_mb_avx2.c | 24 ++++++++++ lib/include/arch_avx2_type1.h | 5 +++ lib/include/mb_mgr_code.h | 8 ++-- lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 122 insertions(+), 5 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index b0731f48..31cf2068 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -133,6 +133,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 #define FLUSH_JOB_SHA1 flush_job_sha1_avx2 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx2 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx @@ -408,7 +410,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, 8); + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx2_t1/sha256_oct_avx2.asm b/lib/avx2_t1/sha256_oct_avx2.asm index 0e768f9b..b7e53781 100644 --- a/lib/avx2_t1/sha256_oct_avx2.asm +++ b/lib/avx2_t1/sha256_oct_avx2.asm @@ -208,6 +208,78 @@ K256: mksection .text +%define XMM_STORAGE 10*16 +%define GP_STORAGE 9*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 + mov [rsp + 3*8], r13 + mov [rsp + 4*8], r14 + mov [rsp + 5*8], r15 +%ifndef LINUX + mov [rsp + 6*8], rsi + mov [rsp + 7*8], rdi + vmovdqa [rsp + 4*16], xmm6 + vmovdqa [rsp + 5*16], xmm7 + vmovdqa [rsp + 6*16], xmm8 + vmovdqa [rsp + 7*16], xmm9 + vmovdqa [rsp + 8*16], xmm10 + vmovdqa [rsp + 9*16], xmm11 + vmovdqa [rsp + 10*16], xmm12 + vmovdqa [rsp + 11*16], xmm13 + vmovdqa [rsp + 12*16], xmm14 + vmovdqa [rsp + 13*16], xmm15 +%endif ; LINUX + mov [rsp + 14*16], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] + mov r13, [rsp + 3*8] + mov r14, [rsp + 4*8] + mov r15, [rsp + 5*8] +%ifndef LINUX + mov rsi, [rsp + 6*8] + mov rdi, [rsp + 7*8] + vmovdqa xmm6, [rsp + 4*16] + vmovdqa xmm7, [rsp + 5*16] + vmovdqa xmm8, [rsp + 6*16] + vmovdqa xmm9, [rsp + 7*16] + vmovdqa xmm10, [rsp + 8*16] + vmovdqa xmm11, [rsp + 9*16] + vmovdqa xmm12, [rsp + 10*16] + vmovdqa xmm13, [rsp + 11*16] + vmovdqa xmm14, [rsp + 12*16] + vmovdqa xmm15, [rsp + 13*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa xmm5, [rsp + 4*16] + vmovdqa xmm5, [rsp + 5*16] + vmovdqa xmm5, [rsp + 6*16] + vmovdqa xmm5, [rsp + 7*16] + vmovdqa xmm5, [rsp + 8*16] + vmovdqa xmm5, [rsp + 9*16] + vmovdqa xmm5, [rsp + 10*16] + vmovdqa xmm5, [rsp + 11*16] + vmovdqa xmm5, [rsp + 12*16] + vmovdqa xmm5, [rsp + 13*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 14*16] ;; rsp pointer +%endmacro + %ifdef LINUX %define arg1 rdi %define arg2 rsi @@ -582,4 +654,12 @@ Lrounds_16_xx: add rsp, FRAMESZ ret +; void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha256_oct_avx2_from_c,function,internal) +call_sha256_oct_avx2_from_c: + FUNC_SAVE + call sha256_oct_avx2 + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index 39854bc4..cfc6f06a 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -50,3 +53,24 @@ IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, call_sha1_x8_avx2_from_c); } + +/* ========================================================================== */ +/* + * SHA256 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 8, 1, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_oct_avx2_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 8, 0, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_oct_avx2_from_c); +} diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index f09fe193..10a75c46 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -58,6 +58,11 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_avx2(const void *key_exp, void *key1, void *key2); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index c54a2777..37d4bef9 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -636,7 +636,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; #endif #ifdef AVX512 @@ -704,7 +704,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_256: -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA256(sha_256_ooo, job); #else IMB_SHA256(state, @@ -845,7 +845,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; #endif #ifdef AVX512 @@ -879,7 +879,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_256: return FLUSH_JOB_SHA256(sha_256_ooo, job); #endif diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 57aa7c06..db486339 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -47,6 +47,9 @@ extern void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, extern void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From 623e504cf8dfa755836308e46ae3e61808e967f3 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 16 Jun 2022 13:10:11 +0000 Subject: [PATCH 187/369] avx512: [SHA256] expose multi-buffer API --- lib/avx512_t1/mb_mgr_avx512.c | 7 ++- lib/avx512_t1/sha256_x16_avx512.asm | 80 +++++++++++++++++++++++++++++ lib/avx512_t1/sha_mb_avx512.c | 24 +++++++++ lib/include/arch_avx512_type1.h | 5 ++ lib/include/mb_mgr_code.h | 14 ----- lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 118 insertions(+), 15 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 46f2fe65..c2a16278 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -118,6 +118,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 #define FLUSH_JOB_SHA1 flush_job_sha1_avx512 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx512 #define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 #define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 @@ -978,7 +980,10 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 16); /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, 16); + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX512_NUM_SHA1_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx512_t1/sha256_x16_avx512.asm b/lib/avx512_t1/sha256_x16_avx512.asm index 76c8e082..f38b49a6 100644 --- a/lib/avx512_t1/sha256_x16_avx512.asm +++ b/lib/avx512_t1/sha256_x16_avx512.asm @@ -565,6 +565,78 @@ PSHUFFLE_BYTE_FLIP_MASK: mksection .text +%define XMM_STORAGE 10*16 +%define GP_STORAGE 9*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 + mov [rsp + 3*8], r13 + mov [rsp + 4*8], r14 + mov [rsp + 5*8], r15 +%ifndef LINUX + mov [rsp + 6*8], rsi + mov [rsp + 7*8], rdi + vmovdqa32 [rsp + 4*16], xmm6 + vmovdqa32 [rsp + 5*16], xmm7 + vmovdqa32 [rsp + 6*16], xmm8 + vmovdqa32 [rsp + 7*16], xmm9 + vmovdqa32 [rsp + 8*16], xmm10 + vmovdqa32 [rsp + 9*16], xmm11 + vmovdqa32 [rsp + 10*16], xmm12 + vmovdqa32 [rsp + 11*16], xmm13 + vmovdqa32 [rsp + 12*16], xmm14 + vmovdqa32 [rsp + 13*16], xmm15 +%endif ; LINUX + mov [rsp + 14*16], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] + mov r13, [rsp + 3*8] + mov r14, [rsp + 4*8] + mov r15, [rsp + 5*8] +%ifndef LINUX + mov rsi, [rsp + 6*8] + mov rdi, [rsp + 7*8] + vmovdqa32 xmm6, [rsp + 4*16] + vmovdqa32 xmm7, [rsp + 5*16] + vmovdqa32 xmm8, [rsp + 6*16] + vmovdqa32 xmm9, [rsp + 7*16] + vmovdqa32 xmm10, [rsp + 8*16] + vmovdqa32 xmm11, [rsp + 9*16] + vmovdqa32 xmm12, [rsp + 10*16] + vmovdqa32 xmm13, [rsp + 11*16] + vmovdqa32 xmm14, [rsp + 12*16] + vmovdqa32 xmm15, [rsp + 13*16] + +%ifdef SAFE_DATA + vpxord xmm5, xmm5, xmm5 + vmovdqa32 xmm5, [rsp + 4*16] + vmovdqa32 xmm5, [rsp + 5*16] + vmovdqa32 xmm5, [rsp + 6*16] + vmovdqa32 xmm5, [rsp + 7*16] + vmovdqa32 xmm5, [rsp + 8*16] + vmovdqa32 xmm5, [rsp + 9*16] + vmovdqa32 xmm5, [rsp + 10*16] + vmovdqa32 xmm5, [rsp + 11*16] + vmovdqa32 xmm5, [rsp + 12*16] + vmovdqa32 xmm5, [rsp + 13*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 14*16] ;; rsp pointer +%endmacro + ;; void sha256_x16_avx512(void **input_data, UINT128 *digest[16], UINT64 size) ;; arg 1 : pointer to SHA256 args structure ;; arg 2 : size (in blocks) ;; assumed to be >= 1 @@ -759,4 +831,12 @@ lastLoop: mov rsp, [rsp + _rsp] ret +; void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha256_x16_avx512_from_c,function,internal) +call_sha256_x16_avx512_from_c: + FUNC_SAVE + call sha256_x16_avx512 + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index 254ebb52..c115b91f 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -50,3 +53,24 @@ IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, call_sha1_x16_avx512_from_c); } + +/* ========================================================================== */ +/* + * SHA256 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 16, 1, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_x16_avx512_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 16, 0, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_x16_avx512_from_c); +} diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index 64ad312e..27379536 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -80,6 +80,11 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 37d4bef9..049c0464 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -636,9 +636,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -704,15 +702,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_256: -#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA256(sha_256_ooo, job); -#else - IMB_SHA256(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif case IMB_AUTH_SHA_384: IMB_SHA384(state, job->src + job->hash_start_src_offset_in_bytes, @@ -845,9 +835,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -879,10 +867,8 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_256: return FLUSH_JOB_SHA256(sha_256_ooo, job); -#endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index db486339..4e3717d6 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -50,6 +50,9 @@ extern void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, extern void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From d3e2d14ccb8b6c825e9701dad5075ec868fd9bae Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 08:29:49 +0000 Subject: [PATCH 188/369] lib: update release notes and readme --- README | 3 ++- README.md | 3 ++- ReleaseNotes.txt | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README b/README index 73e60732..a09b3530 100644 --- a/README +++ b/README @@ -135,7 +135,8 @@ implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | SHA1-96 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index cebef20a..beae19b7 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,8 @@ Table 2. List of supported integrity algorithms and their implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | -| SHA1-96 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | +| SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 16b0ea04..25f8a116 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -10,7 +10,7 @@ Library - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added -- SHA1 multi-buffer implementation added +- SHA1/256 multi-buffer implementation added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) -- GitLab From e07c97059387e27280372978d0193d32cbe3632a Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 11:26:19 +0000 Subject: [PATCH 189/369] test: update to test multiple vectors at a time --- test/sha_test.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/sha_test.c b/test/sha_test.c index 6c9580b9..26b5b6a6 100644 --- a/test/sha_test.c +++ b/test/sha_test.c @@ -597,14 +597,17 @@ sha_test(struct IMB_MGR *mb_mgr) struct test_suite_context sha1_ctx, sha224_ctx, sha256_ctx; struct test_suite_context sha384_ctx, sha512_ctx; int errors; + unsigned i; test_suite_start(&sha1_ctx, "SHA1"); test_suite_start(&sha224_ctx, "SHA224"); test_suite_start(&sha256_ctx, "SHA256"); test_suite_start(&sha384_ctx, "SHA384"); test_suite_start(&sha512_ctx, "SHA512"); - test_sha_vectors(mb_mgr, &sha1_ctx, &sha224_ctx, - &sha256_ctx, &sha384_ctx, &sha512_ctx, 1); + for (i = 1; i <= 17; i++) { + test_sha_vectors(mb_mgr, &sha1_ctx, &sha224_ctx, + &sha256_ctx, &sha384_ctx, &sha512_ctx, i); + } errors = test_suite_end(&sha1_ctx); errors += test_suite_end(&sha224_ctx); errors += test_suite_end(&sha256_ctx); -- GitLab From 51b234f8f59ceb674f2b25c639513d725bea5435 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 15:45:46 +0000 Subject: [PATCH 190/369] sse: [SHA224] expose multi-buffer API --- lib/include/arch_sse_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 14 ++++++++++++++ lib/include/sha_mb_mgr.h | 18 ++++++++++++++++++ lib/intel-ipsec-mb.h | 1 + lib/no-aesni/mb_mgr_sse_no_aesni.c | 12 ++++++++++++ lib/sse_t1/mb_mgr_sse.c | 5 +++++ lib/sse_t1/sha_mb_sse.c | 25 +++++++++++++++++++++++++ lib/x86_64/alloc.c | 1 + 8 files changed, 81 insertions(+) diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index 32052805..a6baa585 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -202,6 +202,11 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_sse(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 049c0464..5956bba7 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -637,6 +637,9 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -696,11 +699,15 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_1: return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: +#ifdef SSE + return SUBMIT_JOB_SHA224(sha_224_ooo, job); +#else IMB_SHA224(state, job->src + job->hash_start_src_offset_in_bytes, job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; +#endif case IMB_AUTH_SHA_256: return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: @@ -836,6 +843,9 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; +#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -867,6 +877,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); +#ifdef SSE + case IMB_AUTH_SHA_224: + return FLUSH_JOB_SHA224(sha_224_ooo, job); +#endif case IMB_AUTH_SHA_256: return FLUSH_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_AES_XCBC: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 4e3717d6..c4225f88 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -75,6 +75,19 @@ void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) digest[lane + 4*16] = H4; } +__forceinline +void sha224_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[lane + 0*16] = SHA224_H0; + digest[lane + 1*16] = SHA224_H1; + digest[lane + 2*16] = SHA224_H2; + digest[lane + 3*16] = SHA224_H3; + digest[lane + 4*16] = SHA224_H4; + digest[lane + 5*16] = SHA224_H5; + digest[lane + 6*16] = SHA224_H6; + digest[lane + 7*16] = SHA224_H7; +} + __forceinline void sha256_mb_init_digest(uint32_t *digest, const unsigned lane) { @@ -94,6 +107,8 @@ sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) { if (sha_type == 1) sha1_mb_init_digest(digest, lane); + else if (sha_type == 224) + sha224_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_mb_init_digest(digest, lane); } @@ -106,6 +121,9 @@ void sha_mb_generic_write_digest(void *dst, const void *src, if (sha_type == 1) copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, offset, lane); + else if (sha_type == 224) + copy_bswap4_array_mb(dst, src, NUM_SHA_224_DIGEST_WORDS, offset, + lane); else if (sha_type == 256) copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, lane); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 19c514e8..13a6807c 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1201,6 +1201,7 @@ typedef struct IMB_MGR { void *snow3g_uia2_ooo; void *sha_1_ooo; void *sha_256_ooo; + void *sha_224_ooo; } IMB_MGR; /** diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 8edf8b33..f309274f 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -88,6 +88,8 @@ #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse_no_aesni #define SUBMIT_JOB_SHA1 submit_job_sha1_sse #define FLUSH_JOB_SHA1 flush_job_sha1_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_sse #define SUBMIT_JOB_SHA256 submit_job_sha256_sse #define FLUSH_JOB_SHA256 flush_job_sha256_sse @@ -773,6 +775,16 @@ reset_ooo_mgrs(IMB_MGR *state) for (j = 0; j < SSE_NUM_SHA1_LANES; j++) sha_1_ooo->ldata[j].job_in_lane = NULL; + /* Init SHA224 out-of-order fields */ + sha_224_ooo->lens[0] = 0; + sha_224_ooo->lens[1] = 0; + sha_224_ooo->lens[2] = 0; + sha_224_ooo->lens[3] = 0; + sha_224_ooo->unused_lanes = 0xF3210; + sha_224_ooo->num_lanes_inuse = 0; + for (j = 0; j < SSE_NUM_SHA256_LANES; j++) + sha_224_ooo->ldata[j].job_in_lane = NULL; + /* Init SHA256 out-of-order fields */ sha_256_ooo->lens[0] = 0; sha_256_ooo->lens[1] = 0; diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index a07db7c5..70e86f55 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -94,6 +94,8 @@ #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse #define SUBMIT_JOB_SHA1 submit_job_sha1_sse #define FLUSH_JOB_SHA1 flush_job_sha1_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_sse #define SUBMIT_JOB_SHA256 submit_job_sha256_sse #define FLUSH_JOB_SHA256 flush_job_sha256_sse @@ -704,6 +706,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); + /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 6905cec8..0029d9a3 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -29,6 +29,10 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + +IMB_JOB *submit_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); @@ -53,6 +57,27 @@ IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) call_sha1_mult_sse_from_c); } +/* ========================================================================== */ +/* + * SHA224 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 1, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha_256_mult_sse_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 0, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha_256_mult_sse_from_c); +} + /* ========================================================================== */ /* * SHA256 MB API diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index c49999ee..11b7d60b 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -85,6 +85,7 @@ const struct { OOO_INFO(snow3g_uea2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO), + OOO_INFO(sha_224_ooo, MB_MGR_SHA_256_OOO), OOO_INFO(sha_256_ooo, MB_MGR_SHA_256_OOO) }; -- GitLab From 98b83b84c0e9aa522081d7f9ae4a980c5fa9ee50 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 16:00:02 +0000 Subject: [PATCH 191/369] avx: [SHA224] expose multi-buffer API --- lib/avx/mb_mgr_avx.c | 5 +++++ lib/avx/sha_mb_avx.c | 24 ++++++++++++++++++++++++ lib/include/arch_avx_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 8 ++++---- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 3c412481..12273cf6 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -123,6 +123,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx #define FLUSH_JOB_SHA1 flush_job_sha1_avx +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx +#define FLUSH_JOB_SHA224 flush_job_sha224_avx #define SUBMIT_JOB_SHA256 submit_job_sha256_avx #define FLUSH_JOB_SHA256 flush_job_sha256_avx @@ -432,6 +434,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, AVX_NUM_SHA1_LANES); + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX_NUM_SHA256_LANES); + /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX_NUM_SHA256_LANES); } diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index b445b24b..03bad7cf 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); @@ -54,6 +57,27 @@ IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) call_sha1_mult_avx_from_c); } +/* ========================================================================== */ +/* + * SHA224 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 1, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha_256_mult_avx_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 4, 0, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha_256_mult_avx_from_c); +} + /* ========================================================================== */ /* * SHA256 MB API diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index 3b81b851..a6ac773d 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -149,6 +149,11 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 5956bba7..e1b368be 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -637,7 +637,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; #endif #ifdef AVX512 @@ -699,7 +699,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_1: return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: -#ifdef SSE +#if defined(SSE) || defined(AVX) return SUBMIT_JOB_SHA224(sha_224_ooo, job); #else IMB_SHA224(state, @@ -843,7 +843,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; #endif #ifdef AVX512 @@ -877,7 +877,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#ifdef SSE +#if defined(SSE) || defined(AVX) case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); #endif -- GitLab From 91f489102faaa5a502461f93be9356da1465d53c Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 16:09:51 +0000 Subject: [PATCH 192/369] avx2: [SHA224] expose multi-buffer API --- lib/avx2_t1/mb_mgr_avx2.c | 5 +++++ lib/avx2_t1/sha_mb_avx2.c | 24 ++++++++++++++++++++++++ lib/include/arch_avx2_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 8 ++++---- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 31cf2068..bc2c1b67 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -133,6 +133,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 #define FLUSH_JOB_SHA1 flush_job_sha1_avx2 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx2 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx2 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 #define FLUSH_JOB_SHA256 flush_job_sha256_avx2 @@ -412,6 +414,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX2_NUM_SHA256_LANES); + /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); } diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index cfc6f06a..a54e60db 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); @@ -54,6 +57,27 @@ IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) call_sha1_x8_avx2_from_c); } +/* ========================================================================== */ +/* + * SHA224 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 8, 1, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_oct_avx2_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 8, 0, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_oct_avx2_from_c); +} + /* ========================================================================== */ /* * SHA256 MB API diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index 10a75c46..bea85e62 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -58,6 +58,11 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index e1b368be..e4ac4b0a 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -637,7 +637,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; #endif #ifdef AVX512 @@ -699,7 +699,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_1: return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA224(sha_224_ooo, job); #else IMB_SHA224(state, @@ -843,7 +843,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; #endif #ifdef AVX512 @@ -877,7 +877,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); #endif -- GitLab From e3d044a1ee36b71fbdbb0c157e4ae3742059f6ea Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 16:22:37 +0000 Subject: [PATCH 193/369] avx512: [SHA224] expose multi-buffer API --- lib/avx512_t1/mb_mgr_avx512.c | 5 +++++ lib/avx512_t1/sha_mb_avx512.c | 25 +++++++++++++++++++++++++ lib/include/arch_avx512_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 14 -------------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 1 + 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index c2a16278..35668c6e 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -118,6 +118,8 @@ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 #define FLUSH_JOB_SHA1 flush_job_sha1_avx512 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx512 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx512 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 #define FLUSH_JOB_SHA256 flush_job_sha256_avx512 @@ -982,6 +984,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, AVX512_NUM_SHA1_LANES); + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX512_NUM_SHA256_LANES); + /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); } diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index c115b91f..56ed096e 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -30,6 +30,9 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); @@ -54,6 +57,28 @@ IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) call_sha1_x16_avx512_from_c); } +/* ========================================================================== */ +/* + * SHA224 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 16, 1, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_x16_avx512_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 16, 0, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_x16_avx512_from_c); +} + + /* ========================================================================== */ /* * SHA256 MB API diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index 27379536..c8533690 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -80,6 +80,11 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index e4ac4b0a..4592b938 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -637,9 +637,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -699,15 +697,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_1: return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: -#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA224(sha_224_ooo, job); -#else - IMB_SHA224(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif case IMB_AUTH_SHA_256: return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: @@ -843,9 +833,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; -#endif #ifdef AVX512 MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -877,10 +865,8 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: return FLUSH_JOB_SHA1(sha_1_ooo, job); -#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); -#endif case IMB_AUTH_SHA_256: return FLUSH_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_AES_XCBC: diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index f309274f..08c0ab07 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -404,6 +404,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; /* Init AES out-of-order fields */ -- GitLab From 8b3eab7a857e3613b530c506f760ea3c94ce5f63 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 17 Jun 2022 16:24:21 +0000 Subject: [PATCH 194/369] lib: update release notes and readme --- README | 1 + README.md | 1 + ReleaseNotes.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index a09b3530..78ab12f7 100644 --- a/README +++ b/README @@ -136,6 +136,7 @@ implementations. | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-224 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index beae19b7..1c7d9d18 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Table 2. List of supported integrity algorithms and their implementations. | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-224 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 25f8a116..86efa10e 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -10,7 +10,7 @@ Library - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added -- SHA1/256 multi-buffer implementation added +- SHA1/224/256 multi-buffer implementation added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) -- GitLab From e1782908afa6ab802677021a3ec75612b91ed159 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 10 Jun 2022 16:37:02 +0000 Subject: [PATCH 195/369] test: [ACVP] support for GMAC --- test/acvp_app_main.c | 142 +++++++++++++++++++++++++++++++------------ 1 file changed, 102 insertions(+), 40 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index c2c16091..49183690 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -45,6 +45,7 @@ static ACVP_RESULT logger(char *msg) IMB_MGR *mb_mgr = NULL; int verbose = 0; int direct_api = 0; /* job API by default */ +int gmac_api = 0; static int aes_cbc_handler(ACVP_TEST_CASE *test_case) { @@ -144,14 +145,20 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) { ACVP_SYM_CIPHER_TC *tc; IMB_JOB *job = NULL; - aes_gcm_init_var_iv_t init_var_iv = mb_mgr->gcm128_init_var_iv; - aes_gcm_enc_dec_update_t update_enc = mb_mgr->gcm128_enc_update; - aes_gcm_enc_dec_finalize_t finalize_enc = mb_mgr->gcm128_enc_finalize; - aes_gcm_enc_dec_update_t update_dec = mb_mgr->gcm128_dec_update; - aes_gcm_enc_dec_finalize_t finalize_dec = mb_mgr->gcm128_dec_finalize; - + aes_gcm_init_var_iv_t gcm_init_var_iv = mb_mgr->gcm128_init_var_iv; + aes_gcm_enc_dec_update_t gcm_update_enc = mb_mgr->gcm128_enc_update; + aes_gcm_enc_dec_finalize_t gcm_finalize_enc = + mb_mgr->gcm128_enc_finalize; + aes_gcm_enc_dec_update_t gcm_update_dec = mb_mgr->gcm128_dec_update; + aes_gcm_enc_dec_finalize_t gcm_finalize_dec = + mb_mgr->gcm128_dec_finalize; + aes_gmac_init_t gmac_init_var = mb_mgr->gmac128_init; + aes_gmac_update_t gmac_update = mb_mgr->gmac128_update; + aes_gmac_finalize_t gmac_finalize = mb_mgr->gmac128_finalize; struct gcm_key_data key; struct gcm_context_data ctx; + IMB_HASH_ALG hash_mode; + int use_gmac_api = 0; if (test_case == NULL) return EXIT_FAILURE; @@ -163,15 +170,26 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) return EXIT_FAILURE; } + if (gmac_api) { + if ((tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->pt_len == 0) || + (tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT && + tc->ct_len == 0)) + use_gmac_api = 1; + } + switch (tc->key_len) { case 128: IMB_AES128_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_128; break; case 192: IMB_AES192_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_192; break; case 256: IMB_AES256_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_256; break; default: fprintf(stderr, "Unsupported AES key length\n"); @@ -184,18 +202,24 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) /* Function pointers already set for 128-bit key */ break; case 192: - init_var_iv = mb_mgr->gcm192_init_var_iv; - update_enc = mb_mgr->gcm192_enc_update; - finalize_enc = mb_mgr->gcm192_enc_finalize; - update_dec = mb_mgr->gcm192_dec_update; - finalize_dec = mb_mgr->gcm192_dec_finalize; + gcm_init_var_iv = mb_mgr->gcm192_init_var_iv; + gcm_update_enc = mb_mgr->gcm192_enc_update; + gcm_finalize_enc = mb_mgr->gcm192_enc_finalize; + gcm_update_dec = mb_mgr->gcm192_dec_update; + gcm_finalize_dec = mb_mgr->gcm192_dec_finalize; + gmac_init_var = mb_mgr->gmac192_init; + gmac_update = mb_mgr->gmac192_update; + gmac_finalize = mb_mgr->gmac192_finalize; break; case 256: - init_var_iv = mb_mgr->gcm256_init_var_iv; - update_enc = mb_mgr->gcm256_enc_update; - finalize_enc = mb_mgr->gcm256_enc_finalize; - update_dec = mb_mgr->gcm256_dec_update; - finalize_dec = mb_mgr->gcm256_dec_finalize; + gcm_init_var_iv = mb_mgr->gcm256_init_var_iv; + gcm_update_enc = mb_mgr->gcm256_enc_update; + gcm_finalize_enc = mb_mgr->gcm256_enc_finalize; + gcm_update_dec = mb_mgr->gcm256_dec_update; + gcm_finalize_dec = mb_mgr->gcm256_dec_finalize; + gmac_init_var = mb_mgr->gmac256_init; + gmac_update = mb_mgr->gmac256_update; + gmac_finalize = mb_mgr->gmac256_finalize; break; default: fprintf(stderr, "Unsupported AES key length\n"); @@ -204,32 +228,54 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) } else { job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len >> 3; - job->cipher_mode = IMB_CIPHER_GCM; - job->hash_alg = IMB_AUTH_AES_GMAC; - job->iv = tc->iv; - job->iv_len_in_bytes = tc->iv_len; + if (use_gmac_api == 1) { + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = hash_mode; + job->u.GMAC._iv = tc->iv; + job->u.GMAC.iv_len_in_bytes = tc->iv_len; + job->u.GMAC._key = &key; + } else { + job->cipher_mode = IMB_CIPHER_GCM; + job->hash_alg = IMB_AUTH_AES_GMAC; + job->u.GCM.aad = tc->aad; + job->u.GCM.aad_len_in_bytes = tc->aad_len; + job->enc_keys = &key; + job->dec_keys = &key; + job->iv = tc->iv; + job->iv_len_in_bytes = tc->iv_len; + } job->cipher_start_src_offset_in_bytes = 0; job->hash_start_src_offset_in_bytes = 0; - job->enc_keys = &key; - job->dec_keys = &key; - job->u.GCM.aad = tc->aad; - job->u.GCM.aad_len_in_bytes = tc->aad_len; job->auth_tag_output_len_in_bytes = tc->tag_len; } if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { if (direct_api == 1) { - init_var_iv(&key, &ctx, tc->iv, tc->iv_len, - tc->aad, tc->aad_len); - update_enc(&key, &ctx, tc->ct, tc->pt, tc->pt_len); - finalize_enc(&key, &ctx, tc->tag, tc->tag_len); + if (use_gmac_api == 1) { + gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); + gmac_update(&key, &ctx, tc->aad, tc->aad_len); + gmac_finalize(&key, &ctx, tc->tag, + tc->tag_len); + } else { + gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, + tc->aad, tc->aad_len); + gcm_update_enc(&key, &ctx, tc->ct, + tc->pt, tc->pt_len); + gcm_finalize_enc(&key, &ctx, tc->tag, + tc->tag_len); + } } else { + if (use_gmac_api == 1) { + job->src = tc->aad; + job->msg_len_to_hash_in_bytes = tc->aad_len; + } else { + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + job->msg_len_to_hash_in_bytes = tc->pt_len; + } job->cipher_direction = IMB_DIR_ENCRYPT; job->chain_order = IMB_ORDER_CIPHER_HASH; - job->src = tc->pt; - job->dst = tc->ct; - job->msg_len_to_cipher_in_bytes = tc->pt_len; - job->msg_len_to_hash_in_bytes = tc->pt_len; job->auth_tag_output = tc->tag; job = IMB_SUBMIT_JOB(mb_mgr); @@ -244,17 +290,30 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) uint8_t res_tag[MAX_TAG_LENGTH]; if (direct_api == 1) { - init_var_iv(&key, &ctx, tc->iv, tc->iv_len, - tc->aad, tc->aad_len); - update_dec(&key, &ctx, tc->pt, tc->ct, tc->ct_len); - finalize_dec(&key, &ctx, res_tag, tc->tag_len); + if (use_gmac_api == 1) { + gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); + gmac_update(&key, &ctx, tc->aad, tc->aad_len); + gmac_finalize(&key, &ctx, tc->tag, tc->tag_len); + } else { + gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, + tc->aad, tc->aad_len); + gcm_update_dec(&key, &ctx, tc->pt, + tc->ct, tc->ct_len); + gcm_finalize_dec(&key, &ctx, + res_tag, tc->tag_len); + } } else { + if (use_gmac_api == 1) { + job->src = tc->aad; + job->msg_len_to_hash_in_bytes = tc->aad_len; + } else { + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + job->msg_len_to_hash_in_bytes = tc->ct_len; + } job->cipher_direction = IMB_DIR_DECRYPT; job->chain_order = IMB_ORDER_HASH_CIPHER; - job->src = tc->ct; - job->dst = tc->pt; - job->msg_len_to_cipher_in_bytes = tc->ct_len; - job->msg_len_to_hash_in_bytes = tc->ct_len; job->auth_tag_output = res_tag; job = IMB_SUBMIT_JOB(mb_mgr); @@ -364,6 +423,7 @@ static void usage(const char *app_name) "--req FILENAME: request file in JSON format (required)\n" "--resp FILENAME: response file in JSON format (required)\n" "--direct-api: uses direct API instead of job API if available\n" + "--gmac-api: uses GMAC API\n" "--arch ARCH: select arch to test (SSE/AVX/AVX2/AVX512)\n" "-h: print this message\n" "-v: verbose, prints extra information\n\n" @@ -431,6 +491,8 @@ int main(int argc, char **argv) i++; } else if (strcmp(argv[i], "--direct-api") == 0) { direct_api = 1; + } else if (strcmp(argv[i], "--gmac-api") == 0) { + gmac_api = 1; } else if (strcmp(argv[i], "-h") == 0) { usage(argv[0]); ret = EXIT_SUCCESS; -- GitLab From b4813e42bb7892604e4563f9f28eb43a98a4a3f2 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 15 Jun 2022 21:38:23 +0000 Subject: [PATCH 196/369] sse_t1: [SNOW3G-UIA2] Add macro generating 5 keystreams per 4 lanes Co-authored-by: Kamila Lipinska Co-authored-by: Marcel Cornu --- lib/sse_t1/snow3g_uea2_by4_sse.asm | 145 +++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 27 deletions(-) diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index 8d7a29e8..3a588561 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -667,17 +667,17 @@ endstruc %define %%OUT_XMM_LANE_1 %2 ;; [out] 128bit keystream for lane 1 %define %%OUT_XMM_LANE_2 %3 ;; [out] 128bit keystream for lane 0 %define %%OUT_XMM_LANE_3 %4 ;; [out] 128bit keystream for lane 3 -%define %%TMPXMM %5 ;; [clobbered] temporary xmm register -%define %%TMPXMM_1 %6 ;; [clobbered] temporary xmm register +%define %%XTMP0 %5 ;; [clobbered] temporary xmm register +%define %%XTMP1 %6 ;; [clobbered] temporary xmm register movdqa %%OUT_XMM_LANE_2, [rsp + _keystream + 0 * 16] movdqa %%OUT_XMM_LANE_1, [rsp + _keystream + 1 * 16] - movdqa %%TMPXMM_1, [rsp + _keystream + 2 * 16] + movdqa %%XTMP1, [rsp + _keystream + 2 * 16] movdqa %%OUT_XMM_LANE_3, [rsp + _keystream + 3 * 16] ;; output looks like: {t0 r1 r0 r3} - TRANSPOSE4_U32 %%OUT_XMM_LANE_2, %%OUT_XMM_LANE_1, %%TMPXMM_1, \ - %%OUT_XMM_LANE_3, %%OUT_XMM_LANE_0, %%TMPXMM + TRANSPOSE4_U32 %%OUT_XMM_LANE_2, %%OUT_XMM_LANE_1, %%XTMP1, \ + %%OUT_XMM_LANE_3, %%OUT_XMM_LANE_0, %%XTMP0 %endmacro @@ -768,10 +768,10 @@ endstruc %define %%STATE %1 ;; [in] ptr to MB_MGR_SNOW3G_OOO structure %define %%LANE %2 ;; [in] nr of lane initialize data in %define %%P_KEY %3 ;; [in] ptr to key -%define %%P_IV %4 ;; [in*] ptr to IV copy (IV bytes swapped in each dw) -%define %%TMPXMM_1 %5 ;; [clobbered] temporary xmm reg -%define %%TMPXMM_2 %6 ;; [clobbered] temporary xmm reg -%define %%TMPXMM_3 %7 ;; [clobbered] temporary xmm reg +%define %%P_IV %4 ;; [in] ptr to IV +%define %%TMPXMM_1 %5 ;; [clobbered] temporary xmm reg +%define %%TMPXMM_2 %6 ;; [clobbered] temporary xmm reg +%define %%TMPXMM_3 %7 ;; [clobbered] temporary xmm reg movd %%TMPXMM_1, [%%P_KEY] ;; key movdqa %%TMPXMM_2, %%TMPXMM_1 @@ -844,24 +844,24 @@ endstruc %define %%IN %3 ;; [clobbered] r64 gp reg temp %define %%OUT %4 ;; [clobbered] r64 gp reg temp %define %%LENGTH %5 ;; [clobbered] r64 gp reg temp -%define %%TMP1_64 %6 ;; [clobbered] r64 gp reg temp -%define %%TMP2_64 %7 ;; [clobbered] r64 gp reg temp -%define %%TMP1 %8 ;; [clobbered] temporary xmm register -%define %%TMP2 %9 ;; [clobbered] temporary xmm register -%define %%TMP3 %10 ;; [clobbered] temporary xmm register -%define %%TMP4 %11 ;; [clobbered] temporary xmm register -%define %%TMP5 %12 ;; [clobbered] temporary xmm register -%define %%TMP6 %13 ;; [clobbered] temporary xmm register -%define %%TMP7 %14 ;; [clobbered] temporary xmm register -%define %%TMP8 %15 ;; [clobbered] temporary xmm register -%define %%TMP9 %16 ;; [clobbered] temporary xmm register -%define %%TMP10 %17 ;; [clobbered] temporary xmm register -%define %%TMP11 %18 ;; [clobbered] temporary xmm register -%define %%TMP12 %19 ;; [clobbered] temporary xmm register -%define %%TMP13 %20 ;; [clobbered] temporary xmm register -%define %%TMP14 %21 ;; [clobbered] temporary xmm register -%define %%TMP15 %22 ;; [clobbered] temporary xmm register -%define %%TMP16 %23 ;; [clobbered] temporary xmm register +%define %%TMP1_64 %6 ;; [clobbered] r64 gp reg temp +%define %%TMP2_64 %7 ;; [clobbered] r64 gp reg temp +%define %%TMP1 %8 ;; [clobbered] temporary xmm register +%define %%TMP2 %9 ;; [clobbered] temporary xmm register +%define %%TMP3 %10 ;; [clobbered] temporary xmm register +%define %%TMP4 %11 ;; [clobbered] temporary xmm register +%define %%TMP5 %12 ;; [clobbered] temporary xmm register +%define %%TMP6 %13 ;; [clobbered] temporary xmm register +%define %%TMP7 %14 ;; [clobbered] temporary xmm register +%define %%TMP8 %15 ;; [clobbered] temporary xmm register +%define %%TMP9 %16 ;; [clobbered] temporary xmm register +%define %%TMP10 %17 ;; [clobbered] temporary xmm register +%define %%TMP11 %18 ;; [clobbered] temporary xmm register +%define %%TMP12 %19 ;; [clobbered] temporary xmm register +%define %%TMP13 %20 ;; [clobbered] temporary xmm register +%define %%TMP14 %21 ;; [clobbered] temporary xmm register +%define %%TMP15 %22 ;; [clobbered] temporary xmm register +%define %%TMP16 %23 ;; [clobbered] temporary xmm register sub %%COMMON_LEN, 1 mov %%LENGTH, %%COMMON_LEN @@ -921,4 +921,95 @@ endstruc %%zero_bytes: %endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Generate 5 double words of key stream for SNOW3G authentication +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro SNOW3G_AUTH_INIT_5_BY_4 22 +%define %%KEY %1 ;; [in] array of pointers to 4 keys +%define %%IV %2 ;; [in] array of pointers to 4 IV's +%define %%DST_PTR %3 ;; [in] destination buffer to put 5DW of keystream for each lane +%define %%TMP1_64 %4 ;; [clobbered] r64 gp reg temp +%define %%TMP2_64 %5 ;; [clobbered] r64 gp reg temp +%define %%TMP1 %6 ;; [clobbered] temporary xmm register +%define %%TMP2 %7 ;; [clobbered] temporary xmm register +%define %%TMP3 %8 ;; [clobbered] temporary xmm register +%define %%TMP4 %9 ;; [clobbered] temporary xmm register +%define %%TMP5 %10 ;; [clobbered] temporary xmm register +%define %%TMP6 %11 ;; [clobbered] temporary xmm register +%define %%TMP7 %12 ;; [clobbered] temporary xmm register +%define %%TMP8 %13 ;; [clobbered] temporary xmm register +%define %%TMP9 %14 ;; [clobbered] temporary xmm register +%define %%TMP10 %15 ;; [clobbered] temporary xmm register +%define %%TMP11 %16 ;; [clobbered] temporary xmm register +%define %%TMP12 %17 ;; [clobbered] temporary xmm register +%define %%TMP13 %18 ;; [clobbered] temporary xmm register +%define %%TMP14 %19 ;; [clobbered] temporary xmm register +%define %%TMP15 %20 ;; [clobbered] temporary xmm register +%define %%TMP16 %21 ;; [clobbered] temporary xmm register +%define %%STATE %22 ;; [in] ptr to LFSR/FSM struct + +%define KEYGEN_STAGE _snow3g_args_LD_ST_MASK +%define INIT1_DONE _snow3g_args_LD_ST_MASK+16 + + ;; Initialize LFSR and FSM registers +%assign i 0 +%rep 4 + mov %%TMP1_64, [%%KEY + i*8] + mov %%TMP2_64, [%%IV + i*8] + SNOW3G_INIT_LANE_SSE %%STATE, i, %%TMP1_64, %%TMP2_64, %%TMP1, %%TMP2, %%TMP3 +%assign i (i+1) +%endrep + + ;; Run 32 iteration in INIT mode (reject keystreams) +%rep 32 + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 +%endrep + ;; Mark INIT1 phase done for all lanes + movdqa %%TMP1, [rel all_fs] + movdqa [state + INIT1_DONE], %%TMP1 + + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + + ;; Put all lanes in KEYGEN state + movdqa %%TMP1, [rel all_fs] + movdqa [state + KEYGEN_STAGE], %%TMP1 + + ;; Generate 4 dw of keystream for each lane +%assign i 0 +%rep 4 + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, i + +%assign i (i+1) +%endrep + TRANSPOSE_4X32 %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, %%TMP6 + + ;; Store 4 dw of keystream for each lane + movdqu [%%DST_PTR + 0*32], %%TMP1 + movdqu [%%DST_PTR + 1*32], %%TMP2 + movdqu [%%DST_PTR + 2*32], %%TMP3 + movdqu [%%DST_PTR + 3*32], %%TMP4 + + ;; Generate final dw of keystream for each lane + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + + ;; Store final dw of keystream for each lane + mov DWORD(%%TMP1_64), [rsp + _keystream + 0*4] + mov [%%DST_PTR + 16 + (0*32)], DWORD(%%TMP1_64) + mov DWORD(%%TMP1_64), [rsp + _keystream + 1*4] + mov [%%DST_PTR + 16 + (1 * 32)], DWORD(%%TMP1_64) + mov DWORD(%%TMP1_64), [rsp + _keystream + 2*4] + mov [%%DST_PTR + 16 + (2*32)], DWORD(%%TMP1_64) + mov DWORD(%%TMP1_64), [rsp + _keystream + 3*4] + mov [%%DST_PTR + 16 + (3 * 32)], DWORD(%%TMP1_64) + +%endmacro + mksection stack-noexec -- GitLab From 04f74c8cbd76d2daa00a0e6b9b900f4608d53441 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 15 Jun 2022 21:47:46 +0000 Subject: [PATCH 197/369] sse_t1: [SNOW3G-UIA2] Add submit and flush module --- lib/Makefile | 1 + ...mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm | 306 ++++++++++++++++++ lib/sse_t1/snow3g_uea2_by4_sse.asm | 16 +- lib/win_x64.mak | 1 + 4 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm diff --git a/lib/Makefile b/lib/Makefile index f364b619..324a825c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -454,6 +454,7 @@ asm_sse_lib_objs := \ mb_mgr_zuc_submit_flush_sse.o \ mb_mgr_zuc_submit_flush_gfni_sse.o \ mb_mgr_snow3g_uea2_submit_flush_x4_sse.o \ + mb_mgr_snow3g_uia2_submit_flush_x4_sse.o \ ethernet_fcs_sse.o \ crc16_x25_sse.o \ crc32_sctp_sse.o \ diff --git a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm new file mode 100644 index 00000000..cc69e15a --- /dev/null +++ b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm @@ -0,0 +1,306 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +%include "include/os.asm" +%include "include/imb_job.asm" +%include "include/mb_mgr_datastruct.asm" +%include "include/reg_sizes.asm" +%include "include/cet.inc" +%include "include/clear_regs.asm" +%include "sse_t1/snow3g_uea2_by4_sse.asm" + +%define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +%define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse +%define SNOW3G_F9_1_BUFFER_INT snow3g_f9_1_buffer_internal_sse + +%define APPEND(a,b) a %+ b + +extern SNOW3G_F9_1_BUFFER_INT + +mksection .text +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%endif + +%define state arg1 +%define job arg2 +%define job_rax rax + +%define tmp_gp0 rbx +%define tmp_gp1 rbp +%define tmp_gp2 r9 +%define tmp_gp3 r10 +%define tmp_gp4 r13 +%define init_lanes r11 +%define tmp_state r12 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Get lane nr from ptr to the list of unused lanes. +;; Remove returned lane nr from the list +;; Increase lanes in use. +;; Put job ptr in appropriate lane field in state (arg %3) +;; Assumptions: +;; In (arg %1) single lane nr takes 4 bits and 1st free lane nr is lowest 4 bits +;; Job ptr in (arg %3) takes 8 bytes +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro GET_UNUSED_LANE_SSE 6 +%define %%LANE_LIST %1 ;; [in] ptr to unused lane list +%define %%LANES_IN_USE %2 ;; [in] ptr to lanes in use count +%define %%JOB_LANES %3 ;; [in] ptr to list of jobs +%define %%JOB %4 ;; [in] ptr to job structure +%define %%LANE_NR %5 ;; [out] GP register to fill with unused lane nr +%define %%UNUSED_LANES %6 ;; [clobbered] GP register + + mov DWORD(%%UNUSED_LANES), [%%LANE_LIST] + mov DWORD(%%LANE_NR), DWORD(%%UNUSED_LANES) + and DWORD(%%LANE_NR), 0x3 + ;; remove picked lane nr from list of unused lanes + shr DWORD(%%UNUSED_LANES), 4 + mov [%%LANE_LIST], DWORD(%%UNUSED_LANES) + + add qword [%%LANES_IN_USE], 1 + mov [%%JOB_LANES + %%LANE_NR*8], %%JOB +%endmacro + +%macro SUBMIT_FLUSH_JOB_SNOW3G_UIA2 22 +%define %%SUBMIT_FLUSH %1 ;; [in] submit/flush selector +%define %%UNUSED_LANES %2 ;; [clobbered] GP register +%define %%LANE %3 ;; [clobbered] GP register +%define %%TGP0 %4 ;; [clobbered] GP register +%define %%TGP1 %5 ;; [clobbered] GP register +%define %%TGP2 %6 ;; [clobbered] GP register +%define %%TMP_XMM_0 %7 ;; [clobbered] xmm register +%define %%TMP_XMM_1 %8 ;; [clobbered] xmm register +%define %%TMP_XMM_2 %9 ;; [clobbered] xmm register +%define %%TMP_XMM_3 %10 ;; [clobbered] xmm register +%define %%TMP_XMM_4 %11 ;; [clobbered] xmm register +%define %%TMP_XMM_5 %12 ;; [clobbered] xmm register +%define %%TMP_XMM_6 %13 ;; [clobbered] xmm register +%define %%TMP_XMM_7 %14 ;; [clobbered] xmm register +%define %%TMP_XMM_8 %15 ;; [clobbered] xmm register +%define %%TMP_XMM_9 %16 ;; [clobbered] xmm register +%define %%TMP_XMM_10 %17 ;; [clobbered] xmm register +%define %%TMP_XMM_11 %18 ;; [clobbered] xmm register +%define %%TMP_XMM_12 %19 ;; [clobbered] xmm register +%define %%TMP_XMM_13 %20 ;; [clobbered] xmm register +%define %%TMP_XMM_14 %21 ;; [clobbered] xmm register +%define %%TMP_XMM_15 %22 ;; [clobbered] xmm register + + SNOW3G_FUNC_START + xor job_rax, job_rax ;; assume NULL return job + +%ifidn %%SUBMIT_FLUSH, submit + GET_UNUSED_LANE_SSE state + _snow3g_unused_lanes, \ + state + _snow3g_lanes_in_use, \ + state + _snow3g_job_in_lane, \ + job, %%LANE, %%UNUSED_LANES + + ;; copy src, key, iv and len to OOO mgr + mov %%TGP0, [job + _hash_start_src_offset_in_bytes] + add %%TGP0, [job + _src] + mov [state + _snow3g_args_in + %%LANE*8], %%TGP0 + + mov %%TGP0, [job + _snow3g_uia2_key] + mov [state + _snow3g_args_keys + %%LANE*8], %%TGP0 + + mov %%TGP0, [job + _snow3g_uia2_iv] + mov [state + _snow3g_args_IV + %%LANE*8], %%TGP0 + + mov %%TGP0, [job + _msg_len_to_hash_in_bits] + mov [state + _snow3g_lens + %%LANE*4], DWORD(%%TGP0) + + cmp qword [state + _snow3g_lanes_in_use], 4 + jne %%return_null_uia2 + + ;; all lanes full but no jobs initialized - do init + ;; at least 1 job initialized - process next job + cmp word [state + _snow3g_init_done], 0 + jz %%init_all_lanes_uia2 + + ;; find next initialized job lane + xor DWORD(%%LANE), DWORD(%%LANE) + bsf WORD(%%LANE), [state + _snow3g_init_done] + +%else ;; FLUSH + + ;; check ooo mgr empty + cmp qword [state + _snow3g_lanes_in_use], 0 + jz %%return_null_uia2 + + ;; check for initialized jobs + xor %%LANE, %%LANE + movzx DWORD(%%TGP0), word [state + _snow3g_init_done] + bsf WORD(%%LANE), WORD(%%TGP0) + jnz %%process_job_uia2 + + ;; no initialized jobs found + ;; - find valid job + ;; - copy valid job fields to empty lanes + ;; - initialize all lanes + + ;; find a valid lane + xor init_lanes, init_lanes +%assign i 0 +%rep 4 + cmp qword [state + _snow3g_job_in_lane + (i*8)], 0 + je APPEND(skip_lane_,i) + mov WORD(%%LANE), i + bts WORD(init_lanes), i ;; build init lanes mask +APPEND(skip_lane_,i): +%assign i (i+1) +%endrep + + ;; copy valid lane pointers to empty lanes + mov %%TGP0, [state + _snow3g_args_in + %%LANE*8] + mov %%TGP1, [state + _snow3g_args_keys + %%LANE*8] + mov %%TGP2, [state + _snow3g_args_IV + %%LANE*8] + +%assign i 0 +%rep 4 + bt WORD(init_lanes), i + jc APPEND(skip_lane_copy_,i) ;; skip copy for valid lanes + ;; empty lane - copy good job pointers + mov [state + _snow3g_args_in + i*8], %%TGP0 + mov [state + _snow3g_args_keys + i*8], %%TGP1 + mov [state + _snow3g_args_IV + i*8], %%TGP2 +APPEND(skip_lane_copy_,i): +%assign i (i+1) +%endrep + jmp %%init_lanes_uia2 + +%endif ;;submit/flush + +%%process_job_uia2: + ;; preserve state for function call + mov tmp_state, state + + mov arg1, [tmp_state + _snow3g_args_in + %%LANE*8] + lea arg2, [%%LANE*8] + lea arg2, [tmp_state + _snow3g_ks + arg2*4] ;; arg2*4 = %%LANE*32 + mov DWORD(arg3), [tmp_state + _snow3g_lens + %%LANE*4] + + call SNOW3G_F9_1_BUFFER_INT + + ;; restore state + mov state, tmp_state + + ;; copy digest temporarily + mov DWORD(%%TGP0), eax + +%%process_completed_job_submit_uia2: + ; process completed job "%%LANE" + ;; - decrement number of jobs in use + sub qword [state + _snow3g_lanes_in_use], 1 + mov job_rax, [state + _snow3g_job_in_lane + %%LANE*8] + mov %%UNUSED_LANES, [state + _snow3g_unused_lanes] + mov qword [state + _snow3g_job_in_lane + %%LANE*8], 0 + or dword [job_rax + _status], IMB_STATUS_COMPLETED_AUTH + ; Copy digest to auth tag output + mov %%TGP1, [job_rax + _auth_tag_output] + mov [%%TGP1], DWORD(%%TGP0) + shl %%UNUSED_LANES, 4 + or %%UNUSED_LANES, %%LANE + mov [state + _snow3g_unused_lanes], %%UNUSED_LANES + btr [state + _snow3g_init_done], WORD(%%LANE) + +%ifdef SAFE_DATA + ;; clear keystream for processed job + pxor %%TMP_XMM_0, %%TMP_XMM_0 + shl WORD(%%LANE), 5 ;; ks stored at 32 byte offsets + movdqa [state + _snow3g_ks + %%LANE], %%TMP_XMM_0 + movdqa [state + _snow3g_ks + 16 + %%LANE], %%TMP_XMM_0 +%endif + + jmp %%return_uia2 + +%%init_all_lanes_uia2: + ;; set initialized lanes mask for all 4 lanes + ;; this is used to update OOO MGR after initialization + mov DWORD(init_lanes), 0xf + +%%init_lanes_uia2: + + ;; multi-buffer init + 5 dw of KS gen + lea %%TGP0, [state + _snow3g_ks] + + SNOW3G_AUTH_INIT_5_BY_4 {state + _snow3g_args_keys}, \ + {state + _snow3g_args_IV}, \ + %%TGP0, %%TGP1, %%TGP2, \ + %%TMP_XMM_0, %%TMP_XMM_1, %%TMP_XMM_2, \ + %%TMP_XMM_3, %%TMP_XMM_4, %%TMP_XMM_5, \ + %%TMP_XMM_6, %%TMP_XMM_7, %%TMP_XMM_8, \ + %%TMP_XMM_9, %%TMP_XMM_10, %%TMP_XMM_11, \ + %%TMP_XMM_12, %%TMP_XMM_13, %%TMP_XMM_14, \ + %%TMP_XMM_15, state + + ;; update init_done for valid initialized lanes + mov [state + _snow3g_init_done], WORD(init_lanes) + bsf WORD(%%LANE), WORD(init_lanes) + + ;; process first job + jmp %%process_job_uia2 + +%%return_null_uia2: + xor job_rax, job_rax + +%%return_uia2: + SNOW3G_FUNC_END + +%endmacro + +; JOB* SUBMIT_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job) +; arg 1 : state +; arg 2 : job +MKGLOBAL(SUBMIT_JOB_SNOW3G_UIA2,function,internal) +SUBMIT_JOB_SNOW3G_UIA2: + endbranch64 + SUBMIT_FLUSH_JOB_SNOW3G_UIA2 submit, tmp_gp0, tmp_gp1, \ + tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ + xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, \ + xmm13, xmm14, xmm15 + ret + +; JOB* FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state) +; arg 1 : state +MKGLOBAL(FLUSH_JOB_SNOW3G_UIA2,function,internal) +FLUSH_JOB_SNOW3G_UIA2: + endbranch64 + SUBMIT_FLUSH_JOB_SNOW3G_UIA2 flush, tmp_gp0, tmp_gp1, \ + tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ + xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ + xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, \ + xmm13, xmm14, xmm15 + ret + +mksection stack-noexec diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index 3a588561..e939cda1 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -170,7 +170,7 @@ mksection .text struc STACK _keystream: resb (4 * 16) -_gpr_save: resq 6 +_gpr_save: resq 8 _rsp_save: resq 1 endstruc @@ -187,11 +187,12 @@ endstruc mov [rsp + _gpr_save + 8 * 2], r12 mov [rsp + _gpr_save + 8 * 3], rsi mov [rsp + _gpr_save + 8 * 4], rdi + mov [rsp + _gpr_save + 8 * 5], r13 %ifdef LINUX - mov [rsp + _gpr_save + 8 * 5], r9 + mov [rsp + _gpr_save + 8 * 6], r9 %else - mov [rsp + _gpr_save + 8 * 5], rcx - mov [rsp + _gpr_save + 8 * 6], rdx + mov [rsp + _gpr_save + 8 * 6], rcx + mov [rsp + _gpr_save + 8 * 7], rdx %endif mov [rsp + _rsp_save], rax ;; original SP %endmacro @@ -205,11 +206,12 @@ endstruc mov r12, [rsp + _gpr_save + 8 * 2] mov rsi, [rsp + _gpr_save + 8 * 3] mov rdi, [rsp + _gpr_save + 8 * 4] + mov r13, [rsp + _gpr_save + 8 * 5] %ifdef LINUX - mov r9, [rsp + _gpr_save + 8 * 5] + mov r9, [rsp + _gpr_save + 8 * 6] %else - mov rcx, [rsp + _gpr_save + 8 * 5] - mov rdx, [rsp + _gpr_save + 8 * 6] + mov rcx, [rsp + _gpr_save + 8 * 6] + mov rdx, [rsp + _gpr_save + 8 * 7] %endif mov rsp, [rsp + _rsp_save] ;; original SP %endmacro diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 17c58f72..8816cf21 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -276,6 +276,7 @@ lib_objs1 = \ $(OBJ_DIR)\mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.obj \ $(OBJ_DIR)\mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.obj \ $(OBJ_DIR)\mb_mgr_snow3g_uea2_submit_flush_x4_sse.obj \ + $(OBJ_DIR)\mb_mgr_snow3g_uia2_submit_flush_x4_sse.obj \ $(OBJ_DIR)\aes_xcbc_expand_key.obj \ $(OBJ_DIR)\md5_one_block.obj \ $(OBJ_DIR)\sha_sse.obj \ -- GitLab From 90f2410dce3ed2f20c89aad6a6f1ceb85198d402 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 15 Jun 2022 21:50:25 +0000 Subject: [PATCH 198/369] sse_t1: [SNOW3G-UIA2] Enable multi-buffer init and keystream gen --- lib/include/arch_sse_type1.h | 4 ++++ lib/include/mb_mgr_code.h | 8 ++++---- lib/sse_t1/mb_mgr_sse.c | 5 +++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index a6baa585..82676b2d 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -230,4 +230,8 @@ IMB_JOB *snow_v_aead_init_sse(IMB_JOB *job); IMB_JOB *submit_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_snow3g_uea2_sse(MB_MGR_SNOW3G_OOO *state); +IMB_JOB *submit_job_snow3g_uia2_sse(MB_MGR_SNOW3G_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_snow3g_uia2_sse(MB_MGR_SNOW3G_OOO *state); + #endif /* IMB_ARCH_SSE_TYPE1_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 4592b938..febcace4 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,7 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; -#ifdef AVX512 +#if defined (SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -718,7 +718,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, job->auth_tag_output_len_in_bytes); case IMB_AUTH_SNOW3G_UIA2_BITLEN: -#ifdef AVX512 +#if defined (SSE) || defined (AVX512) return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); #else IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) @@ -834,7 +834,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; -#ifdef AVX512 +#if defined(SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -891,7 +891,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_ZUC256_EIA3_BITLEN: return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job->auth_tag_output_len_in_bytes); -#ifdef AVX512 +#if defined(SSE) || defined (AVX512) case IMB_AUTH_SNOW3G_UIA2_BITLEN: return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); #endif diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 70e86f55..3ff9fba6 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -209,6 +209,8 @@ #define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse #define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse /* ====================================================================== */ @@ -714,6 +716,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SNOW3G-UEA out-of-order fields */ ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void -- GitLab From 1a7279102261cc02207e544c4313c98aaced424e Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 16 Jun 2022 14:29:45 +0000 Subject: [PATCH 199/369] lib: Document GCM ENC/DEC functions --- lib/intel-ipsec-mb.h | 170 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 139 insertions(+), 31 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 13a6807c..f620c8cd 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -2774,35 +2774,38 @@ IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -/** +/* * Direct GCM API. * Note that GCM is also available through job API. */ /** - * @brief GCM-AES Encryption + * @brief AES-GCM-128 Encryption. * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param out Ciphertext output. Encrypt in-place is allowed. - * @param in Plaintext input. - * @param len Length of data in Bytes for encryption. - * @param iv pointer to 12 byte IV structure. Internally, library - * concates 0x00000001 value to it. - * @param aad Additional Authentication Data (AAD). - * @param aad_len Length of AAD. - * @param auth_tag Authenticated Tag output. - * @param auth_tag_len Authenticated Tag Length in bytes (must be - * a multiple of 4 bytes). Valid values are - * 16 (most likely), 12 or 8. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 */ - IMB_DLL_EXPORT void aes_gcm_enc_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_128_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2810,6 +2813,9 @@ aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_128_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2817,13 +2823,33 @@ aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); - +/** + * @brief AES-GCM-192 Encryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 + */ IMB_DLL_EXPORT void aes_gcm_enc_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_192_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2831,6 +2857,9 @@ aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_192_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2839,6 +2868,24 @@ aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief AES-GCM-256 Encryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 + */ IMB_DLL_EXPORT void aes_gcm_enc_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2846,6 +2893,9 @@ aes_gcm_enc_256_sse(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_256_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2853,6 +2903,9 @@ aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_256_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2862,21 +2915,22 @@ aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data, uint8_t *auth_tag, uint64_t auth_tag_len); /** - * @brief GCM-AES Decryption + * @brief AES-GCM-128 Decryption. * - * @param key_data GCM expanded keys data - * @param context_data GCM operation context data - * @param out Plaintext output. Decrypt in-place is allowed. - * @param in Ciphertext input. - * @param len Length of data in Bytes for decryption. - * @param iv pointer to 12 byte IV structure. Internally, library - * concates 0x00000001 value to it. - * @param aad Additional Authentication Data (AAD). - * @param aad_len Length of AAD. - * @param auth_tag Authenticated Tag output. - * @param auth_tag_len Authenticated Tag Length in bytes (must be - * a multiple of 4 bytes). Valid values are - * 16 (most likely), 12 or 8. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 */ IMB_DLL_EXPORT void aes_gcm_dec_128_sse(const struct gcm_key_data *key_data, @@ -2884,6 +2938,9 @@ aes_gcm_dec_128_sse(const struct gcm_key_data *key_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_128_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2891,6 +2948,9 @@ aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_128_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2899,12 +2959,33 @@ aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief AES-GCM-192 Decryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 + */ IMB_DLL_EXPORT void aes_gcm_dec_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_192_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2912,6 +2993,9 @@ aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_192_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2920,12 +3004,33 @@ aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief AES-GCM-256 Decryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authentication Data (AAD) + * @param [in] aad_len Length of AAD in bytes + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are 16 + * (most likely), 12 or 8 + */ IMB_DLL_EXPORT void aes_gcm_dec_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_256_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, @@ -2933,6 +3038,9 @@ aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_256_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, -- GitLab From 104f815e19d1cfe930f1bbe183acc4876183891a Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 16 Jun 2022 14:40:11 +0000 Subject: [PATCH 200/369] lib: Document GCM UPDATE functions --- lib/intel-ipsec-mb.h | 96 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 84 insertions(+), 12 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index f620c8cd..54716216 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -3106,96 +3106,168 @@ aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data, uint8_t const *aad, uint64_t aad_len); /** - * @brief encrypt a block of a AES-GCM Encryption message + * @brief Encrypt a block of a AES-GCM-128 encryption message. * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param out Ciphertext output. Encrypt in-place is allowed. - * @param in Plaintext input. - * @param len Length of data in Bytes for decryption. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption */ IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_128_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_128_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @brief Encrypt a block of a AES-GCM-192 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption + */ IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_192_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_192_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @brief Encrypt a block of a AES-GCM-256 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Ciphertext output. Encrypt in-place is allowed + * @param [in] in Plaintext input + * @param [in] len Length of data in bytes for encryption + */ IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_256_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_enc_256_update_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); /** - * @brief decrypt a block of a AES-GCM Encryption message + * @brief Decrypt a block of a AES-GCM-128 encryption message. * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param out Plaintext output. Decrypt in-place is allowed. - * @param in Ciphertext input. - * @param len Length of data in Bytes for decryption. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption */ IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_128_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_128_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @brief Decrypt a block of a AES-GCM-192 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption + */ IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_192_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_192_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @brief Decrypt a block of a AES-GCM-256 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] out Plaintext output. Decrypt in-place is allowed + * @param [in] in Ciphertext input + * @param [in] len Length of data in bytes for decryption + */ IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_256_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len); +/** + * @copydoc aes_gcm_dec_256_update_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, -- GitLab From 393194662780b070305f0373fdb53a9ca9473279 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 16 Jun 2022 14:46:56 +0000 Subject: [PATCH 201/369] lib: Document GCM FINALIZE functions --- lib/intel-ipsec-mb.h | 104 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 14 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 54716216..4ee0d6fe 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -3274,98 +3274,174 @@ aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data, uint8_t *out, const uint8_t *in, uint64_t len); /** - * @brief End encryption of a AES-GCM Encryption message + * @brief End encryption of a AES-GCM-128 encryption message. * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param auth_tag Authenticated Tag output. - * @param auth_tag_len Authenticated Tag Length in bytes (must be - * a multiple of 4 bytes). Valid values are - * 16 (most likely), 12 or 8. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. */ IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_128_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_128_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief End encryption of a AES-GCM-192 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. + */ IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_192_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_192_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief End encryption of a AES-GCM-256 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. + */ IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_256_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_enc_256_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); /** - * @brief End decryption of a AES-GCM Encryption message + * @brief End decryption of a AES-GCM-128 encryption message. * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param auth_tag Authenticated Tag output. - * @param auth_tag_len Authenticated Tag Length in bytes (must be - * a multiple of 4 bytes). Valid values are - * 16 (most likely), 12 or 8. + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. */ IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_128_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_128_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief End decryption of a AES-GCM-192 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. + */ IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_192_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_192_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @brief End decryption of a AES-GCM-256 encryption message. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [out] auth_tag Authenticated Tag output + * @param [in] auth_tag_len Authenticated Tag Length in bytes (must be + * a multiple of 4 bytes). Valid values are + * 16 (most likely), 12 or 8. + */ IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_256_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len); +/** + * @copydoc aes_gcm_dec_256_finalize_sse + */ IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, -- GitLab From 83cc0523fa91634ef7dc5cf9d2d368e05b27bbbc Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 16 Jun 2022 14:55:34 +0000 Subject: [PATCH 202/369] lib: Document GCM PRE/PRECOMP functions --- lib/intel-ipsec-mb.h | 87 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 4ee0d6fe..957f1728 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -3448,51 +3448,126 @@ aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data, uint8_t *auth_tag, uint64_t auth_tag_len); /** - * @brief Precomputation of HashKey constants + * @brief Precomputation of AES-GCM-128 HashKey constants. * * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and * shifted_hkey_X_k). * - * @param [in] key_data GCM key data + * @param [in,out] key_data GCM key data */ IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_128_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_128_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data); +/** + * @brief Precomputation of AES-GCM-192 HashKey constants. + * + * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and + * shifted_hkey_X_k). + * + * @param [in,out] key_data GCM key data + */ IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_192_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_192_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data); +/** + * @brief Precomputation of AES-GCM-256 HashKey constants. + * + * Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and + * shifted_hkey_X_k). + * + * @param [in,out] key_data GCM key data + */ IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_256_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data); + +/** + * @copydoc aes_gcm_precomp_256_sse + */ IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data); /** - * @brief Pre-processes GCM key data + * @brief Pre-processes AES-GCM-128 key data. * * Prefills the gcm key data with key values for each round and * the initial sub hash key for tag encoding * - * @param key pointer to key data - * @param key_data GCM expanded key data - * + * @param [in] key Pointer to key data + * @param [out] key_data GCM expanded key data */ IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_128_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_128_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key, struct gcm_key_data *key_data); +/** + * @brief Pre-processes AES-GCM-192 key data. + * + * Prefills the gcm key data with key values for each round and + * the initial sub hash key for tag encoding + * + * @param [in] key Pointer to key data + * @param [out] key_data GCM expanded key data + */ IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_192_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_192_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key, struct gcm_key_data *key_data); +/** + * @brief Pre-processes AES-GCM-256 key data. + * + * Prefills the gcm key data with key values for each round and + * the initial sub hash key for tag encoding + * + * @param [in] key Pointer to key data + * @param [out] key_data GCM expanded key data + */ IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_256_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key, struct gcm_key_data *key_data); +/** + * @copydoc aes_gcm_pre_256_sse + */ IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data); -- GitLab From 9920930188f3b391056f9402ab5bb97fe916e34f Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 27 Jun 2022 20:06:51 +0000 Subject: [PATCH 203/369] Update release notes for burst API --- ReleaseNotes.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 86efa10e..63adef85 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -11,16 +11,21 @@ Library - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added - SHA1/224/256 multi-buffer implementation added +- Burst API added +- Cipher-only burst API added (AES-CBC/CTR support only) +- Hash-only burst API added (HMAC-SHA1/224/256/384/512 support only) Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) Test Applications - GHASH JOB API support added in the test application, fuzzing and xvalid tools +- Burst API support added for supported algorithms Performance Application - GHASH support added (through JOB and direct API) - Support added for SHA1/224/256/384/512 +- Burst API support added for supported algorithms Fixes - Fixed incorrect 8-buffer SNOW3G keystream generation -- GitLab From edb3c470f736d0d1c89481322a2d1a5af0ee48d7 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 28 Jun 2022 13:08:28 +0000 Subject: [PATCH 204/369] Update release notes for SSE SNOW3G optimizations --- README | 2 +- README.md | 2 +- ReleaseNotes.txt | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README b/README index 78ab12f7..9e91ef44 100644 --- a/README +++ b/README @@ -175,7 +175,7 @@ Notes: - CRC7: FP header - CRC6: IUUP header (7) - used only with PON-AES128-CTR cipher -(8) - x16 for init keystream generation, then by32 +(8) - x4/x16 for init keystream generation, then by4/by32 Legend: byY- single buffer Y blocks at a time diff --git a/README.md b/README.md index 1c7d9d18..9b7b89f5 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Notes: - CRC7: FP header - CRC6: IUUP header (7) - used only with PON-AES128-CTR cipher -(8) - x16 for init keystream generation, then by32 +(8) - x4/x16 for init keystream generation, then by4/by32 Legend: ` byY`- single buffer Y blocks at a time diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 63adef85..a18b262e 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -14,6 +14,8 @@ Library - Burst API added - Cipher-only burst API added (AES-CBC/CTR support only) - Hash-only burst API added (HMAC-SHA1/224/256/384/512 support only) +- SNOW3G-UEA2 SSE multi-buffer implementation added +- SNOW3G-UIA2 SSE multi-buffer initialization and keystream generation added Fixes - Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) -- GitLab From a91acaf48de59fa0c2fde6c3689370ce40f869a0 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 29 Jun 2022 16:09:08 +0000 Subject: [PATCH 205/369] lib: Document arch specific JOB API --- lib/intel-ipsec-mb.h | 167 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 145 insertions(+), 22 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 957f1728..2fb79ae8 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1265,7 +1265,7 @@ IMB_DLL_EXPORT const char *imb_get_strerror(int errnum); * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. * - * @return Pointer to allocated memory for MB_MGR structure + * @return Pointer to allocated memory for IMB_MGR structure * @retval NULL on allocation error */ IMB_DLL_EXPORT IMB_MGR *alloc_mb_mgr(uint64_t flags); @@ -1316,36 +1316,159 @@ IMB_DLL_EXPORT IMB_MGR *imb_set_pointers_mb_mgr(void *ptr, const uint64_t flags, */ IMB_DLL_EXPORT uint64_t imb_get_feature_flags(void); +/** + * @brief Initialize Multi-Buffer Manager structure. + * + * Must be called before calling JOB/BURST API. + * + * @param [in,out] state Pointer to IMB_MGR structure + * For binary compatibility between library versions, it + * is recommended to allocate the IMB_MGR structure using + * the alloc_mb_mgr() API + */ IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_avx(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *flush_job_avx(IMB_MGR *state); -IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_next_job_avx(IMB_MGR *state); - +/** + * @copydoc init_mb_mgr_avx + */ IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_avx2(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *flush_job_avx2(IMB_MGR *state); -IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2(IMB_MGR *state); - +/** + * @copydoc init_mb_mgr_avx + */ IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_avx512(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *flush_job_avx512(IMB_MGR *state); -IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512(IMB_MGR *state); -IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512(IMB_MGR *state); - +/** + * @copydoc init_mb_mgr_avx + */ IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state); + + +/** + * @brief Submit job for processing after validating. + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no job completed + * If NULL, imb_get_errno() can be used to check for potential + * error conditions + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx(IMB_MGR *state); +/** + * @copydoc submit_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx2(IMB_MGR *state); +/** + * @copydoc submit_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx512(IMB_MGR *state); +/** + * @copydoc submit_job_avx + */ IMB_DLL_EXPORT IMB_JOB *submit_job_sse(IMB_MGR *state); + +/** + * @brief Submit job for processing without validating. + * + * This is more performant but less secure than submit_job_xxx() + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no job completed + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state); +/** + * @copydoc submit_job_nocheck_avx + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state); +/** + * @copydoc submit_job_nocheck_avx + */ +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state); +/** + * @copydoc submit_job_nocheck_avx + */ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse(IMB_MGR *state); + +/** + * @brief Force processing until next job in queue is completed. + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no more jobs to process + */ +IMB_DLL_EXPORT IMB_JOB *flush_job_avx(IMB_MGR *state); +/** + * @copydoc flush_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *flush_job_avx2(IMB_MGR *state); +/** + * @copydoc flush_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *flush_job_avx512(IMB_MGR *state); +/** + * @copydoc flush_job_avx + */ IMB_DLL_EXPORT IMB_JOB *flush_job_sse(IMB_MGR *state); + +/** + * @brief Get number of jobs queued to be processed. + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Number of jobs in the queue + */ +IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state); +/** + * @copydoc queue_size_avx + */ +IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state); +/** + * @copydoc queue_size_avx + */ +IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state); +/** + * @copydoc queue_size_avx + */ IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state); + +/** + * @brief Get next completed job. + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if next job not complete + */ +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx(IMB_MGR *state); +/** + * @copydoc get_completed_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2(IMB_MGR *state); +/** + * @copydoc get_completed_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512(IMB_MGR *state); +/** + * @copydoc get_completed_job_avx + */ IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state); + +/** + * @brief Get next available job. + * + * @param [in,out] state Pointer to initialized IMB_MGR structure + * + * @return Pointer to next free IMB_JOB in the queue + */ +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx(IMB_MGR *state); +/** + * @copydoc get_next_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2(IMB_MGR *state); +/** + * @copydoc get_next_job_avx + */ +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512(IMB_MGR *state); +/** + * @copydoc get_next_job_avx + */ IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state); /** -- GitLab From 22c04751ccbd1ac295e5e95eb62181ab9fa2ce72 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 29 Jun 2022 16:10:29 +0000 Subject: [PATCH 206/369] lib: Document CFB 1 block API --- lib/intel-ipsec-mb.h | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 2fb79ae8..5c1400e4 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1705,9 +1705,21 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); #define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag) \ ((_mgr)->md5_one_block((_src), (_tag))) -/* AES-CFB API */ -#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _enc_exp_key, _len) \ - ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_enc_exp_key), (_len))) +/** + * @brief AES-CFB-128 Encrypt/Decrypt up to one block. + * + * Processes only one buffer at a time. + * Designed to manage partial blocks of DOCSIS 3.1 SEC BPI. + * + * @param [in] _mgr Pointer to multi-buffer structure + * @param [out] _dst Plaintext/Ciphertext output + * @param [in] _src Plaintext/Ciphertext input + * @param [in] _iv Pointer to 16 byte IV + * @param [in] _exp_key Pointer to expanded AES keys + * @param [in] _len Length of data in bytes + */ +#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \ + ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len))) /* AES-GCM API's */ #define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \ @@ -2883,16 +2895,36 @@ IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, */ IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, void *key2); - +/** + * @brief AES-CFB-128 Encrypt/Decrypt up to one block. + * + * Processes only one buffer at a time. + * Designed to manage partial blocks of DOCSIS 3.1 SEC BPI. + * + * @param [out] out Plaintext/Ciphertext output + * @param [in] in Plaintext/Ciphertext input + * @param [in] iv Pointer to 16 byte IV + * @param [in] keys Pointer to expanded AES keys + * @param [in] len Length of data in bytes + */ IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, const void *iv, const void *keys, uint64_t len); +/** + * @copydoc aes_cfb_128_one_sse + */ IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, const void *iv, const void *keys, uint64_t len); +/** + * @copydoc aes_cfb_128_one_sse + */ IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, const void *iv, const void *keys, uint64_t len); +/** + * @copydoc aes_cfb_128_one_sse + */ IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, const void *iv, const void *keys, uint64_t len); -- GitLab From ab3484213a2c0f3cb7514afd1d4a258cadcb2dae Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Wed, 29 Jun 2022 16:11:09 +0000 Subject: [PATCH 207/369] lib: Document GCM_INIT API --- lib/intel-ipsec-mb.h | 61 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 5c1400e4..1db263c1 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -3205,55 +3205,96 @@ aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data, uint8_t *auth_tag, uint64_t auth_tag_len); /** - * @brief Start a AES-GCM Encryption message - * - * @param key_data GCM expanded key data - * @param context_data GCM operation context data - * @param iv pointer to 12 byte IV structure. Internally, library - * concates 0x00000001 value to it. - * @param aad Additional Authentication Data (AAD). - * @param aad_len Length of AAD. + * @brief Initialize a gcm_context_data structure to prepare for + * AES-GCM-128 Encryption. * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authenticated Data (AAD) + * @param [in] aad_len Length of AAD in bytes */ IMB_DLL_EXPORT void aes_gcm_init_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_128_sse + */ IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_128_sse + */ IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); - +/** + * @brief Initialize a gcm_context_data structure to prepare for + * AES-GCM-192 Encryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authenticated Data (AAD) + * @param [in] aad_len Length of AAD in bytes + */ IMB_DLL_EXPORT void aes_gcm_init_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_192_sse + */ IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_192_sse + */ IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); - +/** + * @brief Initialize a gcm_context_data structure to prepare for + * AES-GCM-256 Encryption. + * + * @param [in] key_data GCM expanded key data + * @param [in,out] context_data GCM operation context data + * @param [in] iv Pointer to 12 byte IV structure + * Internally, the library concatenates 0x00000001 + * to the IV + * @param [in] aad Additional Authenticated Data (AAD) + * @param [in] aad_len Length of AAD in bytes + */ IMB_DLL_EXPORT void aes_gcm_init_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_256_sse + */ IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len); +/** + * @copydoc aes_gcm_init_256_sse + */ IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, -- GitLab From be0e54bdbdd68aa5c0ac63654581912321039077 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 30 Jun 2022 09:15:14 +0000 Subject: [PATCH 208/369] lib: Document JOB API macros --- lib/intel-ipsec-mb.h | 130 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 3 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 1db263c1..d82a6a95 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1481,7 +1481,7 @@ IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state); */ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); -/** +/* * Wrapper macros to call arch API's set up * at init phase of multi-buffer manager. * @@ -1503,26 +1503,150 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); * it can simplify application implementation. * The test app provides example of using the indirect interface. */ -#define IMB_GET_NEXT_JOB(_mgr) ((_mgr)->get_next_job((_mgr))) -#define IMB_SUBMIT_JOB(_mgr) ((_mgr)->submit_job((_mgr))) + +/** + * @brief Get next available job. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Pointer to next free IMB_JOB in the queue + */ +#define IMB_GET_NEXT_JOB(_mgr) ((_mgr)->get_next_job((_mgr))) + +/** + * @brief Submit job for processing after validating. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no job completed + * If NULL, imb_get_errno() can be used to check for potential + * error conditions + */ +#define IMB_SUBMIT_JOB(_mgr) ((_mgr)->submit_job((_mgr))) + +/** + * @brief Submit job for processing without validating. + * + * This is more performant but less secure than submit_job_xxx() + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no job completed + */ #define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr))) + +/** + * @brief Get next completed job. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if next job not complete + */ #define IMB_GET_COMPLETED_JOB(_mgr) ((_mgr)->get_completed_job((_mgr))) + +/** + * @brief Force processing until next job in queue is completed. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Pointer to completed IMB_JOB or NULL if no more jobs to process + */ #define IMB_FLUSH_JOB(_mgr) ((_mgr)->flush_job((_mgr))) + +/** + * @brief Get number of jobs queued to be processed. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * + * @return Number of jobs in the queue + */ #define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr))) + +/** + * Submit multiple jobs to be processed synchronously after validating. + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_BURST(_mgr, _jobs, _n_jobs) \ ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs))) + +/** + * Submit multiple jobs to be processed synchronously without validating. + * + * This is more performant but less secure than IMB_SUBMIT_BURST(). + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs) \ ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs))) + +/** + * Submit multiple cipher jobs to be processed synchronously after validating. + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * @param [in] _cipher Cipher algorithm of type #IMB_CIPHER_MODE + * @param [in] _dir Cipher direction of type #IMB_CIPHER_DIRECTION + * @param [in] _key_size Key size in bytes of type #IMB_KEY_SIZE_BYTES + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, \ _dir, _key_size) \ ((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \ (_cipher), (_dir), (_key_size))) +/** + * Submit multiple cipher jobs to be processed synchronously without validating. + * + * This is more performant but less secure than IMB_SUBMIT_CIPHER_BURST(). + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * @param [in] _cipher Cipher algorithm of type #IMB_CIPHER_MODE + * @param [in] _dir Cipher direction of type #IMB_CIPHER_DIRECTION + * @param [in] _key_size Key size in bytes of type #IMB_KEY_SIZE_BYTES + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, \ _dir, _key_size) \ ((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\ (_cipher), (_dir), (_key_size))) +/** + * Submit multiple hash jobs to be processed synchronously after validating. + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * @param [in] _hash Hash algorithm of type #IMB_HASH_ALG + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash) \ ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash))) + +/** + * Submit multiple hash jobs to be processed synchronously without validating. + * + * This is more performant but less secure than IMB_SUBMIT_HASH_BURST(). + * + * @param [in] _mgr Pointer to initialized IMB_MGR structure + * @param [in,out] _jobs Pointer to array of IMB_JOB structures + * @param [in] _n_jobs Number of jobs to process + * @param [in] _hash Hash algorithm of type #IMB_HASH_ALG + * + * @return Number of completed jobs + */ #define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash) \ ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash))) -- GitLab From 62a24d7a5e351abd4336f48c4b18e97c24c1e264 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 30 Jun 2022 09:49:35 +0000 Subject: [PATCH 209/369] lib: Update key/IV gen API documentation --- lib/intel-ipsec-mb.h | 90 +++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index d82a6a95..637ed0b2 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -2561,12 +2561,12 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); /* Auxiliary functions */ /** - * @brief DES key schedule set up + * @brief DES key schedule set up. * * \a ks buffer needs to accommodate \a DES_KEY_SCHED_SIZE (128) bytes of data. * - * @param ks destination buffer to accommodate DES key schedule - * @param key a pointer to an 8 byte DES key + * @param[out] ks Destination buffer to accommodate DES key schedule + * @param[in] key Pointer to an 8 byte DES key * * @return Operation status * @retval 0 success @@ -3892,90 +3892,102 @@ IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data); /** - * @brief Generation of ZUC Initialization Vectors (for EEA3 and EIA3) + * @brief Generation of ZUC-EEA3 Initialization Vector. * - * @param [in] count COUNT (4 bytes in Little Endian) - * @param [in] bearer BEARER (5 bits) - * @param [in] dir DIRECTION (1 bit) + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] bearer BEARER (5 bits) + * @param [in] dir DIRECTION (1 bit) * @param [out] iv_ptr Pointer to generated IV (16 bytes) * - * @return - * - 0 if success - * - 1 if one or more parameters are wrong + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid */ IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr); +/** + * @brief Generation of ZUC-EIA3 Initialization Vector. + * + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] bearer BEARER (5 bits) + * @param [in] dir DIRECTION (1 bit) + * @param [out] iv_ptr Pointer to generated IV (16 bytes) + * + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid + */ IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr); /** - * @brief Generation of KASUMI F8 Initialization Vector + * @brief Generation of KASUMI F8 Initialization Vector. * - * @param [in] count COUNT (4 bytes in Little Endian) - * @param [in] bearer BEARER (5 bits) - * @param [in] dir DIRECTION (1 bit) + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] bearer BEARER (5 bits) + * @param [in] dir DIRECTION (1 bit) * @param [out] iv_ptr Pointer to generated IV (16 bytes) * - * @return - * - 0 if success - * - 1 if one or more parameters are wrong + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid */ IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr); /** - * @brief Generation of KASUMI F9 Initialization Vector + * @brief Generation of KASUMI F9 Initialization Vector. * - * @param [in] count COUNT (4 bytes in Little Endian) - * @param [in] fresh FRESH (4 bytes in Little Endian) + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] fresh FRESH (4 bytes in Little Endian) * @param [out] iv_ptr Pointer to generated IV (16 bytes) * - * @return - * - 0 if success - * - 1 if one or more parameters are wrong + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid */ IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count, const uint32_t fresh, void *iv_ptr); /** - * @brief Generation of SNOW3G F8 Initialization Vector + * @brief Generation of SNOW3G F8 Initialization Vector. * * Parameters are passed in Little Endian format and - * used to generate the IV in Big Endian format + * used to generate the IV in Big Endian format. * - * @param [in] count COUNT (4 bytes in Little Endian) - * @param [in] bearer BEARER (5 bits) - * @param [in] dir DIRECTION (1 bit) + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] bearer BEARER (5 bits) + * @param [in] dir DIRECTION (1 bit) * @param [out] iv_ptr Pointer to generated IV (16 bytes) in Big Endian format * - * @return - * - 0 if success - * - 1 if one or more parameters are wrong + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid */ IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr); /** - * @brief Generation of SNOW3G F9 Initialization Vector + * @brief Generation of SNOW3G F9 Initialization Vector. * * Parameters are passed in Little Endian format and - * used to generate the IV in Big Endian format + * used to generate the IV in Big Endian format. * - * @param [in] count COUNT (4 bytes in Little Endian) - * @param [in] fresh FRESH (4 bytes in Little Endian) - * @param [in] dir DIRECTION (1 bit) + * @param [in] count COUNT (4 bytes in Little Endian) + * @param [in] fresh FRESH (4 bytes in Little Endian) + * @param [in] dir DIRECTION (1 bit) * @param [out] iv_ptr Pointer to generated IV (16 bytes) in Big Endian format * - * @return - * - 0 if success - * - 1 if one or more parameters are wrong + * @return Operation status + * @retval 0 success + * @retval -1 if one or more parameters are invalid */ IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count, const uint32_t fresh, -- GitLab From 4fda2df9b14d1d48165d0a15e068ca34e799dbd2 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 30 Jun 2022 15:50:39 +0000 Subject: [PATCH 210/369] sse: [SHA1-NI] expose multi-buffer API --- lib/avx/sha_mb_avx.c | 4 +- lib/avx2_t1/sha_mb_avx2.c | 4 +- lib/avx512_t1/sha_mb_avx512.c | 4 +- lib/include/arch_sse_type2.h | 4 ++ lib/include/mb_mgr_code.h | 8 ++++ lib/include/sha_mb_mgr.h | 61 ++++++++++++++++++------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 + lib/sse_t1/mb_mgr_sse.c | 13 +++++- lib/sse_t1/sha_mb_sse.c | 28 +++++++++++- lib/sse_t2/sha1_ni_x2_sse.asm | 72 ++++++++++++++++++++++++++++++ lib/x86_64/ooo_mgr_reset.c | 5 ++- 11 files changed, 176 insertions(+), 29 deletions(-) diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index 03bad7cf..059fe945 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -46,7 +46,7 @@ IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_avx_from_c); + call_sha1_mult_avx_from_c, 0); } IMB_DLL_LOCAL @@ -54,7 +54,7 @@ IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_avx_from_c); + call_sha1_mult_avx_from_c, 0); } /* ========================================================================== */ diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index a54e60db..2f9e5ee5 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -46,7 +46,7 @@ IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 8, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_x8_avx2_from_c); + call_sha1_x8_avx2_from_c, 0); } IMB_DLL_LOCAL @@ -54,7 +54,7 @@ IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 8, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_x8_avx2_from_c); + call_sha1_x8_avx2_from_c, 0); } /* ========================================================================== */ diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index 56ed096e..c787dd2d 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -46,7 +46,7 @@ IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 16, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_x16_avx512_from_c); + call_sha1_x16_avx512_from_c, 0); } IMB_DLL_LOCAL @@ -54,7 +54,7 @@ IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 16, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_x16_avx512_from_c); + call_sha1_x16_avx512_from_c, 0); } /* ========================================================================== */ diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h index 32a7e377..85a2b143 100644 --- a/lib/include/arch_sse_type2.h +++ b/lib/include/arch_sse_type2.h @@ -47,4 +47,8 @@ IMB_JOB *submit_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_256_ni_sse(MB_MGR_HMAC_SHA_256_OOO *state); +IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + +IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + #endif /* IMB_ARCH_SSE_TYPE2_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index febcace4..5ed9d425 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -695,6 +695,10 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return SUBMIT_JOB_SHA1_NI(sha_1_ooo, job); +#endif return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: return SUBMIT_JOB_SHA224(sha_224_ooo, job); @@ -864,6 +868,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return FLUSH_JOB_SHA1_NI(sha_1_ooo, job); +#endif return FLUSH_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index c4225f88..53fdba36 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -53,16 +53,24 @@ extern void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, extern void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, + uint32_t size_in_blocks); __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, - const size_t offset, const unsigned lane) + const size_t offset, const unsigned lane, + const int shani) { uint32_t *outp = (uint32_t *) dst; const uint32_t *inp = (const uint32_t *) src; size_t i; - for (i = 0; i < num; i++) - outp[i] = bswap4(inp[lane + i*offset]); + if (shani) { + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[5*lane + i*offset]); + } else { + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[lane + i*offset]); + } } __forceinline @@ -75,6 +83,16 @@ void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) digest[lane + 4*16] = H4; } +__forceinline +void sha1_ni_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[5*lane + 0] = H0; + digest[5*lane + 1] = H1; + digest[5*lane + 2] = H2; + digest[5*lane + 3] = H3; + digest[5*lane + 4] = H4; +} + __forceinline void sha224_mb_init_digest(uint32_t *digest, const unsigned lane) { @@ -103,11 +121,15 @@ void sha256_mb_init_digest(uint32_t *digest, const unsigned lane) __forceinline void -sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) +sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane, + const int shani) { - if (sha_type == 1) - sha1_mb_init_digest(digest, lane); - else if (sha_type == 224) + if (sha_type == 1){ + if (shani) + sha1_ni_mb_init_digest(digest, lane); + else + sha1_mb_init_digest(digest, lane); + } else if (sha_type == 224) sha224_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_mb_init_digest(digest, lane); @@ -116,17 +138,17 @@ sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) __forceinline void sha_mb_generic_write_digest(void *dst, const void *src, const int sha_type, const size_t offset, - const unsigned lane) + const unsigned lane, const int shani) { if (sha_type == 1) copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, offset, - lane); + lane, shani); else if (sha_type == 224) copy_bswap4_array_mb(dst, src, NUM_SHA_224_DIGEST_WORDS, offset, - lane); + lane, shani); else if (sha_type == 256) copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, - lane); + lane, shani); } __forceinline @@ -181,7 +203,7 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, const unsigned max_jobs, const int is_submit, const int sha_type, const uint64_t blk_size, const uint64_t pad_size, - void (*fn)(SHA1_ARGS *, uint32_t)) + void (*fn)(SHA1_ARGS *, uint32_t), const int shani) { unsigned lane, min_idx; uint64_t min_len; @@ -200,7 +222,7 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; - sha_mb_generic_init(state->args.digest, sha_type, lane); + sha_mb_generic_init(state->args.digest, sha_type, lane, shani); /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; @@ -291,8 +313,13 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; - sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx); + if (shani) { + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 1, min_idx, shani); + } else { + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 16, min_idx, shani); + } ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; @@ -324,7 +351,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; - sha_mb_generic_init(state->args.digest, sha_type, lane); + sha_mb_generic_init(state->args.digest, sha_type, lane, 0); /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; @@ -415,7 +442,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx); + state->args.digest, sha_type, 16, min_idx, 0); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; return ret_job; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 08c0ab07..fb6544de 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -92,6 +92,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_sse #define SUBMIT_JOB_SHA256 submit_job_sha256_sse #define FLUSH_JOB_SHA256 flush_job_sha256_sse +#define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse +#define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 3ff9fba6..1ebddb10 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -98,6 +98,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_sse #define SUBMIT_JOB_SHA256 submit_job_sha256_sse #define FLUSH_JOB_SHA256 flush_job_sha256_sse +#define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse +#define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse @@ -705,8 +707,15 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init AES-CBCS out-of-order fields */ ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); - /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + /* Init SHA1 NI out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, 2); + } else { + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); + } +#endif /* HASH_USE_SHAEXT */ /* Init SHA224 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 0029d9a3..425bfc59 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -36,6 +36,9 @@ IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -46,7 +49,7 @@ IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 1, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_sse_from_c); + call_sha1_mult_sse_from_c, 0); } IMB_DLL_LOCAL @@ -54,7 +57,7 @@ IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 0, 1, IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_sse_from_c); + call_sha1_mult_sse_from_c, 0); } /* ========================================================================== */ @@ -98,3 +101,24 @@ IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha_256_mult_sse_from_c); } + +/* ========================================================================== */ +/* + * SHA1-NI MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 2, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_ni_x2_sse_from_c, 1); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 2, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_ni_x2_sse_from_c, 1); +} diff --git a/lib/sse_t2/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm index f726ea75..f32f1956 100644 --- a/lib/sse_t2/sha1_ni_x2_sse.asm +++ b/lib/sse_t2/sha1_ni_x2_sse.asm @@ -106,6 +106,70 @@ UPPER_WORD_MASK: ;ddq 0xFFFFFFFF000000000000000000000000 ;; arg2 : size (in blocks) ;; assumed to be >= 1 mksection .text + +%define XMM_STORAGE 16*10 +%define GP_STORAGE 8*5 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes +%ifndef LINUX + movdqa [rsp + 0*16], xmm6 + movdqa [rsp + 1*16], xmm7 + movdqa [rsp + 2*16], xmm8 + movdqa [rsp + 3*16], xmm9 + movdqa [rsp + 4*16], xmm10 + movdqa [rsp + 5*16], xmm11 + movdqa [rsp + 6*16], xmm12 + movdqa [rsp + 7*16], xmm13 + movdqa [rsp + 8*16], xmm14 + movdqa [rsp + 9*16], xmm15 + mov [rsp + GP_OFFSET], rdi + mov [rsp + GP_OFFSET + 8], rsi +%endif + mov [rsp + GP_OFFSET + 2*8], rbx + mov [rsp + GP_OFFSET + 3*8], rbp + mov [rsp + GP_OFFSET + 4*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 +%ifndef LINUX + movdqa xmm6, [rsp + 0*16] + movdqa xmm7, [rsp + 1*16] + movdqa xmm8, [rsp + 2*16] + movdqa xmm9, [rsp + 3*16] + movdqa xmm10, [rsp + 4*16] + movdqa xmm11, [rsp + 5*16] + movdqa xmm12, [rsp + 6*16] + movdqa xmm13, [rsp + 7*16] + movdqa xmm14, [rsp + 8*16] + movdqa xmm15, [rsp + 9*16] + mov rdi, [rsp + GP_OFFSET] + mov rsi, [rsp + GP_OFFSET + 8] + +%ifdef SAFE_DATA + pxor xmm5, xmm5 + movdqa [rsp + 0*16], xmm5 + movdqa [rsp + 1*16], xmm5 + movdqa [rsp + 2*16], xmm5 + movdqa [rsp + 3*16], xmm5 + movdqa [rsp + 4*16], xmm5 + movdqa [rsp + 5*16], xmm5 + movdqa [rsp + 6*16], xmm5 + movdqa [rsp + 7*16], xmm5 + movdqa [rsp + 8*16], xmm5 + movdqa [rsp + 9*16], xmm5 +%endif + mov rbx, [rsp + GP_OFFSET + 2*8] + mov rbp, [rsp + GP_OFFSET + 3*8] +%endif + mov rsp, [rsp + GP_OFFSET + 4*8] ;; rsp pointer +%endmacro + MKGLOBAL(sha1_ni,function,internal) align 32 sha1_ni: @@ -489,4 +553,12 @@ done_hash: ret +; void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha1_ni_x2_sse_from_c,function,internal) +call_sha1_ni_x2_sse_from_c: + FUNC_SAVE + call sha1_ni + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 2daec121..c29bda34 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -329,10 +329,11 @@ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SHA_1_OOO *p_mgr = (MB_MGR_SHA_1_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); - if (num_lanes == AVX_NUM_SHA1_LANES) + if (num_lanes == 2) + p_mgr->unused_lanes = 0xF10; /* SHANI */ + else if (num_lanes == AVX_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xF3210; else if (num_lanes == AVX2_NUM_SHA1_LANES) p_mgr->unused_lanes = 0xF76543210; -- GitLab From 2da4528740ae6fb568f591a764c2370639238135 Mon Sep 17 00:00:00 2001 From: Kamila Lipinska Date: Tue, 5 Jul 2022 14:40:09 +0000 Subject: [PATCH 211/369] sse_t1: [SNOW3G-UEA2]: Fix flush for last byte processing --- ...mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 60 ++++++++++--------- lib/sse_t1/snow3g_uea2_by4_sse.asm | 5 +- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index 05f031f6..3f9cae16 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -248,48 +248,52 @@ mksection .text or %%TGP0, %%TGP0 jz %%_len_is_0 + ;; fill %%TMP_XMM_0 with common length values per initialized length + ;; to be subtracted from remaining byte lengths and rounded dw lengths movd %%TMP_XMM_0, DWORD(%%TGP0) pshufd %%TMP_XMM_0, %%TMP_XMM_0, 0 - movdqa %%TMP_XMM_5, [state + ROUNDED_DW_LENS] - pxor %%TMP_XMM_6, %%TMP_XMM_6 - pxor %%TMP_XMM_7, %%TMP_XMM_7 - ;; Create mask with lanes in use - pcmpeqq %%TMP_XMM_6, [state + _snow3g_job_in_lane] - pcmpeqq %%TMP_XMM_7, [state + _snow3g_job_in_lane + 16] - pshufd %%TMP_XMM_6, %%TMP_XMM_6, 0x88 ;; lane order: 1,0,1,0 - pshufd %%TMP_XMM_7, %%TMP_XMM_7, 0x88 ;; lane order: 3,2,3,2 - pblendw %%TMP_XMM_6, %%TMP_XMM_7, 0xf0 - - pxor %%TMP_XMM_6, [rel all_fs] - pand %%TMP_XMM_6, %%TMP_XMM_0 - psubd %%TMP_XMM_5, %%TMP_XMM_6 + pxor %%TMP_XMM_2, %%TMP_XMM_2 + pxor %%TMP_XMM_3, %%TMP_XMM_3 + pcmpeqq %%TMP_XMM_2, [state + _snow3g_job_in_lane] + pcmpeqq %%TMP_XMM_3, [state + _snow3g_job_in_lane + 16] + pshufd %%TMP_XMM_2, %%TMP_XMM_2, 0x88 ;; lane order: 1,0,1,0 + pshufd %%TMP_XMM_3, %%TMP_XMM_3, 0x88 ;; lane order: 3,2,3,2 + pblendw %%TMP_XMM_2, %%TMP_XMM_3, 0xf0 + + pandn %%TMP_XMM_2, %%TMP_XMM_0 + + ;; Decrease rouded dw lengths remaining for processing + movdqa %%TMP_XMM_5, [state + ROUNDED_DW_LENS] + psubd %%TMP_XMM_5, %%TMP_XMM_2 movdqa [state + ROUNDED_DW_LENS], %%TMP_XMM_5 - pslld %%TMP_XMM_0, 2 + ;; Set all bits in dws where rounded dw length is bigger than original + ;; byte lengths and lane is initialized + pslld %%TMP_XMM_0, 2 ;; common length in bytes pand %%TMP_XMM_0, [state + KEYGEN_STAGE] movdqa %%TMP_XMM_1, %%TMP_XMM_0 - - ;; Set all bits in dws where rounded dw length is bigger than original - ;; byte lengths pcmpgtd %%TMP_XMM_1, [state + _snow3g_args_byte_length] + movdqa %%TMP_XMM_2, %%TMP_XMM_1 pand %%TMP_XMM_1, [state + _snow3g_args_byte_length] + + pandn %%TMP_XMM_0, %%TMP_XMM_2 + por %%TMP_XMM_0, %%TMP_XMM_1 + ;; Write outstanding bytes to _snow3g_lens dwords [0:3] and adjust ;; _snow3g_args_byte_length so after common dw length subtraction ;; it is set to 0 pand %%TMP_XMM_1, [rel last_3_bytes] - pcmpeqd %%TMP_XMM_1, [rel zero_xmm] - movdqa %%TMP_XMM_4, %%TMP_XMM_1 - pxor %%TMP_XMM_4, [rel all_fs] - pand %%TMP_XMM_4, [state+_snow3g_lens] - pand %%TMP_XMM_1, [state + _snow3g_args_byte_length] - por %%TMP_XMM_4, %%TMP_XMM_1 - movdqa [state + _snow3g_args_byte_length], %%TMP_XMM_4 - - movdqa %%TMP_XMM_3, [state+_snow3g_args_byte_length] - psubd %%TMP_XMM_3, %%TMP_XMM_0 - movdqa [state+_snow3g_args_byte_length], %%TMP_XMM_3 + pandn %%TMP_XMM_2, [state+_snow3g_lens] + por %%TMP_XMM_1, %%TMP_XMM_2 + movdqa [state + _snow3g_lens], %%TMP_XMM_1 + + ;; Subtract Common dw length from all byte lengths + movdqa %%TMP_XMM_4, [state+_snow3g_args_byte_length] + psubd %%TMP_XMM_4, %%TMP_XMM_0 + movdqa [state+_snow3g_args_byte_length], %%TMP_XMM_4 + ;; Do cipher / clock operation for all lanes and given common length SNOW3G_ENC_DEC state, %%TGP0, %%TGP1, %%TGP2, %%TGP3, %%TGP4, %%TGP5, \ %%TMP_XMM_0, %%TMP_XMM_1, %%TMP_XMM_2, %%TMP_XMM_3, \ diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index e939cda1..350f4972 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -730,11 +730,12 @@ endstruc %else ;; up to 4 bytes (defined by %%length) mov DWORD(%%LENGTH), [%%STATE + _snow3g_lens + %%LANE * 4] cmp %%LENGTH, 4 - jne %%_not_dw + jne %%_not_dw xor DWORD(%%TMP64), [%%IN_PTR] mov dword [%%OUT_PTR], DWORD(%%TMP64) jmp %%_write_done %%_not_dw: + and %%LENGTH, 3 cmp %%LENGTH, 2 jl %%_write_single_byte @@ -745,7 +746,7 @@ endstruc add %%OUT_PTR, 2 and %%LENGTH, 1 je %%_write_done - + shr %%TMP64, 16 %%_write_single_byte: xor BYTE(%%TMP64), [%%IN_PTR] mov byte [%%OUT_PTR], BYTE(%%TMP64) -- GitLab From 0a067b9355e2dcfa034478f434e73c1a5b8d106a Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Mon, 4 Jul 2022 10:38:03 +0000 Subject: [PATCH 212/369] sse: [SHA256-NI] expose multi-buffer API --- README | 4 +- lib/Makefile | 1 + lib/avx/sha_mb_avx.c | 8 +- lib/avx2_t1/sha_mb_avx2.c | 8 +- lib/avx512_t1/sha_mb_avx512.c | 8 +- lib/include/arch_sse_type2.h | 4 + lib/include/mb_mgr_code.h | 8 ++ lib/include/sha_mb_mgr.h | 118 +++++++++++++++++++++-------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 + lib/sse_t1/mb_mgr_sse.c | 13 +++- lib/sse_t1/sha_mb_sse.c | 32 +------- lib/sse_t2/sha256_ni_x2_sse.asm | 75 ++++++++++++++++++ lib/sse_t2/sha_ni_mb_sse.c | 76 +++++++++++++++++++ lib/win_x64.mak | 1 + lib/x86_64/ooo_mgr_reset.c | 2 + 15 files changed, 284 insertions(+), 76 deletions(-) create mode 100644 lib/sse_t2/sha_ni_mb_sse.c diff --git a/README b/README index 9e91ef44..29a800e4 100644 --- a/README +++ b/README @@ -135,9 +135,9 @@ implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-224 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | - | SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/lib/Makefile b/lib/Makefile index 324a825c..3da34bd5 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -257,6 +257,7 @@ c_lib_objs := \ md5_one_block.o \ sha_sse.o \ sha_mb_sse.o \ + sha_ni_mb_sse.o \ sha_avx.o \ sha_mb_avx.o \ sha_mb_avx2.o \ diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index 059fe945..974a4cbe 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -67,7 +67,7 @@ IMB_JOB *submit_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 1, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha_256_mult_avx_from_c); + call_sha_256_mult_avx_from_c, 0); } IMB_DLL_LOCAL @@ -75,7 +75,7 @@ IMB_JOB *flush_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 0, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha_256_mult_avx_from_c); + call_sha_256_mult_avx_from_c, 0); } /* ========================================================================== */ @@ -88,7 +88,7 @@ IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 1, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha_256_mult_avx_from_c); + call_sha_256_mult_avx_from_c, 0); } IMB_DLL_LOCAL @@ -96,5 +96,5 @@ IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 0, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha_256_mult_avx_from_c); + call_sha_256_mult_avx_from_c, 0); } diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index 2f9e5ee5..f5f8b19d 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -67,7 +67,7 @@ IMB_JOB *submit_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 8, 1, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha256_oct_avx2_from_c); + call_sha256_oct_avx2_from_c, 0); } IMB_DLL_LOCAL @@ -75,7 +75,7 @@ IMB_JOB *flush_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 8, 0, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha256_oct_avx2_from_c); + call_sha256_oct_avx2_from_c, 0); } /* ========================================================================== */ @@ -88,7 +88,7 @@ IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 8, 1, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha256_oct_avx2_from_c); + call_sha256_oct_avx2_from_c, 0); } IMB_DLL_LOCAL @@ -96,5 +96,5 @@ IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 8, 0, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha256_oct_avx2_from_c); + call_sha256_oct_avx2_from_c, 0); } diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index c787dd2d..d20062b9 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -67,7 +67,7 @@ IMB_JOB *submit_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 16, 1, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha256_x16_avx512_from_c); + call_sha256_x16_avx512_from_c, 0); } IMB_DLL_LOCAL @@ -75,7 +75,7 @@ IMB_JOB *flush_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 16, 0, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha256_x16_avx512_from_c); + call_sha256_x16_avx512_from_c, 0); } @@ -89,7 +89,7 @@ IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 16, 1, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha256_x16_avx512_from_c); + call_sha256_x16_avx512_from_c, 0); } IMB_DLL_LOCAL @@ -97,5 +97,5 @@ IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 16, 0, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha256_x16_avx512_from_c); + call_sha256_x16_avx512_from_c, 0); } diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h index 85a2b143..60139f3e 100644 --- a/lib/include/arch_sse_type2.h +++ b/lib/include/arch_sse_type2.h @@ -51,4 +51,8 @@ IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + +IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + #endif /* IMB_ARCH_SSE_TYPE2_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 5ed9d425..fe209ead 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -703,6 +703,10 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_224: return SUBMIT_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return SUBMIT_JOB_SHA256_NI(sha_256_ooo, job); +#endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: IMB_SHA384(state, @@ -876,6 +880,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_SHA_224: return FLUSH_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); +#endif return FLUSH_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 53fdba36..a10e9b98 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -55,22 +55,32 @@ extern void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, extern void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); + +extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, - const size_t offset, const unsigned lane, - const int shani) + const size_t offset, const unsigned lane) { uint32_t *outp = (uint32_t *) dst; const uint32_t *inp = (const uint32_t *) src; size_t i; - if (shani) { - for (i = 0; i < num; i++) - outp[i] = bswap4(inp[5*lane + i*offset]); - } else { - for (i = 0; i < num; i++) - outp[i] = bswap4(inp[lane + i*offset]); - } + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[lane + i*offset]); +} + +__forceinline +void copy_bswap4_array_mb_ni(void *dst, const void *src, const size_t num, + const unsigned lane, const int digest_row_sz) +{ + uint32_t *outp = (uint32_t *) dst; + const uint32_t *inp = (const uint32_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap4(inp[digest_row_sz*lane + i]); } __forceinline @@ -119,36 +129,67 @@ void sha256_mb_init_digest(uint32_t *digest, const unsigned lane) digest[lane + 7*16] = SHA256_H7; } +__forceinline +void sha256_ni_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[8*lane + 0] = SHA256_H0; + digest[8*lane + 1] = SHA256_H1; + digest[8*lane + 2] = SHA256_H2; + digest[8*lane + 3] = SHA256_H3; + digest[8*lane + 4] = SHA256_H4; + digest[8*lane + 5] = SHA256_H5; + digest[8*lane + 6] = SHA256_H6; + digest[8*lane + 7] = SHA256_H7; +} + __forceinline void -sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane, - const int shani) +sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) { - if (sha_type == 1){ - if (shani) - sha1_ni_mb_init_digest(digest, lane); - else - sha1_mb_init_digest(digest, lane); - } else if (sha_type == 224) + if (sha_type == 1) + sha1_mb_init_digest(digest, lane); + else if (sha_type == 224) sha224_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_mb_init_digest(digest, lane); } +__forceinline +void +sha_ni_mb_generic_init(void *digest, const int sha_type, const unsigned lane) +{ + if (sha_type == 1) + sha1_ni_mb_init_digest(digest, lane); + else if (sha_type == 256) + sha256_ni_mb_init_digest(digest, lane); +} + __forceinline void sha_mb_generic_write_digest(void *dst, const void *src, const int sha_type, const size_t offset, - const unsigned lane, const int shani) + const unsigned lane) { if (sha_type == 1) copy_bswap4_array_mb(dst, src, NUM_SHA_DIGEST_WORDS, offset, - lane, shani); + lane); else if (sha_type == 224) copy_bswap4_array_mb(dst, src, NUM_SHA_224_DIGEST_WORDS, offset, - lane, shani); + lane); else if (sha_type == 256) copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, - lane, shani); + lane); +} + +__forceinline +void sha_ni_mb_generic_write_digest(void *dst, const void *src, + const int sha_type, const unsigned lane) +{ + if (sha_type == 1) + copy_bswap4_array_mb_ni(dst, src, NUM_SHA_DIGEST_WORDS, + lane, 5); + else if (sha_type == 256) + copy_bswap4_array_mb_ni(dst, src, NUM_SHA_256_DIGEST_WORDS, + lane, 8); } __forceinline @@ -222,7 +263,12 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; - sha_mb_generic_init(state->args.digest, sha_type, lane, shani); + if (shani) + sha_ni_mb_generic_init(state->args.digest, sha_type, + lane); + else + sha_mb_generic_init(state->args.digest, sha_type, + lane); /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; @@ -313,13 +359,12 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; - if (shani) { - sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 1, min_idx, shani); - } else { + if (shani) + sha_ni_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, min_idx); + else sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx, shani); - } + state->args.digest, sha_type, 16, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; @@ -332,7 +377,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, const unsigned max_jobs, const int is_submit, const int sha_type, const uint64_t blk_size, const uint64_t pad_size, - void (*fn)(SHA256_ARGS *, uint32_t)) + void (*fn)(SHA256_ARGS *, uint32_t), const int shani) { unsigned lane, min_idx; uint64_t min_len; @@ -351,7 +396,12 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, state->args.data_ptr[lane] = job->src + job->hash_start_src_offset_in_bytes; - sha_mb_generic_init(state->args.digest, sha_type, lane, 0); + if (shani) + sha_ni_mb_generic_init(state->args.digest, sha_type, + lane); + else + sha_mb_generic_init(state->args.digest, sha_type, + lane); /* copy job data in and set up initial blocks */ state->ldata[lane].job_in_lane = job; @@ -441,8 +491,12 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; - sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx, 0); + if (shani) + sha_ni_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, min_idx); + else + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 16, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; return ret_job; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index fb6544de..87a4492f 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -94,6 +94,8 @@ #define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse #define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 1ebddb10..7d9354f0 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -100,6 +100,8 @@ #define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse #define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse @@ -720,8 +722,15 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA224 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); - /* Init SHA256 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + /* Init SHA1 NI out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); + } else { + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); + } +#endif /* HASH_USE_SHAEXT */ /* Init SNOW3G-UEA out-of-order fields */ ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 425bfc59..a9a612e4 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -36,9 +36,6 @@ IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - /* ========================================================================== */ /* * SHA1 MB API @@ -70,7 +67,7 @@ IMB_JOB *submit_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 1, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha_256_mult_sse_from_c); + call_sha_256_mult_sse_from_c, 0); } IMB_DLL_LOCAL @@ -78,7 +75,7 @@ IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 0, 224, IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, - call_sha_256_mult_sse_from_c); + call_sha_256_mult_sse_from_c, 0); } /* ========================================================================== */ @@ -91,7 +88,7 @@ IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 1, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha_256_mult_sse_from_c); + call_sha_256_mult_sse_from_c, 0); } IMB_DLL_LOCAL @@ -99,26 +96,5 @@ IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_256(state, job, 4, 0, 256, IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, - call_sha_256_mult_sse_from_c); -} - -/* ========================================================================== */ -/* - * SHA1-NI MB API - */ - -IMB_DLL_LOCAL -IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) -{ - return submit_flush_job_sha_1(state, job, 2, 1, 1, - IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_ni_x2_sse_from_c, 1); -} - -IMB_DLL_LOCAL -IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) -{ - return submit_flush_job_sha_1(state, job, 2, 0, 1, - IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_ni_x2_sse_from_c, 1); + call_sha_256_mult_sse_from_c, 0); } diff --git a/lib/sse_t2/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm index e4d8bced..91564561 100644 --- a/lib/sse_t2/sha256_ni_x2_sse.asm +++ b/lib/sse_t2/sha256_ni_x2_sse.asm @@ -126,6 +126,73 @@ PSHUFFLE_BYTE_FLIP_MASK: ;; arg1 : pointer to args ;; arg2 : size (in blocks) ;; assumed to be >= 1 mksection .text + +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + movdqa [rsp + 3*16], xmm6 + movdqa [rsp + 4*16], xmm7 + movdqa [rsp + 5*16], xmm8 + movdqa [rsp + 6*16], xmm9 + movdqa [rsp + 7*16], xmm10 + movdqa [rsp + 8*16], xmm11 + movdqa [rsp + 9*16], xmm12 + movdqa [rsp + 10*16], xmm13 + movdqa [rsp + 11*16], xmm14 + movdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + movdqa xmm6, [rsp + 3*16] + movdqa xmm7, [rsp + 4*16] + movdqa xmm8, [rsp + 5*16] + movdqa xmm9, [rsp + 6*16] + movdqa xmm10, [rsp + 7*16] + movdqa xmm11, [rsp + 8*16] + movdqa xmm12, [rsp + 9*16] + movdqa xmm13, [rsp + 10*16] + movdqa xmm14, [rsp + 11*16] + movdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + pxor xmm5, xmm5 + movdqa xmm5, [rsp + 3*16] + movdqa xmm5, [rsp + 4*16] + movdqa xmm5, [rsp + 5*16] + movdqa xmm5, [rsp + 6*16] + movdqa xmm5, [rsp + 7*16] + movdqa xmm5, [rsp + 8*16] + movdqa xmm5, [rsp + 9*16] + movdqa xmm5, [rsp + 10*16] + movdqa xmm5, [rsp + 11*16] + movdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + MKGLOBAL(sha256_ni,function,internal) align 32 sha256_ni: @@ -609,4 +676,12 @@ done_hash: add rsp, frame_size ret +; void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha256_ni_x2_sse_from_c,function,internal) +call_sha256_ni_x2_sse_from_c: + FUNC_SAVE + call sha256_ni + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/sse_t2/sha_ni_mb_sse.c b/lib/sse_t2/sha_ni_mb_sse.c new file mode 100644 index 00000000..7e6a7547 --- /dev/null +++ b/lib/sse_t2/sha_ni_mb_sse.c @@ -0,0 +1,76 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "include/sha_mb_mgr.h" + +IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); + +IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + +/* ========================================================================== */ +/* + * SHA1-NI MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 2, 1, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_ni_x2_sse_from_c, 1); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_1(state, job, 2, 0, 1, + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_ni_x2_sse_from_c, 1); +} + +/* ========================================================================== */ +/* + * SHA256-NI MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 2, 1, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_ni_x2_sse_from_c, 1); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 2, 0, 256, + IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, + call_sha256_ni_x2_sse_from_c, 1); +} \ No newline at end of file diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 8816cf21..155a3949 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -282,6 +282,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha_sse.obj \ $(OBJ_DIR)\sha_avx.obj \ $(OBJ_DIR)\sha_mb_sse.obj \ + $(OBJ_DIR)\sha_ni_mb_sse.obj \ $(OBJ_DIR)\sha_mb_avx.obj \ $(OBJ_DIR)\sha_mb_avx2.obj \ $(OBJ_DIR)\sha_mb_avx512.obj \ diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index c29bda34..6e36fbb5 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -348,6 +348,8 @@ void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) memset(p_mgr, 0, sizeof(*p_mgr)); + if (num_lanes == 2) + p_mgr->unused_lanes = 0xF10; /* SHANI */ if (num_lanes == AVX_NUM_SHA256_LANES) p_mgr->unused_lanes = 0xF3210; else if (num_lanes == AVX2_NUM_SHA256_LANES) -- GitLab From 9c7a971542b6ea65fa978e85f644987af32ba3a5 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Mon, 4 Jul 2022 14:25:17 +0000 Subject: [PATCH 213/369] test: [ACVP] support for CCM --- test/acvp_app_main.c | 100 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 49183690..4d0232c0 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -416,6 +416,102 @@ static int aes_ctr_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_ccm_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); + DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); + uint8_t res_tag[MAX_TAG_LENGTH]; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + fprintf(stderr, "Unsupported direction\n"); + return EXIT_FAILURE; + } + + switch (tc->key_len) { + case 128: + IMB_AES_KEYEXP_128(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 192: + IMB_AES_KEYEXP_192(mb_mgr, tc->key, enc_keys, dec_keys); + break; + case 256: + IMB_AES_KEYEXP_256(mb_mgr, tc->key, enc_keys, dec_keys); + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len >> 3; + job->cipher_mode = IMB_CIPHER_CCM; + job->hash_alg = IMB_AUTH_AES_CCM; + + job->iv = tc->iv; + job->iv_len_in_bytes = tc->iv_len; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + job->auth_tag_output_len_in_bytes = tc->tag_len; + job->u.CCM.aad = tc->aad; + job->u.CCM.aad_len_in_bytes = tc->aad_len; + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + job->msg_len_to_hash_in_bytes = tc->pt_len; + /* Auth tag must be placed at the end of the ciphertext. */ + job->auth_tag_output = tc->ct + tc->pt_len; + tc->ct_len = tc->pt_len + tc->tag_len; + } else /* DECRYPT */ { + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_hash_in_bytes = tc->ct_len; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + job->auth_tag_output = res_tag; + tc->pt_len = tc->ct_len; + } + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + + if (tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT) { + /* Tag is placed at the end of the ciphertext. */ + const uint8_t *ref_tag = tc->ct + tc->ct_len; + + if (memcmp(res_tag, ref_tag, tc->tag_len) != 0) { + if (verbose) { + hexdump(stdout, "result tag: ", + res_tag, tc->tag_len); + hexdump(stdout, "reference tag: ", + ref_tag, tc->tag_len); + fprintf(stderr, "Invalid tag\n"); + } + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -531,6 +627,10 @@ int main(int argc, char **argv) &aes_ctr_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CCM, + &aes_ccm_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 54c8cbcd56f2c54937f8f7d084ead925fa0792c8 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 7 Jul 2022 11:43:17 +0000 Subject: [PATCH 214/369] sse: [SHA224-NI] expose multi-buffer API --- README | 2 +- README.md | 6 +++--- lib/avx/sha_mb_avx.c | 10 +--------- lib/avx2_t1/sha_mb_avx2.c | 10 +--------- lib/avx512_t1/sha_mb_avx512.c | 10 +--------- lib/include/arch_sse_type2.h | 4 ++++ lib/include/mb_mgr_code.h | 8 ++++++++ lib/include/sha_mb_mgr.h | 21 +++++++++++++++++++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 ++ lib/sse_t1/mb_mgr_sse.c | 17 +++++++++++++---- lib/sse_t1/sha_mb_sse.c | 10 +--------- lib/sse_t2/sha_ni_mb_sse.c | 28 ++++++++++++++++++++++------ 12 files changed, 78 insertions(+), 50 deletions(-) diff --git a/README b/README index 29a800e4..d35900dd 100644 --- a/README +++ b/README @@ -136,7 +136,7 @@ implementations. | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | SHA2-224 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index 9b7b89f5..37701461 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,9 @@ Table 2. List of supported integrity algorithms and their implementations. | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | -| SHA1 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | -| SHA2-224 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | -| SHA2-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | N | +| SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index 974a4cbe..f406bd69 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -26,15 +26,7 @@ *******************************************************************************/ #include "include/sha_mb_mgr.h" - -IMB_JOB *submit_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha224_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +#include "include/arch_avx_type1.h" /* ========================================================================== */ /* diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index f5f8b19d..17319323 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -26,15 +26,7 @@ *******************************************************************************/ #include "include/sha_mb_mgr.h" - -IMB_JOB *submit_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx2(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha224_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +#include "include/arch_avx2_type1.h" /* ========================================================================== */ /* diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index d20062b9..adbeb0e6 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -26,15 +26,7 @@ *******************************************************************************/ #include "include/sha_mb_mgr.h" - -IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha224_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +#include "include/arch_avx512_type1.h" /* ========================================================================== */ /* diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h index 60139f3e..609fb5d9 100644 --- a/lib/include/arch_sse_type2.h +++ b/lib/include/arch_sse_type2.h @@ -51,6 +51,10 @@ IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha224_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + +IMB_JOB *flush_job_sha224_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index fe209ead..4def9dd2 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -701,6 +701,10 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return SUBMIT_JOB_SHA224_NI(sha_224_ooo, job); +#endif return SUBMIT_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: #ifdef HASH_USE_SHAEXT @@ -878,6 +882,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return FLUSH_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) + return FLUSH_JOB_SHA224_NI(sha_224_ooo, job); +#endif return FLUSH_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: #ifdef HASH_USE_SHAEXT diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index a10e9b98..fff7ae3d 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -56,6 +56,9 @@ extern void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, extern void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +extern void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); @@ -116,6 +119,19 @@ void sha224_mb_init_digest(uint32_t *digest, const unsigned lane) digest[lane + 7*16] = SHA224_H7; } +__forceinline +void sha224_ni_mb_init_digest(uint32_t *digest, const unsigned lane) +{ + digest[8*lane + 0] = SHA224_H0; + digest[8*lane + 1] = SHA224_H1; + digest[8*lane + 2] = SHA224_H2; + digest[8*lane + 3] = SHA224_H3; + digest[8*lane + 4] = SHA224_H4; + digest[8*lane + 5] = SHA224_H5; + digest[8*lane + 6] = SHA224_H6; + digest[8*lane + 7] = SHA224_H7; +} + __forceinline void sha256_mb_init_digest(uint32_t *digest, const unsigned lane) { @@ -160,6 +176,8 @@ sha_ni_mb_generic_init(void *digest, const int sha_type, const unsigned lane) { if (sha_type == 1) sha1_ni_mb_init_digest(digest, lane); + else if (sha_type == 224) + sha224_ni_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_ni_mb_init_digest(digest, lane); } @@ -187,6 +205,9 @@ void sha_ni_mb_generic_write_digest(void *dst, const void *src, if (sha_type == 1) copy_bswap4_array_mb_ni(dst, src, NUM_SHA_DIGEST_WORDS, lane, 5); + else if (sha_type == 224) + copy_bswap4_array_mb_ni(dst, src, NUM_SHA_224_DIGEST_WORDS, + lane, 8); else if (sha_type == 256) copy_bswap4_array_mb_ni(dst, src, NUM_SHA_256_DIGEST_WORDS, lane, 8); diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 87a4492f..09d023d8 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -94,6 +94,8 @@ #define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse #define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA224_NI submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 7d9354f0..9dc83f87 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -100,6 +100,8 @@ #define FLUSH_JOB_SHA256 flush_job_sha256_sse #define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse #define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA224_NI submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse @@ -719,15 +721,22 @@ reset_ooo_mgrs(IMB_MGR *state) } #endif /* HASH_USE_SHAEXT */ - /* Init SHA224 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); +#ifdef HASH_USE_SHAEXT + if (state->features & IMB_FEATURE_SHANI) { + /* Init SHA224 NI out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); + } else { + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); + } +#endif /* HASH_USE_SHAEXT */ #ifdef HASH_USE_SHAEXT if (state->features & IMB_FEATURE_SHANI) { - /* Init SHA1 NI out-of-order fields */ + /* Init SHA256 NI out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, 2); } else { - /* Init SHA1 out-of-order fields */ + /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); } #endif /* HASH_USE_SHAEXT */ diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index a9a612e4..1f5ebe33 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -26,15 +26,7 @@ *******************************************************************************/ #include "include/sha_mb_mgr.h" - -IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha224_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +#include "include/arch_sse_type1.h" /* ========================================================================== */ /* diff --git a/lib/sse_t2/sha_ni_mb_sse.c b/lib/sse_t2/sha_ni_mb_sse.c index 7e6a7547..5a6cbec8 100644 --- a/lib/sse_t2/sha_ni_mb_sse.c +++ b/lib/sse_t2/sha_ni_mb_sse.c @@ -26,12 +26,7 @@ *******************************************************************************/ #include "include/sha_mb_mgr.h" - -IMB_JOB *submit_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); - -IMB_JOB *submit_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); -IMB_JOB *flush_job_sha256_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +#include "include/arch_sse_type2.h" /* ========================================================================== */ /* @@ -54,6 +49,27 @@ IMB_JOB *flush_job_sha1_ni_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) call_sha1_ni_x2_sse_from_c, 1); } +/* ========================================================================== */ +/* + * SHA224-NI MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha224_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 2, 1, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_ni_x2_sse_from_c, 1); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha224_ni_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_256(state, job, 2, 0, 224, + IMB_SHA_256_BLOCK_SIZE, SHA224_PAD_SIZE, + call_sha256_ni_x2_sse_from_c, 1); +} + /* ========================================================================== */ /* * SHA256-NI MB API -- GitLab From dd0264a3cb513a3a2d74e2f91c0a94f4d9081abc Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 21 Jun 2022 08:21:10 +0000 Subject: [PATCH 215/369] sse: [SHA512] expose multi-buffer API --- lib/include/arch_sse_type1.h | 5 + lib/include/ipsec_ooo_mgr.h | 11 +- lib/include/mb_mgr_code.h | 18 ++- lib/include/ooo_mgr_reset.h | 3 + lib/include/sha_mb_mgr.h | 201 +++++++++++++++++++++++++++-- lib/intel-ipsec-mb.h | 3 +- lib/no-aesni/mb_mgr_sse_no_aesni.c | 10 ++ lib/sse_t1/mb_mgr_sse.c | 5 + lib/sse_t1/sha512_x2_sse.asm | 74 +++++++++++ lib/sse_t1/sha_mb_sse.c | 21 +++ lib/x86_64/alloc.c | 3 +- lib/x86_64/ooo_mgr_reset.c | 15 +++ 12 files changed, 356 insertions(+), 13 deletions(-) diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index 82676b2d..3f005be7 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -212,6 +212,11 @@ IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha512_sse(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha512_sse(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_sse(const void *key_exp, void *key1, void *key2); uint32_t hec_32_sse(const uint8_t *in); diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 72c4348e..49f92d5f 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -124,7 +124,7 @@ typedef struct { typedef struct { DECLARE_ALIGNED(uint64_t digest[SHA512_DIGEST_SZ], 32); - uint8_t *data_ptr[AVX512_NUM_SHA512_LANES]; + const uint8_t *data_ptr[AVX512_NUM_SHA512_LANES]; } SHA512_ARGS; typedef struct { @@ -373,6 +373,15 @@ typedef struct { uint64_t road_block; } MB_MGR_HMAC_SHA_512_OOO; +typedef struct { + SHA512_ARGS args; + DECLARE_ALIGNED(uint64_t lens[AVX512_NUM_SHA512_LANES], 16); + uint64_t unused_lanes; + HMAC_SHA512_LANE_DATA ldata[AVX512_NUM_SHA512_LANES]; + uint32_t num_lanes_inuse; + uint64_t road_block; +} MB_MGR_SHA_512_OOO; + /* MD5-HMAC out-of-order scheduler fields */ typedef struct { MD5_ARGS args; diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 4def9dd2..47797df8 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -636,8 +636,11 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#endif #if defined (SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -719,11 +722,15 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_512: +#ifdef SSE + return SUBMIT_JOB_SHA512(sha_512_ooo, job); +#else IMB_SHA512(state, job->src + job->hash_start_src_offset_in_bytes, job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; +#endif case IMB_AUTH_ZUC_EIA3_BITLEN: return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); case IMB_AUTH_ZUC256_EIA3_BITLEN: @@ -844,8 +851,11 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#endif #if defined(SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -893,6 +903,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); +#ifdef SSE + case IMB_AUTH_SHA_512: + return FLUSH_JOB_SHA512(sha_512_ooo, job); +#endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/include/ooo_mgr_reset.h b/lib/include/ooo_mgr_reset.h index c8510ad1..1d4c9476 100644 --- a/lib/include/ooo_mgr_reset.h +++ b/lib/include/ooo_mgr_reset.h @@ -70,6 +70,9 @@ void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes); IMB_DLL_LOCAL void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes); +IMB_DLL_LOCAL +void ooo_mgr_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes); + IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes); diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index fff7ae3d..32957d9c 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -62,6 +62,9 @@ extern void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha512_x2_sse_from_c(SHA512_ARGS *args, + uint64_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) @@ -86,6 +89,18 @@ void copy_bswap4_array_mb_ni(void *dst, const void *src, const size_t num, outp[i] = bswap4(inp[digest_row_sz*lane + i]); } +__forceinline +void copy_bswap8_array_mb(void *dst, const void *src, const size_t num, + const size_t offset, const unsigned lane) +{ + uint64_t *outp = (uint64_t *) dst; + const uint64_t *inp = (const uint64_t *) src; + size_t i; + + for (i = 0; i < num; i++) + outp[i] = bswap8(inp[lane + i*offset]); +} + __forceinline void sha1_mb_init_digest(uint32_t *digest, const unsigned lane) { @@ -158,6 +173,19 @@ void sha256_ni_mb_init_digest(uint32_t *digest, const unsigned lane) digest[8*lane + 7] = SHA256_H7; } +__forceinline +void sha512_mb_init_digest(uint64_t *digest, const unsigned lane) +{ + digest[lane + 0*8] = SHA512_H0; + digest[lane + 1*8] = SHA512_H1; + digest[lane + 2*8] = SHA512_H2; + digest[lane + 3*8] = SHA512_H3; + digest[lane + 4*8] = SHA512_H4; + digest[lane + 5*8] = SHA512_H5; + digest[lane + 6*8] = SHA512_H6; + digest[lane + 7*8] = SHA512_H7; +} + __forceinline void sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) @@ -168,6 +196,8 @@ sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) sha224_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_mb_init_digest(digest, lane); + else if (sha_type == 512) + sha512_mb_init_digest(digest, lane); } __forceinline @@ -196,6 +226,9 @@ void sha_mb_generic_write_digest(void *dst, const void *src, else if (sha_type == 256) copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, lane); + else if (sha_type == 512) + copy_bswap8_array_mb(dst, src, NUM_SHA_512_DIGEST_WORDS, offset, + lane); } __forceinline @@ -259,6 +292,29 @@ void sha256_create_extra_blocks(MB_MGR_SHA_256_OOO *state, state->ldata[min_idx].extra_blocks = 0; } +__forceinline +void sha512_create_extra_blocks(MB_MGR_SHA_512_OOO *state, + const uint64_t blk_size, const uint64_t r, + const unsigned min_idx) +{ + HMAC_SHA512_LANE_DATA *ld = &state->ldata[min_idx]; + const uint64_t xblk_size = blk_size*state->ldata[min_idx].extra_blocks; + + memset(ld->extra_block, 0, sizeof(ld->extra_block)); + + var_memcpy(ld->extra_block, state->args.data_ptr[min_idx], r); + ld->extra_block[r] = 0x80; + + store8_be(&ld->extra_block[xblk_size - 8], + ld->job_in_lane->msg_len_to_hash_in_bytes * 8); + + state->args.data_ptr[min_idx] = &ld->extra_block[0]; + + state->lens[min_idx] = (uint16_t)xblk_size; + + state->ldata[min_idx].extra_blocks = 0; +} + __forceinline IMB_JOB * submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, @@ -268,9 +324,7 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, void (*fn)(SHA1_ARGS *, uint32_t), const int shani) { unsigned lane, min_idx; - uint64_t min_len; IMB_JOB *ret_job = NULL; - unsigned i; if (is_submit) { /* @@ -312,6 +366,9 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, } do { + uint64_t min_len; + unsigned i; + if (is_submit) { /* * SUBMIT @@ -382,10 +439,12 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, state->num_lanes_inuse--; if (shani) sha_ni_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, min_idx); + state->args.digest, sha_type, + min_idx); else sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx); + state->args.digest, sha_type, 16, + min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; @@ -401,9 +460,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, void (*fn)(SHA256_ARGS *, uint32_t), const int shani) { unsigned lane, min_idx; - uint64_t min_len; IMB_JOB *ret_job = NULL; - unsigned i; if (is_submit) { /* @@ -445,6 +502,9 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, } do { + uint64_t min_len; + unsigned i; + if (is_submit) { /* * SUBMIT @@ -514,10 +574,135 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, state->num_lanes_inuse--; if (shani) sha_ni_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, min_idx); + state->args.digest, sha_type, + min_idx); else sha_mb_generic_write_digest(ret_job->auth_tag_output, - state->args.digest, sha_type, 16, min_idx); + state->args.digest, sha_type, 16, + min_idx); + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; + state->ldata[min_idx].job_in_lane = NULL; + return ret_job; +} + +__forceinline +IMB_JOB * +submit_flush_job_sha_512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job, + const unsigned max_jobs, const int is_submit, + const int sha_type, const uint64_t blk_size, + const uint64_t pad_size, + void (*fn)(SHA512_ARGS *, uint64_t)) +{ + unsigned lane, min_idx; + IMB_JOB *ret_job = NULL; + + if (is_submit) { + /* + * SUBMIT + * - get a free lane id + */ + + lane = state->unused_lanes & 15; + state->unused_lanes >>= 4; + state->num_lanes_inuse++; + state->args.data_ptr[lane] = + job->src + job->hash_start_src_offset_in_bytes; + + sha_mb_generic_init(state->args.digest, sha_type, lane); + + /* copy job data in and set up initial blocks */ + state->ldata[lane].job_in_lane = job; + state->lens[lane] = (uint16_t)job->msg_len_to_hash_in_bytes; + state->ldata[lane].extra_blocks = 1; + + /* enough jobs to start processing? */ + if (state->num_lanes_inuse != max_jobs) + return NULL; + } else { + /* + * FLUSH + * - find 1st non null job + */ + for (lane = 0; lane < max_jobs; lane++) + if (state->ldata[lane].job_in_lane != NULL) + break; + if (lane >= max_jobs) + return NULL; /* no not null job */ + } + + do { + uint64_t min_len; + unsigned i; + + if (is_submit) { + /* + * SUBMIT + * - find min common length to process + */ + min_idx = 0; + min_len = state->lens[0]; + + for (i = 1; i < max_jobs; i++) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } + } else { + /* + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ + min_idx = lane; + min_len = state->lens[lane]; + + for (i = 0; i < max_jobs; i++) { + if (i == lane) + continue; + + if (state->ldata[i].job_in_lane != NULL) { + if (min_len > state->lens[i]) { + min_idx = i; + min_len = state->lens[i]; + } + } else { + state->args.data_ptr[i] = + state->args.data_ptr[lane]; + state->lens[i] = UINT64_MAX; + } + } + } + + /* subtract min len from all lanes */ + const uint64_t min_len_blk = min_len & (~(blk_size - 1)); + + for (i = 0; i < max_jobs; i++) + state->lens[i] -= min_len_blk; + + const uint64_t r = min_len % blk_size; + + if (r >= (blk_size - pad_size)) + state->ldata[min_idx].extra_blocks = 2; + + /* run the algorithmic code on full selected blocks */ + if(min_len >= blk_size) + (*fn)(&state->args, + (uint64_t)(min_len/blk_size)); + + /* create extra blocks */ + if (state->ldata[min_idx].extra_blocks != 0) + sha512_create_extra_blocks(state, blk_size, r, min_idx); + + } while(state->lens[min_idx] != 0); + + ret_job = state->ldata[min_idx].job_in_lane; + /* put back processed packet into unused lanes, set job as complete */ + state->unused_lanes = (state->unused_lanes << 4) | min_idx; + state->num_lanes_inuse--; + sha_mb_generic_write_digest(ret_job->auth_tag_output, + state->args.digest, sha_type, 8, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; return ret_job; diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 637ed0b2..1ec488c6 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1200,8 +1200,9 @@ typedef struct IMB_MGR { void *snow3g_uea2_ooo; void *snow3g_uia2_ooo; void *sha_1_ooo; - void *sha_256_ooo; void *sha_224_ooo; + void *sha_256_ooo; + void *sha_512_ooo; } IMB_MGR; /** diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 09d023d8..6436968a 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -98,6 +98,8 @@ #define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA512 submit_job_sha512_sse +#define FLUSH_JOB_SHA512 flush_job_sha512_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni @@ -412,6 +414,7 @@ reset_ooo_mgrs(IMB_MGR *state) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; /* Init AES out-of-order fields */ memset(aes128_ooo->lens, 0xFF, @@ -801,6 +804,13 @@ reset_ooo_mgrs(IMB_MGR *state) sha_256_ooo->num_lanes_inuse = 0; for (j = 0; j < SSE_NUM_SHA256_LANES; j++) sha_256_ooo->ldata[j].job_in_lane = NULL; + + /* Init SHA512 out-of-order fields */ + sha_512_ooo->lens[0] = 0; + sha_512_ooo->lens[1] = 0; + sha_512_ooo->unused_lanes = 0xFF0100; + for (j = 0; j < SSE_NUM_SHA512_LANES; j++) + sha_512_ooo->ldata[j].job_in_lane = NULL; } IMB_DLL_LOCAL void diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 9dc83f87..2cf6da48 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -104,6 +104,8 @@ #define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA512 submit_job_sha512_sse +#define FLUSH_JOB_SHA512 flush_job_sha512_sse #define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse #define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse @@ -741,6 +743,9 @@ reset_ooo_mgrs(IMB_MGR *state) } #endif /* HASH_USE_SHAEXT */ + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); + /* Init SNOW3G-UEA out-of-order fields */ ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); diff --git a/lib/sse_t1/sha512_x2_sse.asm b/lib/sse_t1/sha512_x2_sse.asm index c9dd782f..999b75a3 100644 --- a/lib/sse_t1/sha512_x2_sse.asm +++ b/lib/sse_t1/sha512_x2_sse.asm @@ -310,6 +310,72 @@ endstruc ROUND_00_15 %%T1, %%i %endm +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + movdqa [rsp + 3*16], xmm6 + movdqa [rsp + 4*16], xmm7 + movdqa [rsp + 5*16], xmm8 + movdqa [rsp + 6*16], xmm9 + movdqa [rsp + 7*16], xmm10 + movdqa [rsp + 8*16], xmm11 + movdqa [rsp + 9*16], xmm12 + movdqa [rsp + 10*16], xmm13 + movdqa [rsp + 11*16], xmm14 + movdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + movdqa xmm6, [rsp + 3*16] + movdqa xmm7, [rsp + 4*16] + movdqa xmm8, [rsp + 5*16] + movdqa xmm9, [rsp + 6*16] + movdqa xmm10, [rsp + 7*16] + movdqa xmm11, [rsp + 8*16] + movdqa xmm12, [rsp + 9*16] + movdqa xmm13, [rsp + 10*16] + movdqa xmm14, [rsp + 11*16] + movdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + pxor xmm5, xmm5 + movdqa xmm5, [rsp + 3*16] + movdqa xmm5, [rsp + 4*16] + movdqa xmm5, [rsp + 5*16] + movdqa xmm5, [rsp + 6*16] + movdqa xmm5, [rsp + 7*16] + movdqa xmm5, [rsp + 8*16] + movdqa xmm5, [rsp + 9*16] + movdqa xmm5, [rsp + 10*16] + movdqa xmm5, [rsp + 11*16] + movdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + ;; SHA512_ARGS: ;; UINT128 digest[8]; // transposed digests ;; UINT8 *data_ptr[2]; @@ -437,4 +503,12 @@ Lrounds_16_xx: DBGPRINTL "====================== exit sha512_x2_sse code =====================\n" ret +; void call_sha512_x2_sse_from_c(SHA512_ARGS *args, UINT64 size_in_blocks); +MKGLOBAL(call_sha512_x2_sse_from_c,function,internal) +call_sha512_x2_sse_from_c: + FUNC_SAVE + call sha512_x2_sse + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 1f5ebe33..596f5547 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -90,3 +90,24 @@ IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha_256_mult_sse_from_c, 0); } + +/* ========================================================================== */ +/* + * SHA512 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha512_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 1, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x2_sse_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha512_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 0, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x2_sse_from_c); +} diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 11b7d60b..144dc10e 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -86,7 +86,8 @@ const struct { OOO_INFO(snow3g_uia2_ooo, MB_MGR_SNOW3G_OOO), OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO), OOO_INFO(sha_224_ooo, MB_MGR_SHA_256_OOO), - OOO_INFO(sha_256_ooo, MB_MGR_SHA_256_OOO) + OOO_INFO(sha_256_ooo, MB_MGR_SHA_256_OOO), + OOO_INFO(sha_512_ooo, MB_MGR_SHA_512_OOO) }; /** diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index 6e36fbb5..bb65d74d 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -358,6 +358,21 @@ void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) p_mgr->unused_lanes = 0xFEDCBA9876543210; } +IMB_DLL_LOCAL +void ooo_mgr_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) +{ + MB_MGR_SHA_512_OOO *p_mgr = (MB_MGR_SHA_512_OOO *) p_ooo_mgr; + + memset(p_mgr, 0, sizeof(*p_mgr)); + + if (num_lanes == AVX_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xF10; + else if (num_lanes == AVX2_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xF3210; + else if (num_lanes == AVX512_NUM_SHA512_LANES) + p_mgr->unused_lanes = 0xF76543210; +} + IMB_DLL_LOCAL void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes) { -- GitLab From 94c3ba8fa360f6551f7582e563d652c2963b8568 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 21 Jun 2022 09:14:16 +0000 Subject: [PATCH 216/369] avx: [SHA512] expose multi-buffer API --- lib/avx/mb_mgr_avx.c | 5 +++ lib/avx/sha512_x2_avx.asm | 74 ++++++++++++++++++++++++++++++++++++ lib/avx/sha_mb_avx.c | 24 ++++++++++++ lib/include/arch_avx_type1.h | 5 +++ lib/include/mb_mgr_code.h | 8 ++-- lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 12273cf6..6f2c2bd8 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -127,6 +127,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx #define SUBMIT_JOB_SHA256 submit_job_sha256_avx #define FLUSH_JOB_SHA256 flush_job_sha256_avx +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx +#define FLUSH_JOB_SHA512 flush_job_sha512_avx #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx @@ -439,6 +441,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX_NUM_SHA256_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX_NUM_SHA512_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx/sha512_x2_avx.asm b/lib/avx/sha512_x2_avx.asm index 34503e88..865008db 100644 --- a/lib/avx/sha512_x2_avx.asm +++ b/lib/avx/sha512_x2_avx.asm @@ -241,6 +241,72 @@ endstruc %endm +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + vmovdqa [rsp + 3*16], xmm6 + vmovdqa [rsp + 4*16], xmm7 + vmovdqa [rsp + 5*16], xmm8 + vmovdqa [rsp + 6*16], xmm9 + vmovdqa [rsp + 7*16], xmm10 + vmovdqa [rsp + 8*16], xmm11 + vmovdqa [rsp + 9*16], xmm12 + vmovdqa [rsp + 10*16], xmm13 + vmovdqa [rsp + 11*16], xmm14 + vmovdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + vmovdqa xmm6, [rsp + 3*16] + vmovdqa xmm7, [rsp + 4*16] + vmovdqa xmm8, [rsp + 5*16] + vmovdqa xmm9, [rsp + 6*16] + vmovdqa xmm10, [rsp + 7*16] + vmovdqa xmm11, [rsp + 8*16] + vmovdqa xmm12, [rsp + 9*16] + vmovdqa xmm13, [rsp + 10*16] + vmovdqa xmm14, [rsp + 11*16] + vmovdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa xmm5, [rsp + 3*16] + vmovdqa xmm5, [rsp + 4*16] + vmovdqa xmm5, [rsp + 5*16] + vmovdqa xmm5, [rsp + 6*16] + vmovdqa xmm5, [rsp + 7*16] + vmovdqa xmm5, [rsp + 8*16] + vmovdqa xmm5, [rsp + 9*16] + vmovdqa xmm5, [rsp + 10*16] + vmovdqa xmm5, [rsp + 11*16] + vmovdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + ;; SHA512_ARGS: ;; UINT128 digest[8]; // transposed digests ;; UINT8 *data_ptr[2]; @@ -369,4 +435,12 @@ Lrounds_16_xx: ; outer calling routine restores XMM and other GP registers ret +; void call_sha512_x2_avx_from_c(SHA512_ARGS *args, UINT64 size_in_blocks); +MKGLOBAL(call_sha512_x2_avx_from_c,function,internal) +call_sha512_x2_avx_from_c: + FUNC_SAVE + call sha512_x2_avx + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index f406bd69..add37ee3 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx_type1.h" +IMB_JOB *submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -90,3 +93,24 @@ IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha_256_mult_avx_from_c, 0); } + +/* ========================================================================== */ +/* + * SHA512 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 1, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x2_avx_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 0, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x2_avx_from_c); +} diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index a6ac773d..9c7bb420 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -159,6 +159,11 @@ IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + uint32_t hec_32_avx(const uint8_t *in); uint64_t hec_64_avx(const uint8_t *in); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 47797df8..c72debc6 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,7 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #endif #if defined (SSE) || defined (AVX512) @@ -722,7 +722,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_512: -#ifdef SSE +#if defined(SSE) || defined(AVX) return SUBMIT_JOB_SHA512(sha_512_ooo, job); #else IMB_SHA512(state, @@ -853,7 +853,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #endif #if defined(SSE) || defined (AVX512) @@ -903,7 +903,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#ifdef SSE +#if defined(SSE) || defined(AVX) case IMB_AUTH_SHA_512: return FLUSH_JOB_SHA512(sha_512_ooo, job); #endif diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 32957d9c..5c440b1f 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -65,6 +65,9 @@ extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, extern void call_sha512_x2_sse_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +extern void call_sha512_x2_avx_from_c(SHA512_ARGS *args, + uint64_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From 880fcfabc4fe959a6d39e98de7dff42b9ce412ae Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 21 Jun 2022 09:36:02 +0000 Subject: [PATCH 217/369] avx2: [SHA512] expose multi-buffer API --- lib/avx2_t1/mb_mgr_avx2.c | 5 +++ lib/avx2_t1/sha512_x4_avx2.asm | 74 ++++++++++++++++++++++++++++++++++ lib/avx2_t1/sha_mb_avx2.c | 24 +++++++++++ lib/include/arch_avx2_type1.h | 5 +++ lib/include/mb_mgr_code.h | 8 ++-- lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index bc2c1b67..34ea3dab 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -137,6 +137,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx2 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 #define FLUSH_JOB_SHA256 flush_job_sha256_avx2 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx2 #define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx #define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx @@ -419,6 +421,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx2_t1/sha512_x4_avx2.asm b/lib/avx2_t1/sha512_x4_avx2.asm index ce98b76d..88f12566 100644 --- a/lib/avx2_t1/sha512_x4_avx2.asm +++ b/lib/avx2_t1/sha512_x4_avx2.asm @@ -310,6 +310,72 @@ endstruc %endm +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + vmovdqa [rsp + 3*16], xmm6 + vmovdqa [rsp + 4*16], xmm7 + vmovdqa [rsp + 5*16], xmm8 + vmovdqa [rsp + 6*16], xmm9 + vmovdqa [rsp + 7*16], xmm10 + vmovdqa [rsp + 8*16], xmm11 + vmovdqa [rsp + 9*16], xmm12 + vmovdqa [rsp + 10*16], xmm13 + vmovdqa [rsp + 11*16], xmm14 + vmovdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + vmovdqa xmm6, [rsp + 3*16] + vmovdqa xmm7, [rsp + 4*16] + vmovdqa xmm8, [rsp + 5*16] + vmovdqa xmm9, [rsp + 6*16] + vmovdqa xmm10, [rsp + 7*16] + vmovdqa xmm11, [rsp + 8*16] + vmovdqa xmm12, [rsp + 9*16] + vmovdqa xmm13, [rsp + 10*16] + vmovdqa xmm14, [rsp + 11*16] + vmovdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + vpxor xmm5, xmm5, xmm5 + vmovdqa xmm5, [rsp + 3*16] + vmovdqa xmm5, [rsp + 4*16] + vmovdqa xmm5, [rsp + 5*16] + vmovdqa xmm5, [rsp + 6*16] + vmovdqa xmm5, [rsp + 7*16] + vmovdqa xmm5, [rsp + 8*16] + vmovdqa xmm5, [rsp + 9*16] + vmovdqa xmm5, [rsp + 10*16] + vmovdqa xmm5, [rsp + 11*16] + vmovdqa xmm5, [rsp + 12*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + ;; void sha512_x4_avx2(void *STATE, const int INP_SIZE) ;; arg 1 : STATE : pointer to input data ;; arg 2 : INP_SIZE : size of data in blocks (assumed >= 1) @@ -444,4 +510,12 @@ Lrounds_16_xx: ; outer calling routine restores XMM and other GP registers ret +; void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha512_x4_avx2_from_c,function,internal) +call_sha512_x4_avx2_from_c: + FUNC_SAVE + call sha512_x4_avx2 + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index 17319323..d0813518 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx2_type1.h" +IMB_JOB *submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -90,3 +93,24 @@ IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha256_oct_avx2_from_c, 0); } + +/* ========================================================================== */ +/* + * SHA512 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 4, 1, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x4_avx2_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 4, 0, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x4_avx2_from_c); +} diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index bea85e62..ccca8414 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -68,6 +68,11 @@ IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + void aes_cmac_256_subkey_gen_avx2(const void *key_exp, void *key1, void *key2); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index c72debc6..709b3c33 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,7 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #endif #if defined (SSE) || defined (AVX512) @@ -722,7 +722,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_512: -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA512(sha_512_ooo, job); #else IMB_SHA512(state, @@ -853,7 +853,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #endif #if defined(SSE) || defined (AVX512) @@ -903,7 +903,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_512: return FLUSH_JOB_SHA512(sha_512_ooo, job); #endif diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 5c440b1f..e7d100f5 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -68,6 +68,9 @@ extern void call_sha512_x2_sse_from_c(SHA512_ARGS *args, extern void call_sha512_x2_avx_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +extern void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, + uint64_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From a8e43870ba06ffb1b198d3a0dcaf269f12feed62 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 21 Jun 2022 10:00:06 +0000 Subject: [PATCH 218/369] avx512: [SHA512] expose multi-buffer API --- lib/avx512_t1/mb_mgr_avx512.c | 5 ++ lib/avx512_t1/sha512_x8_avx512.asm | 80 ++++++++++++++++++++++++++++++ lib/avx512_t1/sha_mb_avx512.c | 24 +++++++++ lib/include/arch_avx512_type1.h | 5 ++ lib/include/mb_mgr_code.h | 14 ------ lib/include/sha_mb_mgr.h | 3 ++ 6 files changed, 117 insertions(+), 14 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 35668c6e..179dc45a 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -122,6 +122,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx512 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 #define FLUSH_JOB_SHA256 flush_job_sha256_avx512 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx512 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx512 #define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 #define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 @@ -989,6 +991,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX512_NUM_SHA512_LANES); } IMB_DLL_LOCAL void diff --git a/lib/avx512_t1/sha512_x8_avx512.asm b/lib/avx512_t1/sha512_x8_avx512.asm index 28d895b2..af95b8f0 100644 --- a/lib/avx512_t1/sha512_x8_avx512.asm +++ b/lib/avx512_t1/sha512_x8_avx512.asm @@ -412,6 +412,78 @@ PSHUFFLE_BYTE_FLIP_MASK: mksection .text +%define XMM_STORAGE 10*16 +%define GP_STORAGE 9*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~31 ; align rsp to 32 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 + mov [rsp + 3*8], r13 + mov [rsp + 4*8], r14 + mov [rsp + 5*8], r15 +%ifndef LINUX + mov [rsp + 6*8], rsi + mov [rsp + 7*8], rdi + vmovdqa32 [rsp + 4*16], xmm6 + vmovdqa32 [rsp + 5*16], xmm7 + vmovdqa32 [rsp + 6*16], xmm8 + vmovdqa32 [rsp + 7*16], xmm9 + vmovdqa32 [rsp + 8*16], xmm10 + vmovdqa32 [rsp + 9*16], xmm11 + vmovdqa32 [rsp + 10*16], xmm12 + vmovdqa32 [rsp + 11*16], xmm13 + vmovdqa32 [rsp + 12*16], xmm14 + vmovdqa32 [rsp + 13*16], xmm15 +%endif ; LINUX + mov [rsp + 14*16], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] + mov r13, [rsp + 3*8] + mov r14, [rsp + 4*8] + mov r15, [rsp + 5*8] +%ifndef LINUX + mov rsi, [rsp + 6*8] + mov rdi, [rsp + 7*8] + vmovdqa32 xmm6, [rsp + 4*16] + vmovdqa32 xmm7, [rsp + 5*16] + vmovdqa32 xmm8, [rsp + 6*16] + vmovdqa32 xmm9, [rsp + 7*16] + vmovdqa32 xmm10, [rsp + 8*16] + vmovdqa32 xmm11, [rsp + 9*16] + vmovdqa32 xmm12, [rsp + 10*16] + vmovdqa32 xmm13, [rsp + 11*16] + vmovdqa32 xmm14, [rsp + 12*16] + vmovdqa32 xmm15, [rsp + 13*16] + +%ifdef SAFE_DATA + vpxord xmm5, xmm5, xmm5 + vmovdqa32 xmm5, [rsp + 4*16] + vmovdqa32 xmm5, [rsp + 5*16] + vmovdqa32 xmm5, [rsp + 6*16] + vmovdqa32 xmm5, [rsp + 7*16] + vmovdqa32 xmm5, [rsp + 8*16] + vmovdqa32 xmm5, [rsp + 9*16] + vmovdqa32 xmm5, [rsp + 10*16] + vmovdqa32 xmm5, [rsp + 11*16] + vmovdqa32 xmm5, [rsp + 12*16] + vmovdqa32 xmm5, [rsp + 13*16] +%endif +%endif ; LINUX + mov rsp, [rsp + 14*16] ;; rsp pointer +%endmacro + ;; void sha512_x8_avx512(void *input_data, UINT64 *digest[NUM_LANES], const int size) ;; arg 1 : rcx : pointer to input data ;; arg 2 : rdx : pointer to UINT64 digest[8][num_lanes] @@ -591,4 +663,12 @@ lastLoop: ;hash_done: ret +; void call_sha512_x8_avx512_from_c(SHA512_ARGS *args, UINT64 size_in_blocks); +MKGLOBAL(call_sha512_x8_avx512_from_c,function,internal) +call_sha512_x8_avx512_from_c: + FUNC_SAVE + call sha512_x8_avx512 + FUNC_RESTORE + ret + mksection stack-noexec diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index adbeb0e6..1c543b36 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx512_type1.h" +IMB_JOB *submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + /* ========================================================================== */ /* * SHA1 MB API @@ -91,3 +94,24 @@ IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) IMB_SHA_256_BLOCK_SIZE, SHA256_PAD_SIZE, call_sha256_x16_avx512_from_c, 0); } + +/* ========================================================================== */ +/* + * SHA512 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 8, 1, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x8_avx512_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 8, 0, 512, + IMB_SHA_512_BLOCK_SIZE, SHA512_PAD_SIZE, + call_sha512_x8_avx512_from_c); +} diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index c8533690..233785d4 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -90,6 +90,11 @@ IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_snow3g_uea2_avx512(MB_MGR_SNOW3G_OOO *state, IMB_JOB *job); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 709b3c33..379ef332 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,9 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#endif #if defined (SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -722,15 +720,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->status |= IMB_STATUS_COMPLETED_AUTH; return job; case IMB_AUTH_SHA_512: -#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA512(sha_512_ooo, job); -#else - IMB_SHA512(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif case IMB_AUTH_ZUC_EIA3_BITLEN: return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); case IMB_AUTH_ZUC256_EIA3_BITLEN: @@ -853,9 +843,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#endif #if defined(SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -903,10 +891,8 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_512: return FLUSH_JOB_SHA512(sha_512_ooo, job); -#endif case IMB_AUTH_AES_XCBC: return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); case IMB_AUTH_MD5: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index e7d100f5..590ffc5e 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -71,6 +71,9 @@ extern void call_sha512_x2_avx_from_c(SHA512_ARGS *args, extern void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); +extern void call_sha512_x8_avx512_from_c(SHA512_ARGS *args, + uint64_t size_in_blocks); + __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, const size_t offset, const unsigned lane) -- GitLab From 386de2743fb3a2d6fb50d0d2ec3da657195f0a13 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Tue, 21 Jun 2022 10:42:09 +0000 Subject: [PATCH 219/369] lib: update release notes and readme --- README | 1 + README.md | 1 + ReleaseNotes.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index d35900dd..71704eef 100644 --- a/README +++ b/README @@ -138,6 +138,7 @@ implementations. | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index 37701461..e7c2657f 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Table 2. List of supported integrity algorithms and their implementations. | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index a18b262e..d65157a1 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -10,7 +10,7 @@ Library - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added -- SHA1/224/256 multi-buffer implementation added +- SHA1/224/256/512 multi-buffer implementation added - Burst API added - Cipher-only burst API added (AES-CBC/CTR support only) - Hash-only burst API added (HMAC-SHA1/224/256/384/512 support only) -- GitLab From ce335a0f92ebd8cdfa5b70987c09d6fdfaefa07f Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 7 Jul 2022 14:41:24 +0000 Subject: [PATCH 220/369] lib: [SHA] add safe option check for clearing extra block --- lib/include/sha_mb_mgr.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 590ffc5e..d53b7124 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -442,7 +442,10 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, } while(state->lens[min_idx] != 0); ret_job = state->ldata[min_idx].job_in_lane; - +#ifdef SAFE_DATA + if (ret_job->msg_len_to_hash_in_bytes % blk_size) + memset(state->ldata[min_idx].extra_block, 0, blk_size); +#endif /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; @@ -456,7 +459,6 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, min_idx); ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; - return ret_job; } @@ -578,6 +580,10 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, } while(state->lens[min_idx] != 0); ret_job = state->ldata[min_idx].job_in_lane; +#ifdef SAFE_DATA + if (ret_job->msg_len_to_hash_in_bytes % blk_size) + memset(state->ldata[min_idx].extra_block, 0, blk_size); +#endif /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; @@ -707,6 +713,10 @@ submit_flush_job_sha_512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job, } while(state->lens[min_idx] != 0); ret_job = state->ldata[min_idx].job_in_lane; +#ifdef SAFE_DATA + if (ret_job->msg_len_to_hash_in_bytes % blk_size) + memset(state->ldata[min_idx].extra_block, 0, blk_size); +#endif /* put back processed packet into unused lanes, set job as complete */ state->unused_lanes = (state->unused_lanes << 4) | min_idx; state->num_lanes_inuse--; -- GitLab From abf788a14ce9d76ebd7e453b1a66bb06e7d023da Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 8 Jul 2022 15:24:52 +0000 Subject: [PATCH 221/369] test: [ACVP] update gmac direct api decrypt --- test/acvp_app_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 4d0232c0..96ea9f66 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -293,7 +293,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) if (use_gmac_api == 1) { gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); gmac_update(&key, &ctx, tc->aad, tc->aad_len); - gmac_finalize(&key, &ctx, tc->tag, tc->tag_len); + gmac_finalize(&key, &ctx, res_tag, tc->tag_len); } else { gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, tc->aad, tc->aad_len); -- GitLab From 4872d28234ffb53b52974e29b9da1291b2b1ffe0 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 8 Jul 2022 15:20:26 +0000 Subject: [PATCH 222/369] lib: [SHA1-NI] single buffer implementation for sse --- lib/Makefile | 1 + lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm | 24 +- lib/sse_t2/sha1_ni_x1_sse.asm | 330 +++++++++++++++++++ lib/win_x64.mak | 1 + 4 files changed, 341 insertions(+), 15 deletions(-) create mode 100644 lib/sse_t2/sha1_ni_x1_sse.asm diff --git a/lib/Makefile b/lib/Makefile index 3da34bd5..86a5409b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -407,6 +407,7 @@ asm_sse_lib_objs := \ sha512_x2_sse.o \ sha256_mult_sse.o \ sha1_ni_x2_sse.o \ + sha1_ni_x1_sse.o \ sha256_ni_x2_sse.o \ zuc_x4_sse.o \ zuc_x4_gfni_sse.o \ diff --git a/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm index 79e457f1..d4a2db0d 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha1_flush_ni_sse.asm @@ -50,7 +50,7 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" -extern sha1_ni +extern sha1_ni_x1 mksection .rodata default rel @@ -66,9 +66,11 @@ mksection .text %ifdef LINUX %define arg1 rdi %define arg2 rsi +%define arg3 rdx %else %define arg1 rcx %define arg2 rdx +%define arg3 r8 %endif %define state arg1 @@ -93,8 +95,8 @@ mksection .text %define extra_blocks arg2 %define p arg2 -%define tmp4 r8 -%define p2 r8 +%define tmp4 r9 +%define p2 r9 ; This routine clobbers rbx, rbp struc STACK @@ -133,27 +135,19 @@ flush_job_hmac_ni_sse: DBGPRINTL64 "idx:", idx copy_lane_data: - ; copy valid lane (idx) to empty lanes - mov tmp, [state + _args_data_ptr + PTR_SZ*idx] movzx len2, word [state + _lens + idx*2] - - DBGPRINTL64 "ptr", tmp - - ; there are only two lanes so if one is empty it is easy to determine which one - xor idx, 1 - mov [state + _args_data_ptr + PTR_SZ*idx], tmp - xor idx, 1 + mov arg3, idx ; No need to find min length - only two lanes available cmp len2, 0 je len_is_0 - ; Set length on both lanes to 0 - mov dword [state + _lens], 0 + ; Set length on lane to 0 + mov word [state + _lens + idx*2], 0 ; "state" and "args" are the same address, arg1 ; len is arg2 - call sha1_ni + call sha1_ni_x1 ; state is intact len_is_0: diff --git a/lib/sse_t2/sha1_ni_x1_sse.asm b/lib/sse_t2/sha1_ni_x1_sse.asm new file mode 100644 index 00000000..2e7eb8a1 --- /dev/null +++ b/lib/sse_t2/sha1_ni_x1_sse.asm @@ -0,0 +1,330 @@ +;; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Stack must be aligned to 32 bytes before call +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RDX R10 R11 +;; Windows preserves: RAX RBX RCX RBP RSI RDI R8 R9 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RDI R10 R11 +;; Linux preserves: RAX RBX RCX RDX RBP RSI R8 R9 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; +;; Linux/Windows clobbers: xmm0 - xmm15 + +%include "include/os.asm" +;%define DO_DBGPRINT +%include "include/dbgprint.asm" +%include "include/clear_regs.asm" +%include "include/cet.inc" +%include "include/mb_mgr_datastruct.asm" + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define args arg1 +%define NUM_BLKS arg2 +%define lane arg3 +%define tmp arg4 +; reso = resdq => 16 bytes +struc frame +.ABCD_SAVE reso 1 +.E_SAVE reso 1 +.ABCD_SAVEb reso 1 +.E_SAVEb reso 1 +.align resq 1 +endstruc + +%define INP r10 + +%define ABCD xmm0 +%define E0 xmm1 ; Need two E's b/c they ping pong +%define E1 xmm2 +%define MSG0 xmm3 +%define MSG1 xmm4 +%define MSG2 xmm5 +%define MSG3 xmm6 + +%define SHUF_MASK xmm14 +%define E_MASK xmm15 + +mksection .rodata +default rel +align 64 +PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x000102030405060708090a0b0c0d0e0f + dq 0x08090a0b0c0d0e0f, 0x0001020304050607 +UPPER_WORD_MASK: ;ddq 0xFFFFFFFF000000000000000000000000 + dq 0x0000000000000000, 0xFFFFFFFF00000000 + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha1_ni_x1(SHA1_ARGS *args, UINT32 size_in_blocks, uint64_t lane) +;; arg1 : pointer to args +;; arg2 : size (in blocks) ;; assumed to be >= 1 +;; arg3 : lane number + +mksection .text +MKGLOBAL(sha1_ni_x1,function,internal) +align 32 +sha1_ni_x1: + sub rsp, frame_size + + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + + ;; load input pointers + mov INP, [args + _data_ptr_sha1 + lane*PTR_SZ] + + add NUM_BLKS, INP ; pointer to end of data block -> loop exit condition + + ;; load initial digest + mov tmp, SHA1NI_DIGEST_ROW_SIZE + imul tmp, lane + movdqu ABCD, [args + tmp] + pxor E0, E0 + pinsrd E0, [args + tmp + 4*SHA1_DIGEST_WORD_SIZE], 3 + pshufd ABCD, ABCD, 0x1B + + movdqa SHUF_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK] + movdqa E_MASK, [rel UPPER_WORD_MASK] + +loop0: + ;; Copy digests + movdqa [rsp + frame.ABCD_SAVE], ABCD + movdqa [rsp + frame.E_SAVE], E0 + + ;; Only needed if not using sha1nexte for rounds 0-3 + pand E0, E_MASK + + ;; Rounds 0-3 + movdqu MSG0, [INP + 0*16] + pshufb MSG0, SHUF_MASK + paddd E0, MSG0 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + + ;; Rounds 4-7 + movdqu MSG1, [INP + 1*16] + pshufb MSG1, SHUF_MASK + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG0, MSG1 + + ;; Rounds 8-11 + movdqu MSG2, [INP + 2*16] + pshufb MSG2, SHUF_MASK + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 12-15 + movdqu MSG3, [INP + 3*16] + pshufb MSG3, SHUF_MASK + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 0 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 16-19 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 0 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 20-23 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 24-27 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 28-31 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 32-35 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 1 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 36-39 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 1 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 40-43 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 44-47 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 48-51 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 52-55 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 2 + sha1msg1 MSG0, MSG1 + pxor MSG3, MSG1 + + ;; Rounds 56-59 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 2 + sha1msg1 MSG1, MSG2 + pxor MSG0, MSG2 + + ;; Rounds 60-63 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1msg2 MSG0, MSG3 + sha1rnds4 ABCD, E1, 3 + sha1msg1 MSG2, MSG3 + pxor MSG1, MSG3 + + ;; Rounds 64-67 + sha1nexte E0, MSG0 + movdqa E1, ABCD + sha1msg2 MSG1, MSG0 + sha1rnds4 ABCD, E0, 3 + sha1msg1 MSG3, MSG0 + pxor MSG2, MSG0 + + ;; Rounds 68-71 + sha1nexte E1, MSG1 + movdqa E0, ABCD + sha1msg2 MSG2, MSG1 + sha1rnds4 ABCD, E1, 3 + pxor MSG3, MSG1 + + ;; Rounds 72-75 + sha1nexte E0, MSG2 + movdqa E1, ABCD + sha1msg2 MSG3, MSG2 + sha1rnds4 ABCD, E0, 3 + + ;; Rounds 76-79 + sha1nexte E1, MSG3 + movdqa E0, ABCD + sha1rnds4 ABCD, E1, 3 + + ;; Need to rotate E left by 30 + movdqa E1, E0 + pslld E0, 30 + psrld E1, 2 + pxor E0, E1 + + paddd ABCD, [rsp + frame.ABCD_SAVE] + paddd E0, [rsp + frame.E_SAVE] + + add INP, 64 + cmp INP, NUM_BLKS + jne loop0 + + ;; write out digests + pshufd ABCD, ABCD, 0x1B + movdqu [args + tmp], ABCD + pextrd [args + tmp + 4*SHA1_DIGEST_WORD_SIZE], E0, 3 + + ;; update input pointers + mov [args + _data_ptr_sha1 + lane*PTR_SZ], INP + +done_hash: + + ;; Clear stack frame (4*16 bytes) +%ifdef SAFE_DATA + pxor MSG0, MSG0 + pxor MSG1, MSG1 + pxor MSG2, MSG2 + pxor MSG3, MSG3 + + movdqa [rsp + 0*16], MSG0 + movdqa [rsp + 1*16], MSG0 + movdqa [rsp + 2*16], MSG0 + movdqa [rsp + 3*16], MSG0 +%endif + + add rsp, frame_size + + ret + +mksection stack-noexec diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 155a3949..b31233ce 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -227,6 +227,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha1_x4_avx.obj \ $(OBJ_DIR)\sha1_x4_sse.obj \ $(OBJ_DIR)\sha1_ni_x2_sse.obj \ + $(OBJ_DIR)\sha1_ni_x1_sse.obj \ $(OBJ_DIR)\sha1_one_block_avx.obj \ $(OBJ_DIR)\sha1_one_block_sse.obj \ $(OBJ_DIR)\sha1_x8_avx2.obj \ -- GitLab From 4155a2c6798d311ffaa1c9c42418ef39da902d7e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 15 Jun 2022 09:50:57 +0000 Subject: [PATCH 223/369] lib: [JOB API] SAFE_DATA register clearing done at algorithmic level and removed from job submit and flush --- lib/avx/crc32_by8_avx.asm | 4 ++++ lib/avx/crc32_refl_by8_avx.asm | 4 ++++ lib/avx/mb_mgr_zuc_submit_flush_avx.asm | 17 +++++++++++---- lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm | 21 +++++++++++++++++++ .../mb_mgr_hmac_sha1_flush_avx512.asm | 9 ++++++-- .../mb_mgr_hmac_sha1_submit_avx512.asm | 8 ++++++- .../mb_mgr_hmac_sha256_flush_avx512.asm | 7 ++++++- .../mb_mgr_hmac_sha256_submit_avx512.asm | 7 ++++++- .../mb_mgr_hmac_sha512_flush_avx512.asm | 8 ++++++- .../mb_mgr_hmac_sha512_submit_avx512.asm | 8 ++++++- .../mb_mgr_zuc_submit_flush_avx512.asm | 18 +++++++++++++++- lib/avx512_t2/crc32_by16_vclmul_avx512.asm | 7 +++++++ .../crc32_refl_by16_vclmul_avx512.asm | 7 +++++++ lib/include/mb_mgr_code.h | 9 -------- lib/sse_t1/crc32_by8_sse.asm | 4 ++++ lib/sse_t1/crc32_refl_by8_sse.asm | 4 ++++ lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm | 17 ++++++++++++--- 17 files changed, 135 insertions(+), 24 deletions(-) diff --git a/lib/avx/crc32_by8_avx.asm b/lib/avx/crc32_by8_avx.asm index e479784f..2403b0fa 100644 --- a/lib/avx/crc32_by8_avx.asm +++ b/lib/avx/crc32_by8_avx.asm @@ -39,6 +39,7 @@ %include "include/memcpy.asm" %include "include/reg_sizes.asm" %include "include/crc32.inc" +%include "include/clear_regs.asm" [bits 64] default rel @@ -299,6 +300,9 @@ crc32_by8_avx: vpextrd eax, xmm7, 1 .cleanup: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif ret align 32 diff --git a/lib/avx/crc32_refl_by8_avx.asm b/lib/avx/crc32_refl_by8_avx.asm index e6701f38..bbbd4a0a 100644 --- a/lib/avx/crc32_refl_by8_avx.asm +++ b/lib/avx/crc32_refl_by8_avx.asm @@ -39,6 +39,7 @@ %include "include/memcpy.asm" %include "include/reg_sizes.asm" %include "include/crc32_refl.inc" +%include "include/clear_regs.asm" [bits 64] default rel @@ -286,6 +287,9 @@ crc32_refl_by8_avx: vpextrd eax, xmm7, 2 .cleanup: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif not eax ret diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm index f862e6b3..92fb7208 100644 --- a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm +++ b/lib/avx/mb_mgr_zuc_submit_flush_avx.asm @@ -31,6 +31,7 @@ %include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" +%include "include/clear_regs.asm" %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_avx %define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_avx @@ -413,7 +414,9 @@ mksection .text %endif %%return_submit_eea3: - +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -643,7 +646,9 @@ APPEND3(%%skip_eea3_copy_,I,J): SHIFT_GP 1, idx, tmp3, tmp4, left or [state + _zuc_unused_lane_bitmask], BYTE(tmp3) %%return_flush_eea3: - +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -851,7 +856,9 @@ FLUSH_JOB_ZUC256_EEA3: mov [state + _zuc_unused_lanes], unused_lanes %%return_submit_eia3: - +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -1014,7 +1021,9 @@ APPEND(%%skip_eia3_,I): mov [state + _zuc_unused_lanes], unused_lanes %%return_flush_eia3: - +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm index 77b565fd..528f6b06 100644 --- a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm @@ -31,6 +31,7 @@ %include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" +%include "include/clear_regs.asm" %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_avx2 %define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_avx2 @@ -406,6 +407,11 @@ mksection .text %endif %%return_submit_eea3: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%else + vzeroupper +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] @@ -681,6 +687,11 @@ APPEND3(%%skip_eea3_copy_,I,J): SHIFT_GP 1, idx, tmp3, tmp4, left or [state + _zuc_unused_lane_bitmask], BYTE(tmp3) %%return_flush_eea3: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%else + vzeroupper +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] @@ -902,6 +913,11 @@ FLUSH_JOB_ZUC256_EEA3: mov [state + _zuc_unused_lanes], unused_lanes %%return_submit_eia3: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%else + vzeroupper +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] @@ -1065,6 +1081,11 @@ APPEND(%%skip_eia3_,I): mov [state + _zuc_unused_lanes], unused_lanes %%return_flush_eia3: +%ifdef SAFE_DATA + clear_all_xmms_avx_asm +%else + vzeroupper +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm index 1c923c8a..f374e064 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm @@ -44,6 +44,7 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" +%include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -349,10 +350,14 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA -return: - DBGPRINTL "---------- exit hmac flush avx512 -----------" +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif +return: + DBGPRINTL "---------- exit hmac flush avx512 -----------" mov rbp, [rsp + _gpr_save + 8*0] mov r12, [rsp + _gpr_save + 8*1] mov r13, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm index 3f7c505e..f7fc8bfb 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm @@ -47,6 +47,7 @@ %include "include/memcpy.asm" %include "include/const.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -380,8 +381,13 @@ clear_ret: mov dword [lane_data + _outer_block + 16], 0 %endif -return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif + +return: DBGPRINTL "---------- exit sha1 submit -----------" mov rbp, [rsp + _gpr_save + 8*0] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm index 95b5c12d..3a039541 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm @@ -45,6 +45,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/cet.inc" +%include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -411,9 +412,13 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA -return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif +return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm index 46144a51..a54984cc 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm @@ -47,6 +47,7 @@ %include "include/memcpy.asm" %include "include/const.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -426,9 +427,13 @@ clear_ret: %endif %endif ;; SAFE_DATA -return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif +return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm index dd079461..b5101c80 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm @@ -37,6 +37,8 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/cet.inc" +%include "include/clear_regs.asm" + extern sha512_x8_avx512 mksection .rodata @@ -364,9 +366,13 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA -return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif +return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm index eead8a22..b7a9d4be 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm @@ -39,6 +39,8 @@ %include "include/memcpy.asm" %include "include/const.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" + extern sha512_x8_avx512 mksection .rodata @@ -394,9 +396,13 @@ clear_ret: %endif %endif ;; SAFE_DATA -return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif +return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm index aa5cb101..0c2df325 100644 --- a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm @@ -32,6 +32,7 @@ %include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" +%include "include/clear_regs.asm" %ifndef SUBMIT_JOB_ZUC128_EEA3 %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_no_gfni_avx512 @@ -330,8 +331,11 @@ mksection .text %endif %%return_submit_eea3: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper - +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -550,7 +554,11 @@ mksection .text %endrep %endif +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif %%return_flush_eea3: @@ -981,7 +989,11 @@ FLUSH_JOB_ZUC256_EEA3: %endif %%return_submit_eia3: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] @@ -1178,7 +1190,11 @@ FLUSH_JOB_ZUC256_EEA3: %endrep %endif +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper +%endif %%return_flush_eia3: diff --git a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm index 19af2e44..03e38cc3 100644 --- a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm +++ b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm @@ -40,6 +40,8 @@ %include "include/clear_regs.asm" %include "include/crc32.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" + [bits 64] default rel @@ -280,6 +282,11 @@ crc32_by16_vclmul_avx512: vpextrd eax, xmm7, 1 .cleanup: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ret align 32 diff --git a/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm index 3000564d..6b9a735b 100644 --- a/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm +++ b/lib/avx512_t2/crc32_refl_by16_vclmul_avx512.asm @@ -40,6 +40,8 @@ %include "include/clear_regs.asm" %include "include/crc32_refl.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" + [bits 64] default rel @@ -270,6 +272,11 @@ crc32_refl_by16_vclmul_avx512: vpextrd eax, xmm7, 2 .cleanup: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif not eax ret diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 379ef332..5fc8fd8c 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2565,10 +2565,6 @@ submit_job_and_check(IMB_MGR *state, const int run_check) ADV_JOBS(&state->earliest_job); exit: -#ifdef SAFE_DATA - CLEAR_SCRATCH_GPS(); - CLEAR_SCRATCH_SIMD_REGS(); -#endif /* SAFE_DATA */ #ifndef LINUX RESTORE_XMMS(xmm_save); @@ -2618,11 +2614,6 @@ FLUSH_JOB(IMB_MGR *state) if (state->earliest_job == state->next_job) state->earliest_job = -1; /* becomes empty */ -#ifdef SAFE_DATA - CLEAR_SCRATCH_GPS(); - CLEAR_SCRATCH_SIMD_REGS(); -#endif /* SAFE_DATA */ - #ifndef LINUX RESTORE_XMMS(xmm_save); #endif diff --git a/lib/sse_t1/crc32_by8_sse.asm b/lib/sse_t1/crc32_by8_sse.asm index 4c7ba212..3c4dc401 100644 --- a/lib/sse_t1/crc32_by8_sse.asm +++ b/lib/sse_t1/crc32_by8_sse.asm @@ -39,6 +39,7 @@ %include "include/memcpy.asm" %include "include/reg_sizes.asm" %include "include/crc32.inc" +%include "include/clear_regs.asm" %ifndef CRC32_FN %define CRC32_FN crc32_by8_sse @@ -333,6 +334,9 @@ CRC32_FN: pextrd eax, xmm7, 1 .cleanup: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif ret align 32 diff --git a/lib/sse_t1/crc32_refl_by8_sse.asm b/lib/sse_t1/crc32_refl_by8_sse.asm index 1b022506..b5ab5860 100644 --- a/lib/sse_t1/crc32_refl_by8_sse.asm +++ b/lib/sse_t1/crc32_refl_by8_sse.asm @@ -39,6 +39,7 @@ %include "include/memcpy.asm" %include "include/reg_sizes.asm" %include "include/crc32_refl.inc" +%include "include/clear_regs.asm" %ifndef CRC32_REFL_FN %define CRC32_REFL_FN crc32_refl_by8_sse @@ -308,6 +309,9 @@ CRC32_REFL_FN: pextrd eax, xmm7, 2 .cleanup: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif not eax ret diff --git a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm index eb39bfde..d348de2e 100644 --- a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm @@ -28,10 +28,11 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" - %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/cet.inc" +%include "include/clear_regs.asm" + %ifndef SUBMIT_JOB_ZUC128_EEA3 %define SUBMIT_JOB_ZUC128_EEA3 submit_job_zuc_eea3_no_gfni_sse %define FLUSH_JOB_ZUC128_EEA3 flush_job_zuc_eea3_no_gfni_sse @@ -422,6 +423,9 @@ mksection .text %endif %%return_submit_eea3: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -652,7 +656,9 @@ APPEND3(%%skip_eea3_copy_,I,J): SHIFT_GP 1, idx, tmp3, tmp4, left or [state + _zuc_unused_lane_bitmask], BYTE(tmp3) %%return_flush_eea3: - +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -864,7 +870,9 @@ FLUSH_JOB_ZUC256_EEA3: mov [state + _zuc_unused_lanes], unused_lanes %%return_submit_eia3: - +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -1029,6 +1037,9 @@ APPEND(%%skip_eia3_,I): mov [state + _zuc_unused_lanes], unused_lanes %%return_flush_eia3: +%ifdef SAFE_DATA + clear_all_xmms_sse_asm +%endif mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] -- GitLab From 9a577a6e1211d4f018884dc55fa38ccdf0e7c0cc Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:11:33 +0000 Subject: [PATCH 224/369] lib: [ooo mgr] do not clear road block marker on OOO manager reset --- lib/x86_64/ooo_mgr_reset.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index bb65d74d..dcd10b39 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -30,13 +30,14 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" #include "include/ooo_mgr_reset.h" +#include /* offsetof() */ IMB_DLL_LOCAL void ooo_mgr_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_AES_OOO *p_mgr = (MB_MGR_AES_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_AES_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) p_mgr->unused_lanes = 0xF3210; @@ -58,7 +59,7 @@ void ooo_mgr_docsis_aes_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_DOCSIS_AES_OOO *p_mgr = (MB_MGR_DOCSIS_AES_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_DOCSIS_AES_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) p_mgr->unused_lanes = 0xF3210; @@ -73,7 +74,7 @@ void ooo_mgr_cmac_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_CMAC_OOO *p_mgr = (MB_MGR_CMAC_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_CMAC_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) p_mgr->unused_lanes = 0xF3210; @@ -88,7 +89,7 @@ void ooo_mgr_ccm_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_CCM_OOO *p_mgr = (MB_MGR_CCM_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_CCM_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) p_mgr->unused_lanes = 0xF3210; @@ -104,7 +105,7 @@ void ooo_mgr_aes_xcbc_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_AES_XCBC_OOO *p_mgr = (MB_MGR_AES_XCBC_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_AES_XCBC_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) @@ -124,7 +125,7 @@ void ooo_mgr_hmac_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_SHA_1_OOO *p_mgr = (MB_MGR_HMAC_SHA_1_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_1_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -154,7 +155,7 @@ void ooo_mgr_hmac_sha224_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -184,7 +185,7 @@ void ooo_mgr_hmac_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_SHA_256_OOO *p_mgr = (MB_MGR_HMAC_SHA_256_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_256_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -214,7 +215,7 @@ void ooo_mgr_hmac_sha384_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -250,7 +251,7 @@ void ooo_mgr_hmac_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_SHA_512_OOO *p_mgr = (MB_MGR_HMAC_SHA_512_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_SHA_512_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -286,7 +287,7 @@ void ooo_mgr_hmac_md5_reset(void *p_ooo_mgr, const unsigned num_lanes) MB_MGR_HMAC_MD5_OOO *p_mgr = (MB_MGR_HMAC_MD5_OOO *) p_ooo_mgr; unsigned i; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_HMAC_MD5_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); for (i = 0; i < num_lanes; i++) { @@ -310,7 +311,7 @@ void ooo_mgr_zuc_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_ZUC_OOO *p_mgr = (MB_MGR_ZUC_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_ZUC_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) { @@ -329,7 +330,8 @@ IMB_DLL_LOCAL void ooo_mgr_sha1_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SHA_1_OOO *p_mgr = (MB_MGR_SHA_1_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + + memset(p_mgr, 0, offsetof(MB_MGR_SHA_1_OOO,road_block)); if (num_lanes == 2) p_mgr->unused_lanes = 0xF10; /* SHANI */ @@ -346,7 +348,7 @@ void ooo_mgr_sha256_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SHA_256_OOO *p_mgr = (MB_MGR_SHA_256_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_SHA_256_OOO,road_block)); if (num_lanes == 2) p_mgr->unused_lanes = 0xF10; /* SHANI */ @@ -378,7 +380,7 @@ void ooo_mgr_des_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_DES_OOO *p_mgr = (MB_MGR_DES_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_DES_OOO,road_block)); if (num_lanes == 16) p_mgr->unused_lanes = 0xFEDCBA9876543210; @@ -389,7 +391,7 @@ void ooo_mgr_snow3g_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SNOW3G_OOO *p_mgr = (MB_MGR_SNOW3G_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_SNOW3G_OOO,road_block)); memset(p_mgr->lens, 0xff, sizeof(p_mgr->lens)); if (num_lanes == 4) { -- GitLab From cc12cdb613d86ca6eca6b2f0703ed18040b5afdc Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:13:02 +0000 Subject: [PATCH 225/369] avx512: [poly1305] clear registers when SAFE_DATA is on --- lib/avx512_t2/poly_fma_avx512.asm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index a1efe40f..a46d90d5 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -1449,8 +1449,6 @@ APPEND(%%_shuffle_blocks_, i): vmovdqa64 [rsp + _r_save + 64*5], zmm0 %endif - vzeroupper - %%_final_loop: cmp %%LEN, POLY1305_BLOCK_SIZE jb %%_poly1305_blocks_partial @@ -1664,7 +1662,10 @@ APPEND(%%_shuffle_blocks_, i): mov rsp, [rsp + _rsp_save] %ifdef SAFE_DATA - clear_scratch_gps_asm + clear_scratch_gps_asm + clear_all_zmms_asm +%else + vzeroupper %endif ;; SAFE_DATA %endmacro -- GitLab From 01b9d21e5c53413585a69a070cb53079c98d760b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:14:27 +0000 Subject: [PATCH 226/369] avx512: [aes-ecb] zmm vs ymm macro used to clear SIMD registers for SAFE_DATA --- lib/avx512_t2/aes_ecb_vaes_avx512.asm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/avx512_t2/aes_ecb_vaes_avx512.asm b/lib/avx512_t2/aes_ecb_vaes_avx512.asm index aec0951a..8159c5c1 100644 --- a/lib/avx512_t2/aes_ecb_vaes_avx512.asm +++ b/lib/avx512_t2/aes_ecb_vaes_avx512.asm @@ -191,7 +191,7 @@ align 16 %%done: %ifdef SAFE_DATA - clear_all_ymms_asm + clear_all_zmms_asm %else vzeroupper %endif -- GitLab From 9f107bb74cb2c83fc0b5b029e968cbcfda0a71e3 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:18:46 +0000 Subject: [PATCH 227/369] avx512: [aes-cbc] switch to use different manager structure constants and remove obsolete vzeroupper --- lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index d94425c2..b028ec90 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -138,7 +138,6 @@ endstruc %endif mov r15, [GPR_SAVE_AREA + 8*3] add rsp, STACK_size - vzeroupper %endmacro ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -549,7 +548,7 @@ endstruc %if %%MAC_TYPE == MAC_TYPE_XCBC %define %%KP ARG + _aes_xcbc_args_key_tab %else -%define %%KP ARG + _aesarg_key_tab +%define %%KP ARG + _aes_args_key_tab %endif %define %%K00_03_OFFSET 0 %define %%K04_07_OFFSET 64 @@ -752,9 +751,9 @@ endstruc %if %%MAC_TYPE == MAC_TYPE_XCBC -%define %%KP ARG + _aesxcbcarg_key_tab +%define %%KP ARG + _aes_xcbc_args_key_tab %else -%define %%KP ARG + _aesarg_key_tab +%define %%KP ARG + _aes_args_key_tab %endif %define %%K00_03_OFFSET 0 %define %%K04_07_OFFSET 64 @@ -934,10 +933,10 @@ endstruc %define %%IV ARG + _aes_xcbc_args_ICV %define %%IN ARG + _aes_xcbc_args_in %else -%define %%KP ARG + _aesarg_key_tab -%define %%IV ARG + _aesarg_IV -%define %%IN ARG + _aesarg_in -%define %%OUT ARG + _aesarg_out +%define %%KP ARG + _aes_args_key_tab +%define %%IV ARG + _aes_args_IV +%define %%IN ARG + _aes_args_in +%define %%OUT ARG + _aes_args_out %endif ;; check if flush -- GitLab From c80952293e6b0338569bf53e0608ba6dfecae8a7 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:21:25 +0000 Subject: [PATCH 228/369] test: [xvalid] add custom replacements for memset() and memcpy() that don't use SIMD --- test/misc.asm | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/misc.h | 6 ++++ 2 files changed, 86 insertions(+) diff --git a/test/misc.asm b/test/misc.asm index dfdb8bd0..2b62dec7 100644 --- a/test/misc.asm +++ b/test/misc.asm @@ -34,6 +34,12 @@ ;;; - type : function or data ;;; - scope : internal, private, default %define MKGLOBAL(name,type,scope) global name %+ : %+ type scope + +;;; ABI function arguments +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx %endif %ifdef WIN_ABI @@ -42,6 +48,12 @@ ;;; - type : function or data ;;; - scope : internal, private, default (ignored in win64 coff format) %define MKGLOBAL(name,type,scope) global name + +;;; ABI function arguments +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 %endif section .bss @@ -56,14 +68,74 @@ simd_regs: resb 32*64 section .text +;; void *nosimd_memcpy(void *dst, const void *src, size_t n) +MKGLOBAL(nosimd_memcpy,function,) +align 16 +nosimd_memcpy: + pushfq + push arg1 + cld ;; increment dst/src pointers + +%ifdef WIN_ABI + push rdi + push rsi + mov rdi, arg1 ;; arg1 = rcx + mov rsi, arg2 ;; arg2 = rdx + mov rcx, arg3 ;; arg3 = r8 + rep movsb + pop rsi + pop rdi +%endif + +%ifdef LINUX + ;; rdi = arg1 + ;; rsi = arg2 + mov rcx, arg3 ;; arg3 = rdx + rep movsb +%endif + + pop rax ;; return `dst` + popfq + ret + +;; void *nosimd_memset(void *p, int c, size_t n) +MKGLOBAL(nosimd_memset,function,) +align 16 +nosimd_memset: + pushfq + push arg1 + cld ;; increment dst pointer + +%ifdef WIN_ABI + push rdi + mov rdi, arg1 ;; arg1 = rcx + mov rax, arg2 ;; arg2 = rdx + mov rcx, arg3 ;; arg3 = r8 + rep stosb + pop rdi +%endif + +%ifdef LINUX + ;; rdi = arg1 + mov rax, arg2 ;; arg2 = rsi + mov rcx, arg3 ;; arg3 = rdx + rep stosb +%endif + + pop rax ;; return `p` + popfq + ret + ;; Returns RSP pointer with the value BEFORE the call, so 8 bytes need ;; to be added MKGLOBAL(rdrsp,function,) +align 16 rdrsp: lea rax, [rsp + 8] ret MKGLOBAL(dump_gps,function,) +align 16 dump_gps: mov [rel gps], rax @@ -84,6 +156,7 @@ dump_gps: ret MKGLOBAL(dump_xmms_sse,function,) +align 16 dump_xmms_sse: %assign i 0 @@ -97,6 +170,7 @@ dump_xmms_sse: ret MKGLOBAL(dump_xmms_avx,function,) +align 16 dump_xmms_avx: %assign i 0 @@ -110,6 +184,7 @@ dump_xmms_avx: ret MKGLOBAL(dump_ymms,function,) +align 16 dump_ymms: %assign i 0 @@ -123,6 +198,7 @@ dump_ymms: ret MKGLOBAL(dump_zmms,function,) +align 16 dump_zmms: %assign i 0 @@ -140,6 +216,7 @@ dump_zmms: ; ; void clr_scratch_xmms_sse(void) MKGLOBAL(clr_scratch_xmms_sse,function,internal) +align 16 clr_scratch_xmms_sse: %ifdef LINUX @@ -167,6 +244,7 @@ clr_scratch_xmms_sse: ; ; void clr_scratch_xmms_avx(void) MKGLOBAL(clr_scratch_xmms_avx,function,internal) +align 16 clr_scratch_xmms_avx: %ifdef LINUX @@ -190,6 +268,7 @@ clr_scratch_xmms_avx: ; ; void clr_scratch_ymms(void) MKGLOBAL(clr_scratch_ymms,function,internal) +align 16 clr_scratch_ymms: ; On Linux, all YMM registers are scratch registers %ifdef LINUX @@ -219,6 +298,7 @@ clr_scratch_ymms: ; ; void clr_scratch_zmms(void) MKGLOBAL(clr_scratch_zmms,function,internal) +align 16 clr_scratch_zmms: ; On Linux, all ZMM registers are scratch registers diff --git a/test/misc.h b/test/misc.h index 2fc3be28..998e62b2 100644 --- a/test/misc.h +++ b/test/misc.h @@ -55,4 +55,10 @@ void clr_scratch_xmms_avx(void); void clr_scratch_ymms(void); void clr_scratch_zmms(void); +/* custom replacement for memset() */ +void *nosimd_memset(void *p, int c, size_t n); + +/* custom replacement for memcpy() */ +void *nosimd_memcpy(void *dst, const void *src, size_t n); + #endif /* XVALIDAPP_MISC_H */ -- GitLab From 462212e34e2ec62ad311bba4c4bf29276c2c2639 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:24:44 +0000 Subject: [PATCH 229/369] test: [xvalid] replace memset() with memfill() and memcpy() with memcopy() --- test/ipsec_xvalid.c | 95 +++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index f6858754..4590f3b2 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -728,12 +728,12 @@ static void generate_patterns(void) pattern_cipher_key == 0 || pattern_plain_text == 0); - memset(&pattern8_auth_key, pattern_auth_key, - sizeof(pattern8_auth_key)); - memset(&pattern8_cipher_key, pattern_cipher_key, - sizeof(pattern8_cipher_key)); - memset(&pattern8_plain_text, pattern_plain_text, - sizeof(pattern8_plain_text)); + nosimd_memset(&pattern8_auth_key, pattern_auth_key, + sizeof(pattern8_auth_key)); + nosimd_memset(&pattern8_cipher_key, pattern_cipher_key, + sizeof(pattern8_cipher_key)); + nosimd_memset(&pattern8_plain_text, pattern_plain_text, + sizeof(pattern8_plain_text)); printf(">>> Patterns: AUTH_KEY = 0x%02x, CIPHER_KEY = 0x%02x, " "PLAIN_TEXT = 0x%02x\n", @@ -1205,19 +1205,19 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, if (force_pattern) { switch (params->hash_alg) { case IMB_AUTH_AES_XCBC: - memset(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); break; case IMB_AUTH_AES_CMAC: case IMB_AUTH_AES_CMAC_BITLEN: case IMB_AUTH_AES_CMAC_256: - memset(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); - memset(k2, pattern_auth_key, sizeof(keys->k2)); - memset(k3, pattern_auth_key, sizeof(keys->k3)); + nosimd_memset(k2, pattern_auth_key, sizeof(keys->k2)); + nosimd_memset(k3, pattern_auth_key, sizeof(keys->k3)); break; case IMB_AUTH_POLY1305: - memset(k1_expanded, pattern_auth_key, + nosimd_memset(k1_expanded, pattern_auth_key, sizeof(keys->k1_expanded)); break; case IMB_AUTH_HMAC_SHA_1: @@ -1226,14 +1226,16 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_HMAC_SHA_384: case IMB_AUTH_HMAC_SHA_512: case IMB_AUTH_MD5: - memset(ipad, pattern_auth_key, sizeof(keys->ipad)); - memset(opad, pattern_auth_key, sizeof(keys->opad)); + nosimd_memset(ipad, pattern_auth_key, + sizeof(keys->ipad)); + nosimd_memset(opad, pattern_auth_key, + sizeof(keys->opad)); break; case IMB_AUTH_ZUC_EIA3_BITLEN: case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: case IMB_AUTH_KASUMI_UIA1: - memset(k3, pattern_auth_key, sizeof(keys->k3)); + nosimd_memset(k3, pattern_auth_key, sizeof(keys->k3)); break; case IMB_AUTH_AES_CCM: case IMB_AUTH_AES_GMAC: @@ -1267,7 +1269,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_AES_GMAC_192: case IMB_AUTH_AES_GMAC_256: case IMB_AUTH_GHASH: - memset(gdata_key, pattern_auth_key, + nosimd_memset(gdata_key, pattern_auth_key, sizeof(keys->gdata_key)); break; default: @@ -1279,7 +1281,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, switch (params->cipher_mode) { case IMB_CIPHER_GCM: - memset(gdata_key, pattern_cipher_key, + nosimd_memset(gdata_key, pattern_cipher_key, sizeof(keys->gdata_key)); break; case IMB_CIPHER_PON_AES_CNTR: @@ -1290,20 +1292,20 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_CIPHER_DOCSIS_SEC_BPI: case IMB_CIPHER_ECB: case IMB_CIPHER_CBCS_1_9: - memset(enc_keys, pattern_cipher_key, + nosimd_memset(enc_keys, pattern_cipher_key, sizeof(keys->enc_keys)); - memset(dec_keys, pattern_cipher_key, + nosimd_memset(dec_keys, pattern_cipher_key, sizeof(keys->dec_keys)); break; case IMB_CIPHER_DES: case IMB_CIPHER_DES3: case IMB_CIPHER_DOCSIS_DES: - memset(enc_keys, pattern_cipher_key, + nosimd_memset(enc_keys, pattern_cipher_key, sizeof(keys->enc_keys)); break; case IMB_CIPHER_SNOW3G_UEA2_BITLEN: case IMB_CIPHER_KASUMI_UEA1_BITLEN: - memset(k2, pattern_cipher_key, 16); + nosimd_memset(k2, pattern_cipher_key, 16); break; case IMB_CIPHER_ZUC_EEA3: case IMB_CIPHER_CHACHA20: @@ -1311,7 +1313,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_CIPHER_CHACHA20_POLY1305_SGL: case IMB_CIPHER_SNOW_V: case IMB_CIPHER_SNOW_V_AEAD: - memset(k2, pattern_cipher_key, 32); + nosimd_memset(k2, pattern_cipher_key, 32); break; case IMB_CIPHER_NULL: /* No operation needed */ @@ -1339,13 +1341,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_HMAC_SHA_1: /* compute ipad hash */ - memset(buf, 0x36, IMB_SHA1_BLOCK_SIZE); + nosimd_memset(buf, 0x36, IMB_SHA1_BLOCK_SIZE); for (i = 0; i < IMB_SHA1_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, IMB_SHA1_BLOCK_SIZE); + nosimd_memset(buf, 0x5c, IMB_SHA1_BLOCK_SIZE); for (i = 0; i < IMB_SHA1_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad); @@ -1353,13 +1355,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_HMAC_SHA_224: /* compute ipad hash */ - memset(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); + nosimd_memset(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); + nosimd_memset(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad); @@ -1367,13 +1369,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_HMAC_SHA_256: /* compute ipad hash */ - memset(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); + nosimd_memset(buf, 0x36, IMB_SHA_256_BLOCK_SIZE); for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); + nosimd_memset(buf, 0x5c, IMB_SHA_256_BLOCK_SIZE); for (i = 0; i < IMB_SHA_256_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad); @@ -1381,13 +1383,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_HMAC_SHA_384: /* compute ipad hash */ - memset(buf, 0x36, IMB_SHA_384_BLOCK_SIZE); + nosimd_memset(buf, 0x36, IMB_SHA_384_BLOCK_SIZE); for (i = 0; i < IMB_SHA_384_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, IMB_SHA_384_BLOCK_SIZE); + nosimd_memset(buf, 0x5c, IMB_SHA_384_BLOCK_SIZE); for (i = 0; i < IMB_SHA_384_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad); @@ -1395,13 +1397,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_HMAC_SHA_512: /* compute ipad hash */ - memset(buf, 0x36, IMB_SHA_512_BLOCK_SIZE); + nosimd_memset(buf, 0x36, IMB_SHA_512_BLOCK_SIZE); for (i = 0; i < IMB_SHA_512_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, IMB_SHA_512_BLOCK_SIZE); + nosimd_memset(buf, 0x5c, IMB_SHA_512_BLOCK_SIZE); for (i = 0; i < IMB_SHA_512_BLOCK_SIZE; i++) buf[i] ^= auth_key[i]; IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad); @@ -1409,13 +1411,13 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_AUTH_MD5: /* compute ipad hash */ - memset(buf, 0x36, 64); + nosimd_memset(buf, 0x36, 64); for (i = 0; i < 64; i++) buf[i] ^= auth_key[i]; IMB_MD5_ONE_BLOCK(mb_mgr, buf, ipad); /* compute opad hash */ - memset(buf, 0x5c, 64); + nosimd_memset(buf, 0x5c, 64); for (i = 0; i < 64; i++) buf[i] ^= auth_key[i]; IMB_MD5_ONE_BLOCK(mb_mgr, buf, opad); @@ -1425,7 +1427,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_AUTH_ZUC256_EIA3_BITLEN: case IMB_AUTH_SNOW3G_UIA2_BITLEN: case IMB_AUTH_KASUMI_UIA1: - memcpy(k2, auth_key, sizeof(keys->k2)); + nosimd_memcpy(k2, auth_key, sizeof(keys->k2)); break; case IMB_AUTH_AES_GMAC_128: IMB_AES128_GCM_PRE(mb_mgr, auth_key, gdata_key); @@ -1468,7 +1470,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, /* No operation needed */ break; case IMB_AUTH_POLY1305: - memcpy(k1_expanded, auth_key, 32); + nosimd_memcpy(k1_expanded, auth_key, 32); break; default: fprintf(stderr, "Unsupported hash algorithm %u, line %d\n", @@ -1538,7 +1540,7 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, break; case IMB_CIPHER_SNOW3G_UEA2_BITLEN: case IMB_CIPHER_KASUMI_UEA1_BITLEN: - memcpy(k2, ciph_key, 16); + nosimd_memcpy(k2, ciph_key, 16); break; case IMB_CIPHER_ZUC_EEA3: case IMB_CIPHER_CHACHA20: @@ -1547,12 +1549,12 @@ prepare_keys(IMB_MGR *mb_mgr, struct cipher_auth_keys *keys, case IMB_CIPHER_SNOW_V: case IMB_CIPHER_SNOW_V_AEAD: /* Use of: - * memcpy(k2, ciph_key, 32); + * nosimd_memcpy(k2, ciph_key, 32); * leaves sensitive data on the stack. * Copying data in 16 byte chunks instead. */ - memcpy(k2, ciph_key, 16); - memcpy(k2 + 16, ciph_key + 16, 16); + nosimd_memcpy(k2, ciph_key, 16); + nosimd_memcpy(k2 + 16, ciph_key + 16, 16); break; case IMB_CIPHER_NULL: /* No operation needed */ @@ -1609,7 +1611,7 @@ modify_docsis_crc32_test_buf(uint8_t *test_buf, if (buf_size >= (IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE + IMB_DOCSIS_CRC32_TAG_SIZE)) { /* Set plaintext CRC32 in the test buffer */ - memcpy(&test_buf[buf_size - IMB_DOCSIS_CRC32_TAG_SIZE], + nosimd_memcpy(&test_buf[buf_size - IMB_DOCSIS_CRC32_TAG_SIZE], job->auth_tag_output, IMB_DOCSIS_CRC32_TAG_SIZE); } } @@ -1757,8 +1759,8 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, generate_random_buf(auth_iv, MAX_IV_SIZE); generate_random_buf(aad, MAX_AAD_SIZE); if (safe_check) { - memset(ciph_key, pattern_cipher_key, MAX_KEY_SIZE); - memset(auth_key, pattern_auth_key, MAX_KEY_SIZE); + nosimd_memset(ciph_key, pattern_cipher_key, MAX_KEY_SIZE); + nosimd_memset(auth_key, pattern_auth_key, MAX_KEY_SIZE); } else { generate_random_buf(ciph_key, MAX_KEY_SIZE); generate_random_buf(auth_key, MAX_KEY_SIZE); @@ -1835,7 +1837,8 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, } if (safe_check) - memset(test_buf[i], pattern_plain_text, buf_sizes[i]); + nosimd_memset(test_buf[i], pattern_plain_text, + buf_sizes[i]); else generate_random_buf(test_buf[i], buf_sizes[i]); @@ -1945,7 +1948,7 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, * Encrypt + generate digest from encrypted message * using architecture under test */ - memcpy(src_dst_buf[i], test_buf[i], buf_sizes[i]); + nosimd_memcpy(src_dst_buf[i], test_buf[i], buf_sizes[i]); if (fill_job(job, params, src_dst_buf[i], in_digest[i], aad, buf_sizes[i], tag_size, IMB_DIR_ENCRYPT, enc_keys, cipher_iv, auth_iv, i, next_iv) < 0) @@ -2861,7 +2864,7 @@ int main(int argc, char *argv[]) * Disable all the other architectures * and enable only the specified */ - memset(enc_archs, 0, sizeof(enc_archs)); + nosimd_memset(enc_archs, 0, sizeof(enc_archs)); enc_archs[values->arch_type] = 1; i++; } else if (strcmp(argv[i], "--dec-arch") == 0) { @@ -2876,7 +2879,7 @@ int main(int argc, char *argv[]) * Disable all the other architectures * and enable only the specified */ - memset(dec_archs, 0, sizeof(dec_archs)); + nosimd_memset(dec_archs, 0, sizeof(dec_archs)); dec_archs[values->arch_type] = 1; i++; } else if (strcmp(argv[i], "--cipher-algo") == 0) { -- GitLab From d7b491457cf2403003dbce4f15d7e09dbe3f6d21 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:27:37 +0000 Subject: [PATCH 230/369] test: [xvalid] printf() variants affect stack so avoid them on pattern search --- test/ipsec_xvalid.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 4590f3b2..3cf241bc 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -751,6 +751,7 @@ search_patterns(const void *ptr, const size_t mem_size) { const uint8_t *ptr8 = (const uint8_t *) ptr; const size_t limit = mem_size - sizeof(uint64_t); + const char *err_str = ""; int ret = -1; size_t i; @@ -763,14 +764,13 @@ search_patterns(const void *ptr, const size_t mem_size) const uint64_t string = *((const uint64_t *) &ptr8[i]); if (string == pattern8_cipher_key) { - fprintf(stderr, "Part of CIPHER_KEY is present\n"); + err_str = "Part of CIPHER_KEY is present"; ret = 0; } else if (string == pattern8_auth_key) { - fprintf(stderr, "Part of AUTH_KEY is present\n"); + err_str = "Part of AUTH_KEY is present"; ret = 0; } else if (string == pattern8_plain_text) { - fprintf(stderr, - "Part of plain/ciphertext is present\n"); + err_str = "Part of plain/ciphertext is present"; ret = 0; } @@ -779,15 +779,17 @@ search_patterns(const void *ptr, const size_t mem_size) } if (ret != -1) { - size_t len_to_print = mem_size - i; + static uint8_t tb[64]; + const size_t len_to_print = + (mem_size - i) > sizeof(tb) ? sizeof(tb) : mem_size - i; + + nosimd_memcpy(tb, &ptr8[i], len_to_print); + fprintf(stderr, "%s\n", err_str); fprintf(stderr, "Offset = %zu bytes, Addr = %p, RSP = %p\n", i, &ptr8[i], rdrsp()); - if (len_to_print > 64) - len_to_print = 64; - - hexdump_ex(stderr, NULL, &ptr8[i], len_to_print, &ptr8[i]); + hexdump_ex(stderr, NULL, tb, len_to_print, &ptr8[i]); return 0; } -- GitLab From 54e8c1b421b81e83e97585beffbbe39772c071b8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:28:29 +0000 Subject: [PATCH 231/369] test: [xvalid] remove clear_scratch_simd() --- test/ipsec_xvalid.c | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 3cf241bc..5df73d7a 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -1697,29 +1697,6 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) return 0; } -static void -clear_scratch_simd(const IMB_ARCH arch) -{ - switch (arch) { - case IMB_ARCH_SSE: - case IMB_ARCH_NOAESNI: - clr_scratch_xmms_sse(); - break; - case IMB_ARCH_AVX: - clr_scratch_xmms_avx(); - break; - case IMB_ARCH_AVX2: - clr_scratch_ymms(); - break; - case IMB_ARCH_AVX512: - clr_scratch_zmms(); - break; - default: - fprintf(stderr, "Invalid architecture\n"); - exit(EXIT_FAILURE); - } -} - /* Performs test using AES_HMAC or DOCSIS */ static int do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, @@ -1871,7 +1848,6 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Clear scratch registers before expanding keys to prevent * other functions from storing sensitive data in stack */ - clear_scratch_simd(enc_arch); if (prepare_keys(enc_mb_mgr, enc_keys, ciph_key, auth_key, params, 0) < 0) goto exit; @@ -1961,8 +1937,6 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Clear scratch registers before submitting job to prevent * other functions from storing sensitive data in stack */ - if (safe_check) - clear_scratch_simd(enc_arch); job = IMB_SUBMIT_JOB(enc_mb_mgr); if (job) { @@ -2082,8 +2056,6 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Clear scratch registers before submitting job to prevent * other functions from storing sensitive data in stack */ - if (safe_check) - clear_scratch_simd(dec_arch); job = IMB_SUBMIT_JOB(dec_mb_mgr); if (job != NULL) { -- GitLab From 3224138ba5ef6e25fd8bda0fadafdafb9c2adc95 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:31:08 +0000 Subject: [PATCH 232/369] test: [xvalid] add last OOO manager marker field and print OOO manager names on pattern hit --- lib/intel-ipsec-mb.h | 1 + test/ipsec_xvalid.c | 41 +++++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 1ec488c6..2093bac1 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1203,6 +1203,7 @@ typedef struct IMB_MGR { void *sha_224_ooo; void *sha_256_ooo; void *sha_512_ooo; + void *end_ooo; /* add new out-of-order managers above this line */ } IMB_MGR; /** diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 5df73d7a..f2accc39 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -97,8 +97,6 @@ static uint64_t pattern8_cipher_key; static uint64_t pattern8_plain_text; #define MAX_OOO_MGR_SIZE 8192 -#define OOO_MGR_FIRST aes128_ooo -#define OOO_MGR_LAST zuc_eia3_ooo /* Struct storing cipher parameters */ struct params_s { @@ -799,18 +797,20 @@ search_patterns(const void *ptr, const size_t mem_size) static size_t calculate_ooo_mgr_size(const void *ptr) { + const size_t max_size = MAX_OOO_MGR_SIZE - sizeof(uint64_t); size_t i; - for (i = 0; i <= (MAX_OOO_MGR_SIZE - sizeof(uint64_t)); i++) { - const uint64_t end_of_ooo_pattern = 0xDEADCAFEDEADCAFE; + for (i = 0; i <= max_size; i++) { + const uint64_t end_of_ooo_pattern = 0xDEADCAFEDEADCAFEULL; const uint8_t *ptr8 = (const uint8_t *) ptr; - const uint64_t string = *((const uint64_t *) &ptr8[i]); + const uint64_t *ptr64 = (const uint64_t *) &ptr8[i]; - if (string == end_of_ooo_pattern) + if (*ptr64 == end_of_ooo_pattern) return i + sizeof(uint64_t); } /* no marker found */ + fprintf(stderr, "No road-block marker found for %p manager!\n", ptr); return MAX_OOO_MGR_SIZE; } @@ -1680,16 +1680,37 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) } /* search OOO managers */ - for (ooo_ptr = &mgr->OOO_MGR_FIRST, i = 0; - ooo_ptr <= &mgr->OOO_MGR_LAST; + for (ooo_ptr = &mgr->aes128_ooo, i = 0; + ooo_ptr < &mgr->end_ooo; ooo_ptr++, i++) { + static const char * const ooo_names[] = { + "aes128_ooo", "aes192_ooo", "aes256_ooo", + "docsis128_sec_ooo", "docsis128_crc32_sec_ooo", + "docsis256_sec_ooo", "docsis256_crc32_sec_ooo", + "des_enc_ooo", "des_dec_ooo", + "des3_enc_ooo", "des3_dec_ooo", + "docsis_des_enc_ooo", "docsis_des_dec_ooo", + "hmac_sha_1_ooo", + "hmac_sha_224_ooo", "hmac_sha_256_ooo", + "hmac_sha_384_ooo", "hmac_sha_512_ooo", + "hmac_md5_ooo", + "aes_xcbc_ooo", "aes_ccm_ooo", "aes_cmac_ooo", + "zuc_eea3_ooo", "zuc_eia3_ooo", + "aes128_cbcs_ooo", + "zuc256_eea3_ooo", "zuc256_eia3_ooo", + "aes256_ccm_ooo", "aes256_cmac_ooo", + "snow3g_uea2_ooo", "snow3g_uia2_ooo", + "sha_1_ooo", "sha_224_ooo", "sha_256_ooo", + "sha_512_ooo", + "end_ooo" /* add new ooo manager above this line */ + }; void *ooo_mgr_p = *ooo_ptr; if (search_patterns(ooo_mgr_p, get_ooo_mgr_size(ooo_mgr_p, i)) == 0) { fprintf(stderr, - "Pattern found in 000 MGR (%d) after %s data\n", - (int)(ooo_ptr - &mgr->OOO_MGR_FIRST), dir); + "Pattern found in OOO MGR (index=%u,\"%s\") after %s data\n", + i, ooo_names[i], dir); return -1; } } -- GitLab From bc775349ff999b1cfda2204f72d8ad39b288997b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 12:32:01 +0000 Subject: [PATCH 233/369] test: [xvalid] move declaration inside the loop where it's used --- test/ipsec_xvalid.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index f2accc39..ac95edb0 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -2477,11 +2477,12 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, goto exit; } - IMB_HASH_ALG hash_alg; IMB_CIPHER_MODE c_mode; for (c_mode = IMB_CIPHER_CBC; c_mode < IMB_CIPHER_NUM; c_mode++) { + IMB_HASH_ALG hash_alg; + /* Skip IMB_CIPHER_CUSTOM */ if (c_mode == IMB_CIPHER_CUSTOM) continue; -- GitLab From 46b2cf43877421764fd068ad3ae2a726e9250e21 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 18:50:51 +0000 Subject: [PATCH 234/369] sse: [SHA] code indentation fix --- lib/sse_t1/sha_mb_sse.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 596f5547..16c07d44 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -37,16 +37,16 @@ IMB_DLL_LOCAL IMB_JOB *submit_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 1, 1, - IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_sse_from_c, 0); + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_sse_from_c, 0); } IMB_DLL_LOCAL IMB_JOB *flush_job_sha1_sse(MB_MGR_SHA_1_OOO *state, IMB_JOB *job) { return submit_flush_job_sha_1(state, job, 4, 0, 1, - IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, - call_sha1_mult_sse_from_c, 0); + IMB_SHA1_BLOCK_SIZE, SHA1_PAD_SIZE, + call_sha1_mult_sse_from_c, 0); } /* ========================================================================== */ -- GitLab From 73baa6d6472a04928ddc91683934339f2113317d Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 6 Jul 2022 18:53:03 +0000 Subject: [PATCH 235/369] lib: [SHA MGR] code indentation fixes --- lib/include/sha_mb_mgr.h | 47 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index d53b7124..488e5ed4 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -36,7 +36,7 @@ extern void call_sha1_mult_avx_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); extern void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); + uint32_t size_in_blocks); extern void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); @@ -327,10 +327,10 @@ void sha512_create_extra_blocks(MB_MGR_SHA_512_OOO *state, __forceinline IMB_JOB * submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, - const unsigned max_jobs, const int is_submit, - const int sha_type, const uint64_t blk_size, - const uint64_t pad_size, - void (*fn)(SHA1_ARGS *, uint32_t), const int shani) + const unsigned max_jobs, const int is_submit, + const int sha_type, const uint64_t blk_size, + const uint64_t pad_size, + void (*fn)(SHA1_ARGS *, uint32_t), const int shani) { unsigned lane, min_idx; IMB_JOB *ret_job = NULL; @@ -380,9 +380,9 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, if (is_submit) { /* - * SUBMIT - * - find min common length to process - */ + * SUBMIT + * - find min common length to process + */ min_idx = 0; min_len = state->lens[0]; @@ -394,11 +394,11 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, } } else { /* - * FLUSH - * - copy good (not null) lane onto empty lanes - * - find min common length to process across - * - not null lanes - */ + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ min_idx = lane; min_len = state->lens[lane]; @@ -433,7 +433,7 @@ submit_flush_job_sha_1(MB_MGR_SHA_1_OOO *state, IMB_JOB *job, /* run the algorithmic code on full selected blocks */ if(min_len >= blk_size) (*fn)(&state->args, - (uint32_t)(min_len/blk_size)); + (uint32_t)(min_len/blk_size)); /* create extra blocks */ if (state->ldata[min_idx].extra_blocks != 0) @@ -518,9 +518,9 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, if (is_submit) { /* - * SUBMIT - * - find min common length to process - */ + * SUBMIT + * - find min common length to process + */ min_idx = 0; min_len = state->lens[0]; @@ -532,11 +532,11 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, } } else { /* - * FLUSH - * - copy good (not null) lane onto empty lanes - * - find min common length to process across - * - not null lanes - */ + * FLUSH + * - copy good (not null) lane onto empty lanes + * - find min common length to process across + * - not null lanes + */ min_idx = lane; min_len = state->lens[lane]; @@ -571,7 +571,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, /* run the algorithmic code on full selected blocks */ if(min_len >= blk_size) (*fn)(&state->args, - (uint32_t)(min_len/blk_size)); + (uint32_t)(min_len/blk_size)); /* create extra blocks */ if (state->ldata[min_idx].extra_blocks != 0) @@ -595,6 +595,7 @@ submit_flush_job_sha_256(MB_MGR_SHA_256_OOO *state, IMB_JOB *job, sha_mb_generic_write_digest(ret_job->auth_tag_output, state->args.digest, sha_type, 16, min_idx); + ret_job->status |= IMB_STATUS_COMPLETED_AUTH; state->ldata[min_idx].job_in_lane = NULL; return ret_job; -- GitLab From 4562020c2a6a87324bdd6deaef42460add79d5c6 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 12 Jul 2022 14:50:02 +0000 Subject: [PATCH 236/369] lib: [SHA512] don't clear the roadblock on multi-buffer manager init --- lib/x86_64/ooo_mgr_reset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/x86_64/ooo_mgr_reset.c b/lib/x86_64/ooo_mgr_reset.c index dcd10b39..8889a9a3 100644 --- a/lib/x86_64/ooo_mgr_reset.c +++ b/lib/x86_64/ooo_mgr_reset.c @@ -365,7 +365,7 @@ void ooo_mgr_sha512_reset(void *p_ooo_mgr, const unsigned num_lanes) { MB_MGR_SHA_512_OOO *p_mgr = (MB_MGR_SHA_512_OOO *) p_ooo_mgr; - memset(p_mgr, 0, sizeof(*p_mgr)); + memset(p_mgr, 0, offsetof(MB_MGR_SHA_512_OOO,road_block)); if (num_lanes == AVX_NUM_SHA512_LANES) p_mgr->unused_lanes = 0xF10; -- GitLab From 519294a18ed098fc1ea3d2b02313c2deae23d506 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Fri, 1 Jul 2022 15:05:05 +0000 Subject: [PATCH 237/369] test: Fix for static analysis findings --- test/acvp_app_main.c | 2 +- test/hmac_sha256_sha512_test.c | 6 +++++- test/ipsec_xvalid.c | 10 +++++++++- test/snow3g_test.c | 7 ++++++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 96ea9f66..3847f3db 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -422,7 +422,7 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) IMB_JOB *job = NULL; DECLARE_ALIGNED(uint32_t enc_keys[15*4], 16); DECLARE_ALIGNED(uint32_t dec_keys[15*4], 16); - uint8_t res_tag[MAX_TAG_LENGTH]; + uint8_t res_tag[MAX_TAG_LENGTH] = {0}; if (test_case == NULL) return EXIT_FAILURE; diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index 1e7b25ed..4348d0c3 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -1054,7 +1054,7 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, { struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; uint8_t padding[16]; - uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint8_t **auths = NULL; uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; int ret = -1; uint8_t key[IMB_SHA_512_BLOCK_SIZE]; @@ -1065,6 +1065,10 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, size_t digest_len = 0; size_t block_size = 0; + if (num_jobs == 0) + return 0; + + auths = malloc(num_jobs * sizeof(void *)); if (auths == NULL) { fprintf(stderr, "Can't allocate buffer memory\n"); goto end2; diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index ac95edb0..0844463a 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -613,7 +613,7 @@ const uint8_t auth_tag_len_bytes[] = { 16, /* IMB_AUTH_CHACHA20_POLY1305_SGL */ 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ 16, /* IMB_AUTH_SNOW_V_AEAD */ - 16, /* IMB_AUTH_CRC32_ETHERNET_FCS */ + 16, /* IMB_AUTH_AES_GCM_SGL */ 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ 4, /* IMB_AUTH_CRC32_SCTP */ 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ @@ -2229,6 +2229,14 @@ test_single(IMB_MGR *enc_mgr, const IMB_ARCH enc_arch, uint64_t min_aad_sz = 0; uint64_t max_aad_sz, aad_sz; + if (params->hash_alg >= IMB_AUTH_NUM) { + if (verbose) { + fprintf(stderr, "Invalid hash alg\n"); + printf("FAIL\n"); + } + exit(EXIT_FAILURE); + } + if (params->cipher_mode == IMB_CIPHER_GCM) max_aad_sz = MAX_GCM_AAD_SIZE; else if (params->cipher_mode == IMB_CIPHER_CCM) diff --git a/test/snow3g_test.c b/test/snow3g_test.c index c0d93ca6..1b8b920d 100644 --- a/test/snow3g_test.c +++ b/test/snow3g_test.c @@ -2242,6 +2242,11 @@ validate_snow3g_f8_n_blocks_linear_mkeys(struct IMB_MGR *mb_mgr, bitOffsets = malloc(sizeof(*bitOffsets) * numVectors); bitLens = malloc(sizeof(*bitLens) * numVectors); + if (!pSrcBuff || !pSrcBuff_const || !pDstBuff || + !pDstBuff_const || !pIV || !pKey || !pKeySched || + !packetLen || !bitOffsets || !bitLens) + goto snow3g_f8_n_buff_linear_mkey_early_exit; + memset(pSrcBuff, 0, sizeof(*pSrcBuff) * numVectors); memset(pSrcBuff_const, 0, sizeof(*pSrcBuff_const) * numVectors); memset(pDstBuff, 0, sizeof(*pDstBuff) * numVectors); @@ -2468,7 +2473,7 @@ snow3g_f8_n_buff_linear_mkey_exit: if (pIV[i] != NULL) free(pIV[i]); } - +snow3g_f8_n_buff_linear_mkey_early_exit: if (pKey != NULL) free(pKey); if (pSrcBuff != NULL) -- GitLab From 3a2398bf7bc29dc511a98d10c34d06b402307f43 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 6 Jul 2022 09:20:22 +0000 Subject: [PATCH 238/369] perf: support CHACHA20-POLY1305 direct API Add support for CHACHA20-POLY1305 direct API, which is used by default in the application. --gcm-job-api parameter is replaced by --force-job-api to use job API instead of direct API on AES-GCM/GMAC/GHASH and CHACHA20-POLY1305. --- perf/ipsec_perf.c | 121 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 105 insertions(+), 16 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 8dcead19..cc555fdd 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -930,7 +930,7 @@ struct custom_job_params custom_job_params = { }; uint8_t archs[NUM_ARCHS] = {1, 1, 1, 1}; /* uses all function sets */ -int use_gcm_job_api = 0; +int use_job_api = 0; int use_gcm_sgl_api = 0; int use_unhalted_cycles = 0; /* read unhalted cycles instead of tsc */ uint64_t rd_cycles_cost = 0; /* cost of reading unhalted cycles */ @@ -960,7 +960,7 @@ typedef enum { TEST_API_NUMOF } TEST_API; -const char *str_api_list[TEST_API_NUMOF] = {"job", "burst", +const char *str_api_list[TEST_API_NUMOF] = {"single job", "burst", "cipher-only burst", "hash-only burst"}; @@ -2491,6 +2491,85 @@ do_test_gcm(struct params_s *params, return time / num_iter; } +/* Performs test using CHACHA20-POLY1305 direct API */ +static uint64_t +do_test_chacha_poly(struct params_s *params, + const uint32_t num_iter, IMB_MGR *mb_mgr, + uint8_t *p_buffer, imb_uint128_t *p_keys) +{ + uint8_t key[32]; + uint8_t auth_tag[16]; + DECLARE_ALIGNED(uint8_t iv[16], 16); + uint8_t *aad = NULL; + uint64_t time = 0; + uint32_t aux; + struct chacha20_poly1305_context_data chacha_ctx; + static uint32_t index = 0; + uint32_t buf_size = params->size_aes; + unsigned i; + + aad = (uint8_t *) malloc(sizeof(uint8_t) * params->aad_size); + if (!aad) { + fprintf(stderr, "Could not malloc AAD\n"); + free_mem(&p_buffer, &p_keys); + exit(EXIT_FAILURE); + } + +#ifndef _WIN32 + if (use_unhalted_cycles) + time = read_cycles(params->core); + else +#endif + time = __rdtscp(&aux); + + for (i = 0; i < num_iter; i++) { + uint8_t *pb = get_dst_buffer(index, p_buffer); + + if (imix_list_count != 0) + buf_size = get_next_size(i); + + IMB_CHACHA20_POLY1305_INIT(mb_mgr, key, &chacha_ctx, iv, + aad, params->aad_size); + + if (params->cipher_dir == IMB_DIR_ENCRYPT) { + IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, + &chacha_ctx, + pb, + pb, + buf_size); + IMB_CHACHA20_POLY1305_ENC_FINALIZE(mb_mgr, + &chacha_ctx, + auth_tag, + sizeof(auth_tag)); + } else { + IMB_CHACHA20_POLY1305_DEC_UPDATE(mb_mgr, key, + &chacha_ctx, + pb, + pb, + buf_size); + IMB_CHACHA20_POLY1305_DEC_FINALIZE(mb_mgr, + &chacha_ctx, + auth_tag, + sizeof(auth_tag)); + } + index = get_next_index(index); + } +#ifndef _WIN32 + if (use_unhalted_cycles) + time = (read_cycles(params->core) - + rd_cycles_cost) - time; + else +#endif + time = __rdtscp(&aux) - time; + + free(aad); + + if (!num_iter) + return time; + + return time / num_iter; +} + /* Performs test using GCM */ static uint64_t do_test_ghash(struct params_s *params, @@ -2635,15 +2714,24 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, num_iter = iter_scale; params->size_aes = size_aes; - if (params->cipher_mode == TEST_GCM && (!use_gcm_job_api)) { + if (params->cipher_mode == TEST_GCM && (!use_job_api)) { if (job_iter == 0) *times = do_test_gcm(params, 2 * num_iter, mgr, p_buffer, p_keys); else *times = do_test_gcm(params, job_iter, mgr, p_buffer, p_keys); + } else if (params->cipher_mode == TEST_AEAD_CHACHA20 && (!use_job_api)) { + if (job_iter == 0) + *times = do_test_chacha_poly(params, + 2 * num_iter, mgr, + p_buffer, p_keys); + else + *times = do_test_chacha_poly(params, + job_iter, mgr, + p_buffer, p_keys); } else if (params->hash_alg == TEST_AUTH_GHASH && - (!use_gcm_job_api)) { + (!use_job_api)) { if (job_iter == 0) *times = do_test_ghash(params, 2 * num_iter, mgr, p_buffer, p_keys); @@ -2979,8 +3067,8 @@ static void usage(void) "-o val: Use for the SHA size increment, default is 24\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n" - "--gcm-job-api: use JOB API for GCM/GHASH perf tests" - " (direct GCM/GHASH API is default)\n" + "--force-job-api: use JOB API" + " (direct API used for GCM/GHASH/CHACHA20_POLY1305 API by default)\n" "--gcm-sgl-api: use direct SGL API for GCM perf tests" " (direct GCM API is default)\n" "--threads num: for the number of threads to run" @@ -3447,8 +3535,8 @@ int main(int argc, char *argv[]) flags &= (~IMB_FLAG_SHANI_OFF); } else if (strcmp(argv[i], "--shani-off") == 0) { flags |= IMB_FLAG_SHANI_OFF; - } else if (strcmp(argv[i], "--gcm-job-api") == 0) { - use_gcm_job_api = 1; + } else if (strcmp(argv[i], "--force-job-api") == 0) { + use_job_api = 1; } else if (strcmp(argv[i], "--gcm-sgl-api") == 0) { use_gcm_sgl_api = 1; } else if (strcmp(argv[i], "--quick") == 0) { @@ -3801,14 +3889,15 @@ int main(int argc, char *argv[]) "Library version: %s\n", sha_size_incr, IMB_VERSION_STR, imb_get_version_str()); - fprintf(stderr, "API type: %s", str_api_list[test_api]); - if (test_api != TEST_API_JOB) - fprintf(stderr, " (burst size = %u)\n", burst_size); - else - fprintf(stderr, "\n"); - - fprintf(stderr, "GCM API type: %s\n", - use_gcm_job_api ? str_api_list[test_api] : "direct"); + if (!use_job_api) + fprintf(stderr, "API type: direct\n"); + else { + fprintf(stderr, "API type: %s", str_api_list[test_api]); + if (test_api != TEST_API_JOB) + fprintf(stderr, " (burst size = %u)\n", burst_size); + else + fprintf(stderr, "\n"); + } if (custom_job_params.cipher_mode == TEST_GCM) fprintf(stderr, "GCM AAD = %"PRIu64"\n", gcm_aad_size); -- GitLab From 82893e4cc0897d17cd2cbd22227f56071dc15f92 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 24 Jun 2022 08:07:46 +0000 Subject: [PATCH 239/369] perf: add SGL support Added --segment-size parameter to use SGL API (when supported), specifying the maximum size of each segment (last segment may be smaller than this value). --- perf/ipsec_perf.c | 238 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 210 insertions(+), 28 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index cc555fdd..59477275 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -966,6 +966,7 @@ const char *str_api_list[TEST_API_NUMOF] = {"single job", "burst", static TEST_API test_api = TEST_API_JOB; /* test job API by default */ static uint32_t burst_size = 0; /* num jobs to pass to burst API */ +static uint32_t segment_size = 0; /* segment size to test SGL (0 = no SGL) */ static volatile int timebox_on = 1; /* flag to stop the test loop */ static int use_timebox = 1; /* time-box feature on/off flag */ @@ -1455,7 +1456,10 @@ translate_cipher_mode(const enum test_cipher_mode_e test_mode) c_mode = IMB_CIPHER_DOCSIS_DES; break; case TEST_GCM: - c_mode = IMB_CIPHER_GCM; + if (segment_size != 0) + c_mode = IMB_CIPHER_GCM_SGL; + else + c_mode = IMB_CIPHER_GCM; break; case TEST_CCM: c_mode = IMB_CIPHER_CCM; @@ -1486,7 +1490,11 @@ translate_cipher_mode(const enum test_cipher_mode_e test_mode) c_mode = IMB_CIPHER_CHACHA20; break; case TEST_AEAD_CHACHA20: - c_mode = IMB_CIPHER_CHACHA20_POLY1305; + if (segment_size != 0) + c_mode = IMB_CIPHER_CHACHA20_POLY1305_SGL; + else + c_mode = IMB_CIPHER_CHACHA20_POLY1305; + break; case TEST_SNOW_V: c_mode = IMB_CIPHER_SNOW_V; @@ -1570,6 +1578,31 @@ set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, } } +static inline void +set_sgl_job_fields(IMB_JOB *job, const uint8_t *src, uint8_t *dst, + const uint32_t *keys, const uint8_t *aad, + const uint64_t len, + IMB_SGL_STATE state, + struct gcm_context_data *gcm_ctx, + struct chacha20_poly1305_context_data *cp_ctx) +{ + job->src = src; + job->dst = dst; + job->msg_len_to_cipher_in_bytes = len; + job->msg_len_to_hash_in_bytes = len; + if (job->cipher_mode == IMB_CIPHER_GCM_SGL) { + job->u.GCM.aad = aad; + job->u.GCM.ctx = gcm_ctx; + } else { + job->u.CHACHA20_POLY1305.aad = aad; + job->u.CHACHA20_POLY1305.ctx = cp_ctx; + } + + job->enc_keys = job->dec_keys = keys; + job->sgl_state = state; +}; + + static void set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, uint64_t *xgem_hdr_list, struct params_s *params) @@ -1730,7 +1763,10 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.hash_alg = IMB_AUTH_AES_CCM; break; case TEST_HASH_GCM: - job_template.hash_alg = IMB_AUTH_AES_GMAC; + if (segment_size != 0) + job_template.hash_alg = IMB_AUTH_GCM_SGL; + else + job_template.hash_alg = IMB_AUTH_AES_GMAC; break; case TEST_DOCSIS_CRC32: job_template.hash_alg = IMB_AUTH_DOCSIS_CRC32; @@ -1761,7 +1797,10 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.hash_alg = IMB_AUTH_POLY1305; break; case TEST_AEAD_POLY1305: - job_template.hash_alg = IMB_AUTH_CHACHA20_POLY1305; + if (segment_size != 0) + job_template.hash_alg = IMB_AUTH_CHACHA20_POLY1305_SGL; + else + job_template.hash_alg = IMB_AUTH_CHACHA20_POLY1305; break; case TEST_PON_CRC_BIP: job_template.hash_alg = IMB_AUTH_PON_CRC_BIP; @@ -1889,7 +1928,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* Translating enum to the API's one */ job_template.cipher_mode = translate_cipher_mode(params->cipher_mode); job_template.key_len_in_bytes = params->aes_key_size; - if (job_template.cipher_mode == IMB_CIPHER_GCM) { + if (job_template.cipher_mode == IMB_CIPHER_GCM || + job_template.cipher_mode == IMB_CIPHER_GCM_SGL) { switch (params->aes_key_size) { case IMB_KEY_128_BYTES: IMB_AES128_GCM_PRE(mb_mgr, gcm_key, &gdata_key); @@ -1948,7 +1988,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job_template.iv_len_in_bytes = 0; else if (job_template.cipher_mode == IMB_CIPHER_CHACHA20) job_template.iv_len_in_bytes = 12; - else if (job_template.cipher_mode == IMB_CIPHER_CHACHA20_POLY1305) { + else if (job_template.cipher_mode == IMB_CIPHER_CHACHA20_POLY1305 || + job_template.cipher_mode == IMB_CIPHER_CHACHA20_POLY1305_SGL) { job_template.hash_start_src_offset_in_bytes = 0; job_template.cipher_start_src_offset_in_bytes = 0; job_template.enc_keys = k1_expanded; @@ -2180,10 +2221,76 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } else { /* test job api */ for (i = 0; (i < num_iter) && timebox_on; i++) { + job = IMB_GET_NEXT_JOB(mb_mgr); *job = job_template; - set_job_fields(job, p_buffer, p_keys, i, index); + if (segment_size != 0) { + /* SGL */ + struct gcm_context_data gcm_ctx; + struct chacha20_poly1305_context_data cp_ctx; + const uint32_t *keys = + (const uint32_t *) get_key_pointer(index, + p_keys); + + uint8_t *src = get_src_buffer(index, p_buffer); + uint8_t *dst = get_dst_buffer(index, p_buffer); + uint8_t *aad = src; + + set_sgl_job_fields(job, NULL, NULL, keys, aad, 0, + IMB_SGL_INIT, &gcm_ctx, &cp_ctx); +#ifdef DEBUG + job = IMB_SUBMIT_JOB(mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); +#endif + + const uint32_t num_segs = cipher_size_list[0] / + segment_size; + const uint32_t final_seg_sz = cipher_size_list[0] % + segment_size; + uint32_t j; + + for (j = 0; j < num_segs; j++) { + job = IMB_GET_NEXT_JOB(mb_mgr); + *job = job_template; + + set_sgl_job_fields(job, &src[j*segment_size], + &dst[j*segment_size], keys, + aad, segment_size, + IMB_SGL_UPDATE, + &gcm_ctx, &cp_ctx); +#ifdef DEBUG + job = IMB_SUBMIT_JOB(mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); +#endif + } + if (final_seg_sz != 0) { + job = IMB_GET_NEXT_JOB(mb_mgr); + *job = job_template; + + set_sgl_job_fields(job, &src[j*segment_size], + &dst[j*segment_size], keys, + aad, final_seg_sz, + IMB_SGL_UPDATE, + &gcm_ctx, &cp_ctx); +#ifdef DEBUG + job = IMB_SUBMIT_JOB(mb_mgr); +#else + job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); +#endif + } + job = IMB_GET_NEXT_JOB(mb_mgr); + *job = job_template; + + set_sgl_job_fields(job, NULL, NULL, + keys, aad, + 0, + IMB_SGL_COMPLETE, + &gcm_ctx, &cp_ctx); + } else /* Linear buffer */ + set_job_fields(job, p_buffer, p_keys, i, index); index = get_next_index(index); #ifdef DEBUG @@ -2275,17 +2382,46 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, uint8_t auth_tag[12]; DECLARE_ALIGNED(uint8_t iv[16], 16); - for (i = 0; i < num_iter; i++) { - uint8_t *pb = get_dst_buffer(index, p_buffer); + /* SGL */ + if (segment_size != 0) { + for (i = 0; i < num_iter; i++) { + uint8_t *pb = get_dst_buffer(index, p_buffer); - if (imix_list_count != 0) - buf_size = get_next_size(i); + if (imix_list_count != 0) + buf_size = get_next_size(i); + + const uint32_t num_segs = buf_size / segment_size; + const uint32_t final_seg_sz = buf_size % segment_size; + uint32_t j; + + init(gdata_key, gdata_ctx, iv, aad, aad_size); + for (j = 0; j < num_segs; j++) + update(gdata_key, gdata_ctx, + &pb[j*segment_size], + &pb[j*segment_size], + segment_size); + if (final_seg_sz != 0) + update(gdata_key, gdata_ctx, + &pb[j*segment_size], + &pb[j*segment_size], + final_seg_sz); + finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); + + index = get_next_index(index); + } + } else { + for (i = 0; i < num_iter; i++) { + uint8_t *pb = get_dst_buffer(index, p_buffer); - init(gdata_key, gdata_ctx, iv, aad, aad_size); - update(gdata_key, gdata_ctx, pb, pb, buf_size); - finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); + if (imix_list_count != 0) + buf_size = get_next_size(i); - index = get_next_index(index); + init(gdata_key, gdata_ctx, iv, aad, aad_size); + update(gdata_key, gdata_ctx, pb, pb, buf_size); + finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); + + index = get_next_index(index); + } } } @@ -2329,6 +2465,10 @@ do_test_gcm(struct params_s *params, uint64_t time = 0; uint32_t aux; + /* Force SGL API if segment size is not 0 */ + if (segment_size != 0) + use_gcm_sgl_api = 1; + key = (uint8_t *) malloc(sizeof(uint8_t) * params->aes_key_size); if (!key) { fprintf(stderr, "Could not malloc key\n"); @@ -2505,8 +2645,9 @@ do_test_chacha_poly(struct params_s *params, uint32_t aux; struct chacha20_poly1305_context_data chacha_ctx; static uint32_t index = 0; - uint32_t buf_size = params->size_aes; - unsigned i; + uint32_t num_segs; + uint32_t final_seg_sz; + unsigned i, j; aad = (uint8_t *) malloc(sizeof(uint8_t) * params->aad_size); if (!aad) { @@ -2515,6 +2656,14 @@ do_test_chacha_poly(struct params_s *params, exit(EXIT_FAILURE); } + if (segment_size != 0) { + num_segs = params->size_aes / segment_size; + final_seg_sz = params->size_aes % segment_size; + } else { + num_segs = 0; + final_seg_sz = params->size_aes; + } + #ifndef _WIN32 if (use_unhalted_cycles) time = read_cycles(params->core); @@ -2525,28 +2674,51 @@ do_test_chacha_poly(struct params_s *params, for (i = 0; i < num_iter; i++) { uint8_t *pb = get_dst_buffer(index, p_buffer); - if (imix_list_count != 0) - buf_size = get_next_size(i); + if (imix_list_count != 0) { + uint32_t buf_size = get_next_size(i); + + if (segment_size != 0) { + num_segs = buf_size / segment_size; + final_seg_sz = buf_size % segment_size; + } else { + num_segs = 0; + final_seg_sz = buf_size; + } + } IMB_CHACHA20_POLY1305_INIT(mb_mgr, key, &chacha_ctx, iv, aad, params->aad_size); if (params->cipher_dir == IMB_DIR_ENCRYPT) { - IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, + for (j = 0; j < num_segs; j++) + IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, + &chacha_ctx, + &pb[j*segment_size], + &pb[j*segment_size], + segment_size); + if (final_seg_sz != 0) + IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, &chacha_ctx, - pb, - pb, - buf_size); + &pb[j*segment_size], + &pb[j*segment_size], + final_seg_sz); IMB_CHACHA20_POLY1305_ENC_FINALIZE(mb_mgr, &chacha_ctx, auth_tag, sizeof(auth_tag)); - } else { - IMB_CHACHA20_POLY1305_DEC_UPDATE(mb_mgr, key, + } else { /* IMB_DIR_DECRYPT */ + for (j = 0; j < num_segs; j++) + IMB_CHACHA20_POLY1305_ENC_UPDATE(mb_mgr, key, + &chacha_ctx, + &pb[j*segment_size], + &pb[j*segment_size], + segment_size); + if (final_seg_sz != 0) + IMB_CHACHA20_POLY1305_DEC_UPDATE(mb_mgr, key, &chacha_ctx, - pb, - pb, - buf_size); + &pb[j*segment_size], + &pb[j*segment_size], + final_seg_sz); IMB_CHACHA20_POLY1305_DEC_FINALIZE(mb_mgr, &chacha_ctx, auth_tag, @@ -3086,6 +3258,7 @@ static void usage(void) " - range: test multiple sizes with following format" " min:step:max (e.g. 16:16:256)\n" " (-o still applies for MAC)\n" + "--segment-size: size of segment to test SGL (default: 0)\n" "--imix: set numbers that establish occurrence proportions" " between packet sizes.\n" " It requires a list of sizes through --job-size.\n" @@ -3694,6 +3867,15 @@ int main(int argc, char *argv[]) "more than %d\n", MAX_BURST_SIZE); return EXIT_FAILURE; } + } else if (strcmp(argv[i], "--segment-size") == 0) { + i = get_next_num_arg((const char * const *)argv, i, + argc, &segment_size, + sizeof(segment_size)); + if (segment_size > (JOB_SIZE_TOP)) { + fprintf(stderr, "Segment size cannot be " + "more than %d\n", JOB_SIZE_TOP); + return EXIT_FAILURE; + } } else if (strcmp(argv[i], "--no-time-box") == 0) { use_timebox = 0; } else { -- GitLab From a7bd331438a345c8f24f3666e9a084b12bb9f741 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 24 Jun 2022 14:34:54 +0000 Subject: [PATCH 240/369] lib: new SGL job API --- lib/avx512_t2/poly_fma_avx512.asm | 2 +- lib/include/job_api_gcm.h | 136 +++++++- lib/include/mb_mgr_code.h | 10 + lib/intel-ipsec-mb.h | 33 +- lib/x86_64/chacha20_poly1305.c | 524 ++++++++++++++---------------- perf/ipsec_perf.c | 189 ++++++----- 6 files changed, 510 insertions(+), 384 deletions(-) diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index a46d90d5..d13d7b58 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -1778,7 +1778,7 @@ poly1305_aead_complete_fma_avx512: ;; clear Poly key %ifdef SAFE_DATA - vpxorq ymm0, ymm0 + vpxorq xmm0, xmm0 vmovdqu64 [arg2], ymm0 %endif diff --git a/lib/include/job_api_gcm.h b/lib/include/job_api_gcm.h index 6e8180f7..6190ce7c 100644 --- a/lib/include/job_api_gcm.h +++ b/lib/include/job_api_gcm.h @@ -48,11 +48,31 @@ submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) IMB_AES128_GCM_ENC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES128_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); + IMB_AES128_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } break; case IMB_KEY_192_BYTES: if (job->sgl_state == IMB_SGL_INIT) @@ -67,11 +87,31 @@ submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) + IMB_AES192_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES192_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); IMB_AES192_GCM_ENC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + } break; case IMB_KEY_256_BYTES: default: @@ -87,15 +127,35 @@ submit_gcm_sgl_enc(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) + IMB_AES256_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES256_GCM_ENC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); IMB_AES256_GCM_ENC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + } break; } - job->status |= IMB_STATUS_COMPLETED_CIPHER; + job->status = IMB_STATUS_COMPLETED; return job; } @@ -118,11 +178,31 @@ submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) IMB_AES128_GCM_DEC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES128_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES128_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); + IMB_AES128_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } break; case IMB_KEY_192_BYTES: if (job->sgl_state == IMB_SGL_INIT) @@ -137,11 +217,31 @@ submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) IMB_AES192_GCM_DEC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES192_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES192_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); + IMB_AES192_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } break; case IMB_KEY_256_BYTES: default: @@ -157,15 +257,35 @@ submit_gcm_sgl_dec(IMB_MGR *state, IMB_JOB *job) job->u.GCM.ctx, job->dst, job->src, job->msg_len_to_cipher_in_bytes); - else /* FINALIZE */ + else if (job->sgl_state == IMB_SGL_COMPLETE) IMB_AES256_GCM_DEC_FINALIZE(state, job->enc_keys, job->u.GCM.ctx, job->auth_tag_output, job->auth_tag_output_len_in_bytes); + else { /* IMB_SGL_ALL */ + unsigned int i; + + IMB_AES256_GCM_INIT_VAR_IV(state, job->enc_keys, + job->u.GCM.ctx, + job->iv, + job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes); + for (i = 0; i < job->num_sgl_io_segs; i++) + IMB_AES256_GCM_DEC_UPDATE(state, job->enc_keys, + job->u.GCM.ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len); + IMB_AES256_GCM_ENC_FINALIZE(state, job->enc_keys, + job->u.GCM.ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } break; } - job->status |= IMB_STATUS_COMPLETED_CIPHER; + job->status = IMB_STATUS_COMPLETED; return job; } diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 5fc8fd8c..e65a30e5 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2740,9 +2740,19 @@ uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, else SUBMIT_JOB_AES_GCM_DEC(state, job); completed_jobs++; + } else if (job->cipher_mode == IMB_CIPHER_GCM_SGL) { + if (job->cipher_direction == IMB_DIR_ENCRYPT) + submit_gcm_sgl_enc(state, job); + else + submit_gcm_sgl_dec(state, job); + completed_jobs++; } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { SUBMIT_JOB_CHACHA20_POLY1305(state, job); completed_jobs++; + } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == + job->cipher_mode) { + SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); + completed_jobs++; } else { if (submit_new_job(state, job) != NULL) completed_jobs++; diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 2093bac1..4942cc0e 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -448,9 +448,19 @@ typedef enum { typedef enum { IMB_SGL_INIT = 0, IMB_SGL_UPDATE, - IMB_SGL_COMPLETE + IMB_SGL_COMPLETE, + IMB_SGL_ALL } IMB_SGL_STATE; +/** + * Input/output SGL segment structure. + */ +struct IMB_SGL_IOV { + const void *in; /**< Input segment */ + void *out; /**< Output segment */ + uint64_t len; /** Length of segment */ +}; + /** * Job structure. * @@ -474,10 +484,20 @@ typedef struct IMB_JOB { const void *enc_keys; /**< Encryption key pointer */ const void *dec_keys; /**< Decryption key pointer */ uint64_t key_len_in_bytes; /**< Key length in bytes */ - const uint8_t *src; /**< Input buffer. May be ciphertext or plaintext. - In-place ciphering allowed. */ - uint8_t *dst; /**< Output buffer. May be ciphertext or plaintext. - In-place ciphering allowed, i.e. dst = src. */ + union { + const uint8_t *src; /**< Input buffer. + May be ciphertext or plaintext. + In-place ciphering allowed. */ + const struct IMB_SGL_IOV *sgl_io_segs; + /**< Pointer to array of input/output SGL segments */ + }; + union { + uint8_t *dst; /**< Output buffer. + May be ciphertext or plaintext. + In-place ciphering allowed, i.e. dst = src. */ + uint64_t num_sgl_io_segs; + /**< Number of input/output SGL segments */ + }; union { uint64_t cipher_start_src_offset_in_bytes; /**< Offset into input buffer to start ciphering (in bytes) */ @@ -610,7 +630,8 @@ typedef struct IMB_JOB { /**< Customer hash function */ IMB_SGL_STATE sgl_state; - /**< SGL state (IMB_SGL_INIT/IMB_SGL_UPDATE/IMB_SGL_COMPLETE) */ + /**< SGL state (IMB_SGL_INIT/IMB_SGL_UPDATE/IMB_SGL_COMPLETE/ + IMB_SGL_ALL) */ union { struct _CBCS_specific_fields { diff --git a/lib/x86_64/chacha20_poly1305.c b/lib/x86_64/chacha20_poly1305.c index 50899cfe..c462392d 100644 --- a/lib/x86_64/chacha20_poly1305.c +++ b/lib/x86_64/chacha20_poly1305.c @@ -162,41 +162,69 @@ void init_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, job->dst, job->msg_len_to_cipher_in_bytes, job->enc_keys, ctx, arch); } - job->status |= IMB_STATUS_COMPLETED; + job->status = IMB_STATUS_COMPLETED; } __forceinline -void update_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, - const unsigned ifma) +void update_chacha20_poly1305_direct(const void *key, + struct chacha20_poly1305_context_data *ctx, + void *dst, const void *src, + const uint64_t len, + const IMB_CIPHER_DIRECTION dir, + const IMB_ARCH arch, + const unsigned check_param, + const unsigned ifma) { - struct chacha20_poly1305_context_data *ctx = - job->u.CHACHA20_POLY1305.ctx; - uint64_t hash_len = job->msg_len_to_hash_in_bytes; +#ifdef SAFE_PARAM + if (check_param) { + /* reset error status */ + imb_set_errno(NULL, 0); + + if (key == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_KEY); + return; + } + if (ctx == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_CTX); + return; + } + if (src == NULL && len != 0) { + imb_set_errno(NULL, IMB_ERR_NULL_SRC); + return; + } + if (dst == NULL && len != 0) { + imb_set_errno(NULL, IMB_ERR_NULL_DST); + return; + } + } +#else + (void) check_param; +#endif uint64_t bytes_to_copy = 0; uint64_t remain_bytes_to_fill = (16 - ctx->remain_ct_bytes); uint64_t remain_ct_bytes; const uint8_t *remain_ct_ptr; + const uint8_t *src8 = (const uint8_t *) src; + uint8_t *dst8 = (uint8_t *) dst; + uint64_t length = len; /* Need to copy more bytes into scratchpad */ if ((ctx->remain_ct_bytes > 0) && (remain_bytes_to_fill > 0)) { - if (hash_len < remain_bytes_to_fill) - bytes_to_copy = hash_len; + if (len < remain_bytes_to_fill) + bytes_to_copy = length; else bytes_to_copy = remain_bytes_to_fill; } /* Increment total hash length */ - ctx->hash_len += job->msg_len_to_hash_in_bytes; + ctx->hash_len += length; - if (job->cipher_direction == IMB_DIR_ENCRYPT) { - chacha20_enc_dec_ks(job->src + - job->cipher_start_src_offset_in_bytes, - job->dst, job->msg_len_to_cipher_in_bytes, - job->enc_keys, ctx, arch); + if (dir == IMB_DIR_ENCRYPT) { + chacha20_enc_dec_ks(src, dst, length, key, ctx, arch); /* Copy more bytes on Poly scratchpad */ memcpy_asm(ctx->poly_scratch + ctx->remain_ct_bytes, - job->dst, bytes_to_copy, arch); + dst, bytes_to_copy, arch); ctx->remain_ct_bytes += bytes_to_copy; /* @@ -209,16 +237,16 @@ void update_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, ctx->remain_ct_bytes = 0; } - hash_len -= bytes_to_copy; - remain_ct_bytes = hash_len & HASH_REMAIN_CLAMP; - hash_len &= hash_len & HASH_LEN_CLAMP; + length -= bytes_to_copy; + remain_ct_bytes = length & HASH_REMAIN_CLAMP; + length &= HASH_LEN_CLAMP; /* compute hash after cipher on encrypt */ - poly1305_aead_update(job->dst + bytes_to_copy, - hash_len, ctx->hash, ctx->poly_key, arch, + poly1305_aead_update(dst8 + bytes_to_copy, + length, ctx->hash, ctx->poly_key, arch, ifma); - remain_ct_ptr = job->dst + bytes_to_copy + hash_len; + remain_ct_ptr = dst8 + bytes_to_copy + length; /* copy last bytes of ciphertext (less than 16 bytes) */ memcpy_asm(ctx->poly_scratch, remain_ct_ptr, remain_ct_bytes, arch); @@ -226,8 +254,7 @@ void update_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, } else { /* Copy more bytes on Poly scratchpad */ memcpy_asm(ctx->poly_scratch + ctx->remain_ct_bytes, - job->src + job->hash_start_src_offset_in_bytes, - bytes_to_copy, arch); + src, bytes_to_copy, arch); ctx->remain_ct_bytes += bytes_to_copy; /* @@ -240,28 +267,37 @@ void update_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, ctx->remain_ct_bytes = 0; } - hash_len -= bytes_to_copy; - remain_ct_bytes = hash_len & HASH_REMAIN_CLAMP; - hash_len &= hash_len & HASH_LEN_CLAMP; + length -= bytes_to_copy; + remain_ct_bytes = length & HASH_REMAIN_CLAMP; + length &= HASH_LEN_CLAMP; /* compute hash first on decrypt */ - poly1305_aead_update(job->src + - job->hash_start_src_offset_in_bytes + - bytes_to_copy, hash_len, ctx->hash, ctx->poly_key, - arch, ifma); + poly1305_aead_update(src8 + bytes_to_copy, length, ctx->hash, + ctx->poly_key, arch, ifma); - remain_ct_ptr = job->src + job->hash_start_src_offset_in_bytes - + bytes_to_copy + hash_len; + remain_ct_ptr = src8 + bytes_to_copy + length; /* copy last bytes of ciphertext (less than 16 bytes) */ memcpy_asm(ctx->poly_scratch, remain_ct_ptr, remain_ct_bytes, arch); ctx->remain_ct_bytes += remain_ct_bytes; - chacha20_enc_dec_ks(job->src + - job->cipher_start_src_offset_in_bytes, - job->dst, job->msg_len_to_cipher_in_bytes, - job->enc_keys, ctx, arch); + chacha20_enc_dec_ks(src, dst, len, key, ctx, arch); } - job->status |= IMB_STATUS_COMPLETED; +} + +__forceinline +void update_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, + const unsigned ifma) +{ + update_chacha20_poly1305_direct(job->enc_keys, + job->u.CHACHA20_POLY1305.ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->cipher_direction, + arch, + 0, ifma); + job->status = IMB_STATUS_COMPLETED; } __forceinline @@ -370,13 +406,136 @@ void complete_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, clear_mem(ctx->last_ks, sizeof(ctx->last_ks)); clear_mem(ctx->poly_key, sizeof(ctx->poly_key)); #endif - job->status |= IMB_STATUS_COMPLETED; + job->status = IMB_STATUS_COMPLETED; +} + +__forceinline +void init_chacha20_poly1305_direct(const void *key, + struct chacha20_poly1305_context_data *ctx, + const void *iv, const void *aad, + const uint64_t aad_len, const IMB_ARCH arch, + const unsigned check_params, + const unsigned ifma) +{ +#ifdef SAFE_PARAM + if (check_params) { + /* reset error status */ + imb_set_errno(NULL, 0); + + if (key == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_KEY); + return; + } + if (ctx == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_CTX); + return; + } + if (iv == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_IV); + return; + } + if (aad == NULL && aad_len != 0) { + imb_set_errno(NULL, IMB_ERR_NULL_AAD); + return; + } + } +#else + (void) check_params; +#endif + ctx->hash[0] = 0; + ctx->hash[1] = 0; + ctx->hash[2] = 0; + ctx->aad_len = aad_len; + ctx->hash_len = 0; + ctx->last_block_count = 0; + ctx->remain_ks_bytes = 0; + ctx->remain_ct_bytes = 0; + + /* Store IV */ + memcpy_asm(ctx->IV, iv, 12, arch); + + /* Generate Poly key */ + if (arch == IMB_ARCH_SSE) + poly1305_key_gen_sse(key, iv, ctx->poly_key); + else + poly1305_key_gen_avx(key, iv, ctx->poly_key); + + /* Calculate hash over AAD */ + poly1305_aead_update(aad, aad_len, ctx->hash, ctx->poly_key, + arch, ifma); +} + +__forceinline +void +finalize_chacha20_poly1305_direct(struct chacha20_poly1305_context_data *ctx, + void *tag, const uint64_t tag_len, + const IMB_ARCH arch, + const unsigned check_params, + const unsigned ifma) +{ +#ifdef SAFE_PARAM + if (check_params) { + /* reset error status */ + imb_set_errno(NULL, 0); + + if (ctx == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_CTX); + return; + } + if (tag == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_AUTH); + return; + } + if (tag_len == 0 || tag_len > 16) { + imb_set_errno(NULL, IMB_ERR_AUTH_TAG_LEN); + return; + } + } +#else + (void) check_params; +#endif + uint64_t last[2]; + uint8_t auth_tag[16]; + + if (ctx->remain_ct_bytes > 0) { + poly1305_aead_update(ctx->poly_scratch, + ctx->remain_ct_bytes, + ctx->hash, + ctx->poly_key, + arch, ifma); + ctx->remain_ct_bytes = 0; + } + + /* + * Construct extra block with AAD and message lengths for + * authentication + */ + last[0] = ctx->aad_len; + last[1] = ctx->hash_len; + poly1305_aead_update(last, sizeof(last), ctx->hash, ctx->poly_key, + arch, ifma); + + /* Finalize AEAD Poly1305 (final reduction and +S) */ + poly1305_aead_complete(ctx->hash, ctx->poly_key, auth_tag, arch, ifma); + + /* Copy N bytes of tag */ + memcpy_asm((uint8_t *) tag, auth_tag, tag_len, arch); + + /* Clear sensitive data from the context */ +#ifdef SAFE_DATA + clear_mem(ctx->last_ks, sizeof(ctx->last_ks)); + clear_mem(ctx->poly_key, sizeof(ctx->poly_key)); +#endif } __forceinline IMB_JOB *aead_chacha20_poly1305_sgl(IMB_JOB *job, const IMB_ARCH arch, const unsigned ifma) { + unsigned i; + struct chacha20_poly1305_context_data *ctx = + job->u.CHACHA20_POLY1305.ctx; + switch (job->sgl_state) { case IMB_SGL_INIT: init_chacha20_poly1305(job, arch, ifma); @@ -385,8 +544,35 @@ IMB_JOB *aead_chacha20_poly1305_sgl(IMB_JOB *job, const IMB_ARCH arch, update_chacha20_poly1305(job, arch, ifma); break; case IMB_SGL_COMPLETE: - default: complete_chacha20_poly1305(job, arch, ifma); + break; + case IMB_SGL_ALL: + default: + init_chacha20_poly1305_direct(job->enc_keys, + ctx, + job->iv, job->u.CHACHA20_POLY1305.aad, + job->u.CHACHA20_POLY1305.aad_len_in_bytes, + arch, 0, ifma); + for (i = 0; i < job->num_sgl_io_segs; i++) + update_chacha20_poly1305_direct(job->enc_keys, + ctx, + job->sgl_io_segs[i].out, + job->sgl_io_segs[i].in, + job->sgl_io_segs[i].len, + job->cipher_direction, + arch, + 0, ifma); + + finalize_chacha20_poly1305_direct(ctx, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes, + arch, 0, ifma); + /* Clear sensitive data from the context */ +#ifdef SAFE_DATA + clear_mem(ctx->last_ks, sizeof(ctx->last_ks)); + clear_mem(ctx->poly_key, sizeof(ctx->poly_key)); +#endif + job->status = IMB_STATUS_COMPLETED; } return job; @@ -490,7 +676,7 @@ IMB_JOB *aead_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, /* Finalize AEAD Poly1305 (final reduction and +S) */ poly1305_aead_complete(hash, ks, job->auth_tag_output, arch, ifma); - job->status |= IMB_STATUS_COMPLETED; + job->status = IMB_STATUS_COMPLETED; return job; } @@ -555,57 +741,6 @@ IMB_JOB *aead_chacha20_poly1305_sgl_avx512(IMB_MGR *mgr, IMB_JOB *job) return aead_chacha20_poly1305_sgl(job, IMB_ARCH_AVX512, 0); } -__forceinline -void init_chacha20_poly1305_direct(const void *key, - struct chacha20_poly1305_context_data *ctx, - const void *iv, const void *aad, - const uint64_t aad_len, const IMB_ARCH arch, - const unsigned ifma) -{ -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (key == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_KEY); - return; - } - if (ctx == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_CTX); - return; - } - if (iv == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_IV); - return; - } - if (aad == NULL && aad_len != 0) { - imb_set_errno(NULL, IMB_ERR_NULL_AAD); - return; - } -#endif - ctx->hash[0] = 0; - ctx->hash[1] = 0; - ctx->hash[2] = 0; - ctx->aad_len = aad_len; - ctx->hash_len = 0; - ctx->last_block_count = 0; - ctx->remain_ks_bytes = 0; - ctx->remain_ct_bytes = 0; - - /* Store IV */ - memcpy_asm(ctx->IV, iv, 12, arch); - - /* Generate Poly key */ - if (arch == IMB_ARCH_SSE) - poly1305_key_gen_sse(key, iv, ctx->poly_key); - else - poly1305_key_gen_avx(key, iv, ctx->poly_key); - - /* Calculate hash over AAD */ - poly1305_aead_update(aad, aad_len, ctx->hash, ctx->poly_key, - arch, ifma); -} - IMB_DLL_LOCAL void init_chacha20_poly1305_sse(const void *key, struct chacha20_poly1305_context_data *ctx, @@ -613,7 +748,7 @@ void init_chacha20_poly1305_sse(const void *key, const uint64_t aad_len) { init_chacha20_poly1305_direct(key, ctx, iv, aad, - aad_len, IMB_ARCH_SSE, 0); + aad_len, IMB_ARCH_SSE, 1, 0); } IMB_DLL_LOCAL @@ -623,7 +758,7 @@ void init_chacha20_poly1305_avx(const void *key, const uint64_t aad_len) { init_chacha20_poly1305_direct(key, ctx, iv, aad, - aad_len, IMB_ARCH_AVX, 0); + aad_len, IMB_ARCH_AVX, 1, 0); } IMB_DLL_LOCAL @@ -633,7 +768,7 @@ void init_chacha20_poly1305_avx512(const void *key, const uint64_t aad_len) { init_chacha20_poly1305_direct(key, ctx, iv, aad, - aad_len, IMB_ARCH_AVX512, 0); + aad_len, IMB_ARCH_AVX512, 1, 0); } IMB_DLL_LOCAL @@ -643,121 +778,7 @@ void init_chacha20_poly1305_fma_avx512(const void *key, const uint64_t aad_len) { init_chacha20_poly1305_direct(key, ctx, iv, aad, - aad_len, IMB_ARCH_AVX512, 1); -} - -__forceinline -void update_chacha20_poly1305_direct(const void *key, - struct chacha20_poly1305_context_data *ctx, - void *dst, const void *src, - const uint64_t len, - const IMB_CIPHER_DIRECTION dir, - const IMB_ARCH arch, - const unsigned ifma) -{ -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (key == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_KEY); - return; - } - if (ctx == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_CTX); - return; - } - if (src == NULL && len != 0) { - imb_set_errno(NULL, IMB_ERR_NULL_SRC); - return; - } - if (dst == NULL && len != 0) { - imb_set_errno(NULL, IMB_ERR_NULL_DST); - return; - } -#endif - uint64_t bytes_to_copy = 0; - uint64_t remain_bytes_to_fill = (16 - ctx->remain_ct_bytes); - uint64_t remain_ct_bytes; - const uint8_t *remain_ct_ptr; - const uint8_t *src8 = (const uint8_t *) src; - uint8_t *dst8 = (uint8_t *) dst; - uint64_t length = len; - - /* Need to copy more bytes into scratchpad */ - if ((ctx->remain_ct_bytes > 0) && (remain_bytes_to_fill > 0)) { - if (len < remain_bytes_to_fill) - bytes_to_copy = length; - else - bytes_to_copy = remain_bytes_to_fill; - } - - /* Increment total hash length */ - ctx->hash_len += length; - - if (dir == IMB_DIR_ENCRYPT) { - chacha20_enc_dec_ks(src, dst, length, key, ctx, arch); - - /* Copy more bytes on Poly scratchpad */ - memcpy_asm(ctx->poly_scratch + ctx->remain_ct_bytes, - dst, bytes_to_copy, arch); - ctx->remain_ct_bytes += bytes_to_copy; - - /* - * Compute hash on remaining bytes of previous segment and - * first bytes of this segment (if there are 16 bytes) - */ - if (ctx->remain_ct_bytes == 16) { - poly1305_aead_update(ctx->poly_scratch, 16, ctx->hash, - ctx->poly_key, arch, ifma); - ctx->remain_ct_bytes = 0; - } - - length -= bytes_to_copy; - remain_ct_bytes = length & HASH_REMAIN_CLAMP; - length &= length & HASH_LEN_CLAMP; - - /* compute hash after cipher on encrypt */ - poly1305_aead_update(dst8 + bytes_to_copy, - length, ctx->hash, ctx->poly_key, arch, - ifma); - - remain_ct_ptr = dst8 + bytes_to_copy + length; - /* copy last bytes of ciphertext (less than 16 bytes) */ - memcpy_asm(ctx->poly_scratch, remain_ct_ptr, remain_ct_bytes, - arch); - ctx->remain_ct_bytes += remain_ct_bytes; - } else { - /* Copy more bytes on Poly scratchpad */ - memcpy_asm(ctx->poly_scratch + ctx->remain_ct_bytes, - src, bytes_to_copy, arch); - ctx->remain_ct_bytes += bytes_to_copy; - - /* - * Compute hash on remaining bytes of previous segment and - * first bytes of this segment (if there are 16 bytes) - */ - if (ctx->remain_ct_bytes == 16) { - poly1305_aead_update(ctx->poly_scratch, 16, ctx->hash, - ctx->poly_key, arch, ifma); - ctx->remain_ct_bytes = 0; - } - - length -= bytes_to_copy; - remain_ct_bytes = length & HASH_REMAIN_CLAMP; - length &= length & HASH_LEN_CLAMP; - - /* compute hash first on decrypt */ - poly1305_aead_update(src8 + bytes_to_copy, length, ctx->hash, - ctx->poly_key, arch, ifma); - - remain_ct_ptr = src8 + bytes_to_copy + length; - /* copy last bytes of ciphertext (less than 16 bytes) */ - memcpy_asm(ctx->poly_scratch, remain_ct_ptr, remain_ct_bytes, - arch); - ctx->remain_ct_bytes += remain_ct_bytes; - chacha20_enc_dec_ks(src, dst, len, key, ctx, arch); - } + aad_len, IMB_ARCH_AVX512, 1, 1); } void update_enc_chacha20_poly1305_sse(const void *key, @@ -766,7 +787,7 @@ void update_enc_chacha20_poly1305_sse(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_ENCRYPT, IMB_ARCH_SSE, 0); + IMB_DIR_ENCRYPT, IMB_ARCH_SSE, 1, 0); } void update_enc_chacha20_poly1305_avx(const void *key, @@ -775,7 +796,7 @@ void update_enc_chacha20_poly1305_avx(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_ENCRYPT, IMB_ARCH_AVX, 0); + IMB_DIR_ENCRYPT, IMB_ARCH_AVX, 1, 0); } void update_enc_chacha20_poly1305_avx2(const void *key, @@ -784,7 +805,7 @@ void update_enc_chacha20_poly1305_avx2(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_ENCRYPT, IMB_ARCH_AVX2, 0); + IMB_DIR_ENCRYPT, IMB_ARCH_AVX2, 1, 0); } @@ -794,7 +815,7 @@ void update_enc_chacha20_poly1305_avx512(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_ENCRYPT, IMB_ARCH_AVX512, 0); + IMB_DIR_ENCRYPT, IMB_ARCH_AVX512, 1, 0); } void update_enc_chacha20_poly1305_fma_avx512(const void *key, @@ -803,7 +824,7 @@ void update_enc_chacha20_poly1305_fma_avx512(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_ENCRYPT, IMB_ARCH_AVX512, 1); + IMB_DIR_ENCRYPT, IMB_ARCH_AVX512, 1, 1); } void update_dec_chacha20_poly1305_sse(const void *key, @@ -812,7 +833,7 @@ void update_dec_chacha20_poly1305_sse(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_DECRYPT, IMB_ARCH_SSE, 0); + IMB_DIR_DECRYPT, IMB_ARCH_SSE, 1, 0); } void update_dec_chacha20_poly1305_avx(const void *key, @@ -821,7 +842,7 @@ void update_dec_chacha20_poly1305_avx(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_DECRYPT, IMB_ARCH_AVX, 0); + IMB_DIR_DECRYPT, IMB_ARCH_AVX, 1, 0); } void update_dec_chacha20_poly1305_avx2(const void *key, @@ -830,7 +851,7 @@ void update_dec_chacha20_poly1305_avx2(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_DECRYPT, IMB_ARCH_AVX2, 0); + IMB_DIR_DECRYPT, IMB_ARCH_AVX2, 1, 0); } void update_dec_chacha20_poly1305_avx512(const void *key, @@ -839,7 +860,7 @@ void update_dec_chacha20_poly1305_avx512(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_DECRYPT, IMB_ARCH_AVX512, 0); + IMB_DIR_DECRYPT, IMB_ARCH_AVX512, 1, 0); } void update_dec_chacha20_poly1305_fma_avx512(const void *key, @@ -848,76 +869,19 @@ void update_dec_chacha20_poly1305_fma_avx512(const void *key, const uint64_t len) { update_chacha20_poly1305_direct(key, ctx, dst, src, len, - IMB_DIR_DECRYPT, IMB_ARCH_AVX512, 1); -} - -__forceinline -void -finalize_chacha20_poly1305_direct(struct chacha20_poly1305_context_data *ctx, - void *tag, const uint64_t tag_len, - const IMB_ARCH arch, const unsigned ifma) -{ -#ifdef SAFE_PARAM - /* reset error status */ - imb_set_errno(NULL, 0); - - if (ctx == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_CTX); - return; - } - if (tag == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_AUTH); - return; - } - if (tag_len == 0 || tag_len > 16) { - imb_set_errno(NULL, IMB_ERR_AUTH_TAG_LEN); - return; - } -#endif - uint64_t last[2]; - uint8_t auth_tag[16]; - - if (ctx->remain_ct_bytes > 0) { - poly1305_aead_update(ctx->poly_scratch, - ctx->remain_ct_bytes, - ctx->hash, - ctx->poly_key, - arch, ifma); - ctx->remain_ct_bytes = 0; - } - - /* - * Construct extra block with AAD and message lengths for - * authentication - */ - last[0] = ctx->aad_len; - last[1] = ctx->hash_len; - poly1305_aead_update(last, sizeof(last), ctx->hash, ctx->poly_key, - arch, ifma); - - /* Finalize AEAD Poly1305 (final reduction and +S) */ - poly1305_aead_complete(ctx->hash, ctx->poly_key, auth_tag, arch, ifma); - - /* Copy N bytes of tag */ - memcpy_asm((uint8_t *) tag, auth_tag, tag_len, arch); - - /* Clear sensitive data from the context */ -#ifdef SAFE_DATA - clear_mem(ctx->last_ks, sizeof(ctx->last_ks)); - clear_mem(ctx->poly_key, sizeof(ctx->poly_key)); -#endif + IMB_DIR_DECRYPT, IMB_ARCH_AVX512, 1, 1); } void finalize_chacha20_poly1305_sse(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len) { - finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_SSE, 0); + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_SSE, 1, 0); } void finalize_chacha20_poly1305_avx(struct chacha20_poly1305_context_data *ctx, void *tag, const uint64_t tag_len) { - finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_AVX, 0); + finalize_chacha20_poly1305_direct(ctx, tag, tag_len, IMB_ARCH_AVX, 1, 0); } void finalize_chacha20_poly1305_avx512( @@ -925,7 +889,7 @@ void finalize_chacha20_poly1305_avx512( void *tag, const uint64_t tag_len) { finalize_chacha20_poly1305_direct(ctx, tag, tag_len, - IMB_ARCH_AVX512, 0); + IMB_ARCH_AVX512, 1, 0); } void finalize_chacha20_poly1305_fma_avx512( @@ -933,5 +897,5 @@ void finalize_chacha20_poly1305_fma_avx512( void *tag, const uint64_t tag_len) { finalize_chacha20_poly1305_direct(ctx, tag, tag_len, - IMB_ARCH_AVX512, 1); + IMB_ARCH_AVX512, 1, 1); } diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 59477275..2d29a4c4 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -104,6 +104,7 @@ typedef cpuset_t cpu_set_t; #define BITS(x) (sizeof(x) * 8) #define DIM(x) (sizeof(x)/sizeof(x[0])) +#define DIV_ROUND_UP(x, y) ((x + y - 1) / y) #define MAX_NUM_THREADS 16 /* Maximum number of threads that can be created */ @@ -911,6 +912,7 @@ uint32_t *hash_size_list = NULL; uint64_t *xgem_hdr_list = NULL; uint16_t imix_list_count = 0; uint32_t average_job_size = 0; +uint32_t max_job_size = 0; /* Size of IMIX list (needs to be multiple of 2) */ #define JOB_SIZE_IMIX_LIST 1024 @@ -1579,17 +1581,27 @@ set_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, } static inline void -set_sgl_job_fields(IMB_JOB *job, const uint8_t *src, uint8_t *dst, - const uint32_t *keys, const uint8_t *aad, - const uint64_t len, - IMB_SGL_STATE state, - struct gcm_context_data *gcm_ctx, +set_sgl_job_fields(IMB_JOB *job, uint8_t *p_buffer, imb_uint128_t *p_keys, + const uint32_t size_idx, const uint32_t buf_index, + struct IMB_SGL_IOV *sgl, struct gcm_context_data *gcm_ctx, struct chacha20_poly1305_context_data *cp_ctx) { + uint8_t *src = get_src_buffer(buf_index, p_buffer); + uint8_t *dst = get_dst_buffer(buf_index, p_buffer); + uint8_t *aad = src; + uint32_t buf_size; + job->src = src; job->dst = dst; - job->msg_len_to_cipher_in_bytes = len; - job->msg_len_to_hash_in_bytes = len; + + /* If IMIX testing is being done, set the buffer size to cipher and hash + * going through the list of sizes precalculated */ + if (imix_list_count != 0) { + uint32_t list_idx = size_idx & (JOB_SIZE_IMIX_LIST - 1); + + job->msg_len_to_cipher_in_bytes = cipher_size_list[list_idx]; + } + buf_size = (uint32_t) job->msg_len_to_cipher_in_bytes; if (job->cipher_mode == IMB_CIPHER_GCM_SGL) { job->u.GCM.aad = aad; job->u.GCM.ctx = gcm_ctx; @@ -1597,11 +1609,30 @@ set_sgl_job_fields(IMB_JOB *job, const uint8_t *src, uint8_t *dst, job->u.CHACHA20_POLY1305.aad = aad; job->u.CHACHA20_POLY1305.ctx = cp_ctx; } + job->enc_keys = job->dec_keys = + (const uint32_t *) get_key_pointer(buf_index, + p_keys); + job->sgl_state = IMB_SGL_ALL; - job->enc_keys = job->dec_keys = keys; - job->sgl_state = state; -}; + const uint32_t num_segs = buf_size / segment_size; + const uint32_t final_seg_sz = buf_size % segment_size; + unsigned i; + + job->num_sgl_io_segs = num_segs; + for (i = 0; i < num_segs; i++) { + sgl[i].in = &src[i * segment_size]; + sgl[i].out = &dst[i * segment_size]; + sgl[i].len = segment_size; + } + if (final_seg_sz != 0) { + sgl[i].in = &src[num_segs * segment_size]; + sgl[i].out = &dst[num_segs * segment_size]; + sgl[i].len = final_seg_sz; + (job->num_sgl_io_segs)++; + } + job->sgl_io_segs = sgl; +}; static void set_size_lists(uint32_t *cipher_size_list, uint32_t *hash_size_list, @@ -1715,6 +1746,10 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, uint8_t gcm_key[32]; uint8_t next_iv[IMB_AES_BLOCK_SIZE]; IMB_JOB jobs[MAX_BURST_SIZE]; + struct gcm_context_data gcm_ctx[MAX_BURST_SIZE]; + struct chacha20_poly1305_context_data cp_ctx[MAX_BURST_SIZE]; + struct IMB_SGL_IOV *sgl[MAX_BURST_SIZE] = {NULL}; + uint32_t max_num_segs = 1; memset(&job_template, 0, sizeof(IMB_JOB)); @@ -1722,6 +1757,19 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, and set the XGEM header in case PON is used. */ set_size_lists(cipher_size_list, hash_size_list, xgem_hdr_list, params); + if (segment_size != 0) + max_num_segs = DIV_ROUND_UP(job_sizes[RANGE_MAX], + segment_size); + + for (i = 0; i < MAX_BURST_SIZE; i++) { + sgl[i] = malloc(sizeof(struct IMB_SGL_IOV) * + max_num_segs); + if (sgl[i] == NULL) { + fprintf(stderr, "malloc() failed\n"); + goto exit; + } + } + /* * If single size is used, set the cipher and hash lengths in the * job template, so they don't have to be set in every job @@ -2026,7 +2074,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, it_next.it_value.tv_usec = (TIMEOUT_MS % 1000) * 1000; if (setitimer(ITIMER_REAL, &it_next, NULL)) { perror("setitimer(one-shot)"); - exit(EXIT_FAILURE); + goto exit; } #else /* _WIN32 */ /* create the timer queue */ @@ -2034,7 +2082,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, if (NULL == hTimeboxQueue) { fprintf(stderr, "CreateTimerQueue() error %u\n", (unsigned) GetLastError()); - exit(EXIT_FAILURE); + goto exit; } /* set a timer to call the timebox */ @@ -2044,7 +2092,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, NULL, TIMEOUT_MS, 0, 0)) { fprintf(stderr, "CreateTimerQueueTimer() error %u\n", (unsigned) GetLastError()); - exit(EXIT_FAILURE); + goto exit; } #endif timebox_on = 1; @@ -2070,9 +2118,18 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, job = &jobs[i]; *job = job_template; - set_job_fields(job, p_buffer, p_keys, i, index); + if (segment_size != 0) + set_sgl_job_fields(job, p_buffer, + p_keys, i, + index, sgl[i], + &gcm_ctx[i], + &cp_ctx[i]); + else + set_job_fields(job, p_buffer, p_keys, + i, index); index = get_next_index(index); + } /* submit burst */ #ifdef DEBUG @@ -2221,75 +2278,15 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } else { /* test job api */ for (i = 0; (i < num_iter) && timebox_on; i++) { - job = IMB_GET_NEXT_JOB(mb_mgr); *job = job_template; - if (segment_size != 0) { - /* SGL */ - struct gcm_context_data gcm_ctx; - struct chacha20_poly1305_context_data cp_ctx; - const uint32_t *keys = - (const uint32_t *) get_key_pointer(index, - p_keys); - - uint8_t *src = get_src_buffer(index, p_buffer); - uint8_t *dst = get_dst_buffer(index, p_buffer); - uint8_t *aad = src; - - set_sgl_job_fields(job, NULL, NULL, keys, aad, 0, - IMB_SGL_INIT, &gcm_ctx, &cp_ctx); -#ifdef DEBUG - job = IMB_SUBMIT_JOB(mb_mgr); -#else - job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); -#endif - - const uint32_t num_segs = cipher_size_list[0] / - segment_size; - const uint32_t final_seg_sz = cipher_size_list[0] % - segment_size; - uint32_t j; - - for (j = 0; j < num_segs; j++) { - job = IMB_GET_NEXT_JOB(mb_mgr); - *job = job_template; - - set_sgl_job_fields(job, &src[j*segment_size], - &dst[j*segment_size], keys, - aad, segment_size, - IMB_SGL_UPDATE, - &gcm_ctx, &cp_ctx); -#ifdef DEBUG - job = IMB_SUBMIT_JOB(mb_mgr); -#else - job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); -#endif - } - if (final_seg_sz != 0) { - job = IMB_GET_NEXT_JOB(mb_mgr); - *job = job_template; - - set_sgl_job_fields(job, &src[j*segment_size], - &dst[j*segment_size], keys, - aad, final_seg_sz, - IMB_SGL_UPDATE, - &gcm_ctx, &cp_ctx); -#ifdef DEBUG - job = IMB_SUBMIT_JOB(mb_mgr); -#else - job = IMB_SUBMIT_JOB_NOCHECK(mb_mgr); -#endif - } - job = IMB_GET_NEXT_JOB(mb_mgr); - *job = job_template; - - set_sgl_job_fields(job, NULL, NULL, - keys, aad, - 0, - IMB_SGL_COMPLETE, - &gcm_ctx, &cp_ctx); - } else /* Linear buffer */ + if (segment_size != 0) + set_sgl_job_fields(job, p_buffer, p_keys, + i, index, + sgl[0], &gcm_ctx[0], + &cp_ctx[0]); + else set_job_fields(job, p_buffer, p_keys, i, index); index = get_next_index(index); @@ -2304,7 +2301,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, fprintf(stderr, "failed job, status:%d\n", job->status); - return 1; + goto exit; } #endif job = IMB_GET_COMPLETED_JOB(mb_mgr); @@ -2321,7 +2318,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, "failed job, status:%d, " "error code:%d, %s\n", job->status, errc, imb_get_strerror(errc)); - return 1; + goto exit; } #else (void)job; @@ -2330,6 +2327,11 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } /* if test_api */ + for (i = 0; i < MAX_BURST_SIZE; i++) { + free(sgl[i]); + sgl[i] = NULL; + } + #ifndef _WIN32 if (use_unhalted_cycles) time = (read_cycles(params->core) - rd_cycles_cost) - time; @@ -2346,7 +2348,7 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, if (setitimer(ITIMER_REAL, &it_disarm, NULL)) { perror("setitimer(disarm)"); - exit(EXIT_FAILURE); + goto exit; } #else /* _WIN32 */ /* delete all timeboxes in the timer queue */ @@ -2366,6 +2368,12 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, return time; return time / num_iter; + +exit: + for (i = 0; i < MAX_BURST_SIZE; i++) + free(sgl[i]); + + exit(EXIT_FAILURE); } static void @@ -2405,7 +2413,8 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, &pb[j*segment_size], &pb[j*segment_size], final_seg_sz); - finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); + finalize(gdata_key, gdata_ctx, auth_tag, + sizeof(auth_tag)); index = get_next_index(index); } @@ -2418,7 +2427,8 @@ run_gcm_sgl(aes_gcm_init_t init, aes_gcm_enc_dec_update_t update, init(gdata_key, gdata_ctx, iv, aad, aad_size); update(gdata_key, gdata_ctx, pb, pb, buf_size); - finalize(gdata_key, gdata_ctx, auth_tag, sizeof(auth_tag)); + finalize(gdata_key, gdata_ctx, auth_tag, + sizeof(auth_tag)); index = get_next_index(index); } @@ -2728,11 +2738,11 @@ do_test_chacha_poly(struct params_s *params, } #ifndef _WIN32 if (use_unhalted_cycles) - time = (read_cycles(params->core) - - rd_cycles_cost) - time; + time = (read_cycles(params->core) - + rd_cycles_cost) - time; else #endif - time = __rdtscp(&aux) - time; + time = __rdtscp(&aux) - time; free(aad); @@ -2893,7 +2903,8 @@ process_variant(IMB_MGR *mgr, const enum arch_type_e arch, else *times = do_test_gcm(params, job_iter, mgr, p_buffer, p_keys); - } else if (params->cipher_mode == TEST_AEAD_CHACHA20 && (!use_job_api)) { + } else if (params->cipher_mode == TEST_AEAD_CHACHA20 && + (!use_job_api)) { if (job_iter == 0) *times = do_test_chacha_poly(params, 2 * num_iter, mgr, -- GitLab From 969213a1ff07a8e4248ecba7c830f44254cc3e52 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 5 Jul 2022 12:06:50 +0000 Subject: [PATCH 241/369] test: add single job SGL API tests --- test/chacha20_poly1305_test.c | 207 ++++++++++++++++++++++++++++++++- test/gcm_test.c | 212 +++++++++++++++++++++++++++++++++- test/utils.h | 1 + 3 files changed, 414 insertions(+), 6 deletions(-) diff --git a/test/chacha20_poly1305_test.c b/test/chacha20_poly1305_test.c index 09da80ac..88689196 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/chacha20_poly1305_test.c @@ -633,6 +633,202 @@ test_aead_vectors(struct IMB_MGR *mb_mgr, } +static void +test_single_job_sgl(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const uint32_t buffer_sz, + const uint32_t seg_sz, + const IMB_CIPHER_DIRECTION cipher_dir) +{ + struct IMB_JOB *job; + uint8_t *in_buffer = NULL; + uint8_t **segments = NULL; + uint8_t linear_digest[DIGEST_SZ]; + uint8_t sgl_digest[DIGEST_SZ]; + uint8_t key[KEY_SZ]; + unsigned i; + uint8_t aad[AAD_SZ]; + uint8_t iv[IV_SZ]; + struct chacha20_poly1305_context_data chacha_ctx; + uint32_t last_seg_sz = buffer_sz % seg_sz; + struct IMB_SGL_IOV *sgl_segs = NULL; + const uint32_t num_segments = DIV_ROUND_UP(buffer_sz, seg_sz); + + sgl_segs = malloc(sizeof(struct IMB_SGL_IOV) * num_segments); + + if (last_seg_sz == 0) + last_seg_sz = seg_sz; + + in_buffer = malloc(buffer_sz); + if (in_buffer == NULL) { + fprintf(stderr, "Could not allocate memory for input buffer\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } + + /* + * Initialize tags with different values, to make sure the comparison + * is false if they are not updated by the library + */ + memset(sgl_digest, 0, DIGEST_SZ); + memset(linear_digest, 0xFF, DIGEST_SZ); + + generate_random_buf(in_buffer, buffer_sz); + generate_random_buf(key, KEY_SZ); + generate_random_buf(iv, IV_SZ); + generate_random_buf(aad, AAD_SZ); + + segments = malloc(num_segments * sizeof(*segments)); + if (segments == NULL) { + fprintf(stderr, + "Could not allocate memory for segments array\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } + memset(segments, 0, num_segments * sizeof(*segments)); + + for (i = 0; i < (num_segments - 1); i++) { + segments[i] = malloc(seg_sz); + if (segments[i] == NULL) { + fprintf(stderr, + "Could not allocate memory for segment %u\n", + i); + test_suite_update(ctx, 0, 1); + goto exit; + } + memcpy(segments[i], in_buffer + seg_sz * i, seg_sz); + sgl_segs[i].in = segments[i]; + sgl_segs[i].out = segments[i]; + sgl_segs[i].len = seg_sz; + } + segments[i] = malloc(last_seg_sz); + if (segments[i] == NULL) { + fprintf(stderr, "Could not allocate memory for segment %u\n", + i); + test_suite_update(ctx, 0, 1); + goto exit; + } + memcpy(segments[i], in_buffer + seg_sz * i, last_seg_sz); + sgl_segs[i].in = segments[i]; + sgl_segs[i].out = segments[i]; + sgl_segs[i].len = last_seg_sz; + + /* Process linear (single segment) buffer */ + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = cipher_dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; + job->hash_alg = IMB_AUTH_CHACHA20_POLY1305; + job->enc_keys = key; + job->dec_keys = key; + job->src = in_buffer; + job->dst = in_buffer; + job->key_len_in_bytes = KEY_SZ; + + job->u.CHACHA20_POLY1305.aad = aad; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = AAD_SZ; + + job->iv = iv; + job->iv_len_in_bytes = IV_SZ; + job->msg_len_to_cipher_in_bytes = buffer_sz; + job->cipher_start_src_offset_in_bytes = 0; + + job->msg_len_to_hash_in_bytes = buffer_sz; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = linear_digest; + job->auth_tag_output_len_in_bytes = DIGEST_SZ; + + job = IMB_SUBMIT_JOB(mb_mgr); + + if (job->status == IMB_STATUS_COMPLETED) + test_suite_update(ctx, 1, 0); + else { + fprintf(stderr, "job status returned as not successful" + " for the linear buffer\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } + + /* Process multi-segment buffer */ + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = cipher_dir; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305_SGL; + job->hash_alg = IMB_AUTH_CHACHA20_POLY1305_SGL; + job->enc_keys = key; + job->dec_keys = key; + job->key_len_in_bytes = KEY_SZ; + + job->u.CHACHA20_POLY1305.aad = aad; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = AAD_SZ; + job->u.CHACHA20_POLY1305.ctx = &chacha_ctx; + + job->iv = iv; + job->iv_len_in_bytes = IV_SZ; + job->cipher_start_src_offset_in_bytes = 0; + + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output = sgl_digest; + job->auth_tag_output_len_in_bytes = DIGEST_SZ; + + job->num_sgl_io_segs = num_segments; + job->sgl_state = IMB_SGL_ALL; + job->sgl_io_segs = sgl_segs; + job = IMB_SUBMIT_JOB(mb_mgr); + + if (job->status == IMB_STATUS_COMPLETED) { + for (i = 0; i < (num_segments - 1); i++) { + if (memcmp(in_buffer + i*seg_sz, segments[i], + seg_sz) != 0) { + printf("ciphertext mismatched " + "in segment number %u " + "(segment size = %u)\n", + i, seg_sz); + hexdump(stderr, "Linear output", + in_buffer + i*seg_sz, seg_sz); + hexdump(stderr, "SGL output", segments[i], + seg_sz); + test_suite_update(ctx, 0, 1); + goto exit; + } + } + /* Check last segment */ + if (memcmp(in_buffer + i*seg_sz, segments[i], + last_seg_sz) != 0) { + printf("ciphertext mismatched " + "in segment number %u (segment size = %u)\n", + i, seg_sz); + hexdump(stderr, "Linear output", + in_buffer + i*seg_sz, last_seg_sz); + hexdump(stderr, "SGL output", segments[i], last_seg_sz); + test_suite_update(ctx, 0, 1); + } + if (memcmp(sgl_digest, linear_digest, 16) != 0) { + printf("hash mismatched (segment size = %u)\n", + seg_sz); + hexdump(stderr, "Linear digest", + linear_digest, DIGEST_SZ); + hexdump(stderr, "SGL digest", sgl_digest, DIGEST_SZ); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + } else { + fprintf(stderr, "job status returned as not successful" + " for the segmented buffer\n"); + test_suite_update(ctx, 0, 1); + } + +exit: + free(sgl_segs); + free(in_buffer); + if (segments != NULL) { + for (i = 0; i < num_segments; i++) + free(segments[i]); + free(segments); + } +} + static void test_sgl(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, @@ -679,16 +875,16 @@ test_sgl(struct IMB_MGR *mb_mgr, generate_random_buf(iv, IV_SZ); generate_random_buf(aad, AAD_SZ); - segments = malloc(num_segments * 8); + segments = malloc(num_segments * sizeof(*segments)); if (segments == NULL) { fprintf(stderr, "Could not allocate memory for segments array\n"); test_suite_update(ctx, 0, 1); goto exit; } - memset(segments, 0, num_segments * 8); + memset(segments, 0, num_segments * sizeof(*segments)); - segment_sizes = malloc(num_segments * 4); + segment_sizes = malloc(num_segments * sizeof(*segment_sizes)); if (segment_sizes == NULL) { fprintf(stderr, "Could not allocate memory for array of sizes\n"); @@ -968,6 +1164,11 @@ chacha20_poly1305_test(struct IMB_MGR *mb_mgr) test_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, IMB_DIR_DECRYPT, 1, 0); test_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, IMB_DIR_ENCRYPT, 1, 1); test_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, IMB_DIR_DECRYPT, 1, 1); + /* Single job SGL API */ + test_single_job_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, + IMB_DIR_ENCRYPT); + test_single_job_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, + IMB_DIR_DECRYPT); /* Direct API */ test_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, IMB_DIR_ENCRYPT, 0, 1); test_sgl(mb_mgr, &ctx, BUF_SZ, seg_sz, IMB_DIR_DECRYPT, 0, 1); diff --git a/test/gcm_test.c b/test/gcm_test.c index 071624ae..93094027 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -1908,6 +1908,56 @@ job_sgl_aes_gcm(IMB_MGR *p_mgr, return 0; } +static int +aes_gcm_single_job_sgl(IMB_MGR *mb_mgr, + IMB_CIPHER_DIRECTION cipher_dir, + const struct gcm_key_data *key, + const uint64_t key_len, + struct IMB_SGL_IOV *sgl_segs, + const unsigned num_sgl_segs, + const uint8_t *iv, const uint64_t iv_len, + const uint8_t *aad, const uint64_t aad_len, + uint8_t *auth_tag, const uint64_t auth_tag_len, + struct gcm_context_data *ctx) +{ + IMB_JOB *job; + + job = IMB_GET_NEXT_JOB(mb_mgr); + if (!job) { + fprintf(stderr, "failed to get job\n"); + return -1; + } + + job->cipher_mode = IMB_CIPHER_GCM_SGL; + job->cipher_direction = cipher_dir; + job->hash_alg = IMB_AUTH_GCM_SGL; + job->chain_order = + (cipher_dir == IMB_DIR_ENCRYPT) ? IMB_ORDER_CIPHER_HASH : + IMB_ORDER_HASH_CIPHER; + job->enc_keys = key; + job->dec_keys = key; + job->key_len_in_bytes = key_len; + job->num_sgl_io_segs = num_sgl_segs; + job->sgl_io_segs = sgl_segs; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = iv; + job->iv_len_in_bytes = iv_len; + job->u.GCM.aad = aad; + job->u.GCM.aad_len_in_bytes = aad_len; + job->auth_tag_output = auth_tag; + job->auth_tag_output_len_in_bytes = auth_tag_len; + job->u.GCM.ctx = ctx; + job->sgl_state = IMB_SGL_ALL; + job = IMB_SUBMIT_JOB(mb_mgr); + + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "failed job, status:%d\n", job->status); + return -1; + } + + return 0; +} + static int job_sgl_aes_gcm_enc(IMB_MGR *p_mgr, const struct gcm_key_data *key, @@ -2620,6 +2670,157 @@ test_gmac(struct test_suite_context *ts128, } } +static void +test_single_job_sgl(struct IMB_MGR *mb_mgr, + struct test_suite_context *ctx, + const uint32_t key_sz, + const uint32_t buffer_sz, + const uint32_t seg_sz, + const IMB_CIPHER_DIRECTION cipher_dir) +{ + uint8_t *in_buffer = NULL; + uint8_t **segments = NULL; + uint8_t linear_digest[DIGEST_SZ]; + uint8_t sgl_digest[DIGEST_SZ]; + uint8_t k[MAX_KEY_SZ]; + unsigned int i; + uint8_t aad[AAD_SZ]; + uint8_t iv[IV_SZ]; + struct gcm_context_data gcm_ctx; + struct gcm_key_data key; + uint32_t last_seg_sz = buffer_sz % seg_sz; + struct IMB_SGL_IOV *sgl_segs = NULL; + const uint32_t num_segments = DIV_ROUND_UP(buffer_sz, seg_sz); + + if (last_seg_sz == 0) + last_seg_sz = seg_sz; + + sgl_segs = malloc(sizeof(struct IMB_SGL_IOV) * num_segments); + + in_buffer = malloc(buffer_sz); + if (in_buffer == NULL) { + fprintf(stderr, "Could not allocate memory for input buffer\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } + + /* + * Initialize tags with different values, to make sure the comparison + * is false if they are not updated by the library + */ + memset(sgl_digest, 0, DIGEST_SZ); + memset(linear_digest, 0xFF, DIGEST_SZ); + + generate_random_buf(in_buffer, buffer_sz); + generate_random_buf(k, key_sz); + generate_random_buf(iv, IV_SZ); + generate_random_buf(aad, AAD_SZ); + + if (key_sz == IMB_KEY_128_BYTES) + IMB_AES128_GCM_PRE(mb_mgr, k, &key); + else if (key_sz == IMB_KEY_192_BYTES) + IMB_AES192_GCM_PRE(mb_mgr, k, &key); + else /* key_sz == 32 */ + IMB_AES256_GCM_PRE(mb_mgr, k, &key); + + segments = malloc(num_segments * sizeof(*segments)); + if (segments == NULL) { + fprintf(stderr, + "Could not allocate memory for segments array\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } + memset(segments, 0, num_segments * sizeof(*segments)); + + for (i = 0; i < (num_segments - 1); i++) { + segments[i] = malloc(seg_sz); + if (segments[i] == NULL) { + fprintf(stderr, + "Could not allocate memory for segment %u\n", + i); + test_suite_update(ctx, 0, 1); + goto exit; + } + memcpy(segments[i], in_buffer + seg_sz * i, seg_sz); + sgl_segs[i].in = segments[i]; + sgl_segs[i].out = segments[i]; + sgl_segs[i].len = seg_sz; + } + segments[i] = malloc(last_seg_sz); + if (segments[i] == NULL) { + fprintf(stderr, "Could not allocate memory for segment %u\n", + i); + test_suite_update(ctx, 0, 1); + goto exit; + } + memcpy(segments[i], in_buffer + seg_sz * i, last_seg_sz); + sgl_segs[i].in = segments[i]; + sgl_segs[i].out = segments[i]; + sgl_segs[i].len = last_seg_sz; + + /* Process linear (single segment) buffer */ + if (aes_gcm_job(mb_mgr, cipher_dir, &key, key_sz, + in_buffer, in_buffer, buffer_sz, iv, IV_SZ, aad, AAD_SZ, + linear_digest, DIGEST_SZ, + &gcm_ctx, IMB_CIPHER_GCM, 0) < 0) { + test_suite_update(ctx, 0, 1); + goto exit; + } else + test_suite_update(ctx, 1, 0); + + /* Process multi-segment buffer */ + aes_gcm_single_job_sgl(mb_mgr, cipher_dir, &key, key_sz, + sgl_segs, num_segments, + iv, IV_SZ, aad, AAD_SZ, + sgl_digest, DIGEST_SZ, &gcm_ctx); + + for (i = 0; i < (num_segments - 1); i++) { + if (memcmp(in_buffer + i*seg_sz, segments[i], + seg_sz) != 0) { + printf("ciphertext mismatched " + "in segment number %u " + "(segment size = %u)\n", + i, seg_sz); + hexdump(stderr, "Expected output", + in_buffer + i*seg_sz, seg_sz); + hexdump(stderr, "SGL output", segments[i], + seg_sz); + test_suite_update(ctx, 0, 1); + goto exit; + } + } + /* Check last segment */ + if (memcmp(in_buffer + i*seg_sz, segments[i], + last_seg_sz) != 0) { + printf("ciphertext mismatched " + "in segment number %u (segment size = %u)\n", + i, seg_sz); + hexdump(stderr, "Expected output", + in_buffer + i*seg_sz, last_seg_sz); + hexdump(stderr, "SGL output", segments[i], last_seg_sz); + test_suite_update(ctx, 0, 1); + } + if (memcmp(sgl_digest, linear_digest, 16) != 0) { + printf("hash mismatched (segment size = %u)\n", + seg_sz); + hexdump(stderr, "Expected digest", + linear_digest, DIGEST_SZ); + hexdump(stderr, "SGL digest", sgl_digest, DIGEST_SZ); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + +exit: + free(sgl_segs); + free(in_buffer); + if (segments != NULL) { + for (i = 0; i < num_segments; i++) + free(segments[i]); + free(segments); + } +} + static void test_sgl(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, @@ -2673,16 +2874,16 @@ test_sgl(struct IMB_MGR *mb_mgr, else /* key_sz == 32 */ IMB_AES256_GCM_PRE(mb_mgr, k, &key); - segments = malloc(num_segments * 8); + segments = malloc(num_segments * sizeof(*segments)); if (segments == NULL) { fprintf(stderr, "Could not allocate memory for segments array\n"); test_suite_update(ctx, 0, 1); goto exit; } - memset(segments, 0, num_segments * 8); + memset(segments, 0, num_segments * sizeof(*segments)); - segment_sizes = malloc(num_segments * 4); + segment_sizes = malloc(num_segments * sizeof(*segment_sizes)); if (segment_sizes == NULL) { fprintf(stderr, "Could not allocate memory for array of sizes\n"); @@ -2901,6 +3102,11 @@ int gcm_test(IMB_MGR *p_mgr) IMB_DIR_ENCRYPT, 1); test_sgl(p_mgr, ctx, key_sz, buf_sz, seg_sz, IMB_DIR_DECRYPT, 1); + /* Single job SGL API */ + test_single_job_sgl(p_mgr, ctx, key_sz, buf_sz, seg_sz, + IMB_DIR_ENCRYPT); + test_single_job_sgl(p_mgr, ctx, key_sz, buf_sz, seg_sz, + IMB_DIR_DECRYPT); /* Direct API */ test_sgl(p_mgr, ctx, key_sz, buf_sz, seg_sz, IMB_DIR_ENCRYPT, 0); diff --git a/test/utils.h b/test/utils.h index 2be1cd41..84a89349 100644 --- a/test/utils.h +++ b/test/utils.h @@ -32,6 +32,7 @@ #include #define DIM(_x) (sizeof(_x)/sizeof(_x[0])) +#define DIV_ROUND_UP(x, y) ((x + y - 1) / y) void hexdump(FILE *fp, const char *msg, const void *p, size_t len); void hexdump_ex(FILE *fp, const char *msg, const void *p, size_t len, -- GitLab From 508c11ec204713cf8fc3b892eae797a25b70f05e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 14 Jul 2022 13:15:46 +0000 Subject: [PATCH 242/369] lib: use -fPIC also for static library build q Co-authored-by: Dan Zimmerman --- lib/Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 86a5409b..ccfc3fd4 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -90,7 +90,7 @@ CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ib $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) CET_LDFLAGS=-r -z ibt -z shstk endif -CFLAGS := -DNO_COMPAT_IMB_API_053 $(EXTRA_CFLAGS) $(INCLUDES) \ +CFLAGS := -fPIC -DNO_COMPAT_IMB_API_053 $(EXTRA_CFLAGS) $(INCLUDES) \ -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ -Wcast-qual -Wundef -Wwrite-strings \ -Wformat -Wformat-security \ @@ -210,7 +210,6 @@ endif # so or static build ifeq ($(SHARED),y) -CFLAGS += -fPIC ifneq ($(MINGW),0) LIBNAME = $(LIB).dll else @@ -222,7 +221,6 @@ ifeq ($(CC_HAS_CET),1) LDFLAGS += -fcf-protection=full -Wl,-z,ibt -Wl,-z,shstk -Wl,-z,cet-report=error endif else -CFLAGS += -fPIE LIBNAME = $(LIB).a LIBPERM = 0644 LDFLAGS += -g -- GitLab From 4cbfc40654127d447f77936889839191fd4c21a1 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 15 Jul 2022 12:58:02 +0000 Subject: [PATCH 243/369] lib: [no-aesni] use common OOO manager reset API's --- lib/no-aesni/mb_mgr_sse_no_aesni.c | 434 +++-------------------------- 1 file changed, 39 insertions(+), 395 deletions(-) diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 6436968a..0e57b945 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -46,6 +46,8 @@ #include "include/arch_noaesni.h" #include "include/arch_sse_type1.h" +#include "include/ooo_mgr_reset.h" + /* ====================================================================== */ #define SAVE_XMMS save_xmms @@ -383,434 +385,76 @@ submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job) static void reset_ooo_mgrs(IMB_MGR *state) { - unsigned int j; - uint8_t *p; - size_t size; - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_sec_ooo = state->docsis128_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis128_crc32_sec_ooo = - state->docsis128_crc32_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_sec_ooo = state->docsis256_sec_ooo; - MB_MGR_DOCSIS_AES_OOO *docsis256_crc32_sec_ooo = - state->docsis256_crc32_sec_ooo; - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = state->zuc256_eea3_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; - MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; - /* Init AES out-of-order fields */ - memset(aes128_ooo->lens, 0xFF, - sizeof(aes128_ooo->lens)); - memset(&aes128_ooo->lens[0], 0, - sizeof(aes128_ooo->lens[0]) * 4); - memset(aes128_ooo->job_in_lane, 0, - sizeof(aes128_ooo->job_in_lane)); - aes128_ooo->unused_lanes = 0xF3210; - aes128_ooo->num_lanes_inuse = 0; - - - memset(aes192_ooo->lens, 0xFF, - sizeof(aes192_ooo->lens)); - memset(&aes192_ooo->lens[0], 0, - sizeof(aes192_ooo->lens[0]) * 4); - memset(aes192_ooo->job_in_lane, 0, - sizeof(aes192_ooo->job_in_lane)); - aes192_ooo->unused_lanes = 0xF3210; - aes192_ooo->num_lanes_inuse = 0; - - - memset(aes256_ooo->lens, 0xFF, - sizeof(aes256_ooo->lens)); - memset(&aes256_ooo->lens[0], 0, - sizeof(aes256_ooo->lens[0]) * 4); - memset(aes256_ooo->job_in_lane, 0, - sizeof(aes256_ooo->job_in_lane)); - aes256_ooo->unused_lanes = 0xF3210; - aes256_ooo->num_lanes_inuse = 0; - + ooo_mgr_aes_reset(state->aes128_ooo, 4); + ooo_mgr_aes_reset(state->aes192_ooo, 4); + ooo_mgr_aes_reset(state->aes256_ooo, 4); /* DOCSIS SEC BPI uses same settings as AES CBC */ - memset(docsis128_sec_ooo->lens, 0xFF, - sizeof(docsis128_sec_ooo->lens)); - memset(&docsis128_sec_ooo->lens[0], 0, - sizeof(docsis128_sec_ooo->lens[0]) * 4); - memset(docsis128_sec_ooo->job_in_lane, 0, - sizeof(docsis128_sec_ooo->job_in_lane)); - docsis128_sec_ooo->unused_lanes = 0xF3210; - docsis128_sec_ooo->num_lanes_inuse = 0; - - memset(docsis128_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis128_crc32_sec_ooo->lens)); - memset(&docsis128_crc32_sec_ooo->lens[0], 0, - sizeof(docsis128_crc32_sec_ooo->lens[0]) * 4); - memset(docsis128_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis128_crc32_sec_ooo->job_in_lane)); - docsis128_crc32_sec_ooo->unused_lanes = 0xF3210; - docsis128_crc32_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_sec_ooo->lens, 0xFF, - sizeof(docsis256_sec_ooo->lens)); - memset(&docsis256_sec_ooo->lens[0], 0, - sizeof(docsis256_sec_ooo->lens[0]) * 4); - memset(docsis256_sec_ooo->job_in_lane, 0, - sizeof(docsis256_sec_ooo->job_in_lane)); - docsis256_sec_ooo->unused_lanes = 0xF3210; - docsis256_sec_ooo->num_lanes_inuse = 0; - - memset(docsis256_crc32_sec_ooo->lens, 0xFF, - sizeof(docsis256_crc32_sec_ooo->lens)); - memset(&docsis256_crc32_sec_ooo->lens[0], 0, - sizeof(docsis256_crc32_sec_ooo->lens[0]) * 4); - memset(docsis256_crc32_sec_ooo->job_in_lane, 0, - sizeof(docsis256_crc32_sec_ooo->job_in_lane)); - docsis256_crc32_sec_ooo->unused_lanes = 0xF3210; - docsis256_crc32_sec_ooo->num_lanes_inuse = 0; + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 4); /* Init ZUC out-of-order fields */ - memset(zuc_eea3_ooo->lens, 0, - sizeof(zuc_eea3_ooo->lens)); - memset(zuc_eea3_ooo->job_in_lane, 0, - sizeof(zuc_eea3_ooo->job_in_lane)); - zuc_eea3_ooo->unused_lanes = 0xFF03020100; - zuc_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc_eea3_ooo->state, 0, - sizeof(zuc_eea3_ooo->state)); - zuc_eea3_ooo->init_not_done = 0; - zuc_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc_eia3_ooo->lens, 0xFF, - sizeof(zuc_eia3_ooo->lens)); - memset(zuc_eia3_ooo->job_in_lane, 0, - sizeof(zuc_eia3_ooo->job_in_lane)); - zuc_eia3_ooo->unused_lanes = 0xFF03020100; - zuc_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc_eia3_ooo->state, 0, - sizeof(zuc_eia3_ooo->state)); - zuc_eia3_ooo->init_not_done = 0; - zuc_eia3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eea3_ooo->lens, 0, - sizeof(zuc256_eea3_ooo->lens)); - memset(zuc256_eea3_ooo->job_in_lane, 0, - sizeof(zuc256_eea3_ooo->job_in_lane)); - zuc256_eea3_ooo->unused_lanes = 0xFF03020100; - zuc256_eea3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eea3_ooo->state, 0, - sizeof(zuc256_eea3_ooo->state)); - zuc256_eea3_ooo->init_not_done = 0; - zuc256_eea3_ooo->unused_lane_bitmask = 0x0f; - - memset(zuc256_eia3_ooo->lens, 0xFF, - sizeof(zuc256_eia3_ooo->lens)); - memset(zuc256_eia3_ooo->job_in_lane, 0, - sizeof(zuc256_eia3_ooo->job_in_lane)); - zuc256_eia3_ooo->unused_lanes = 0xFF03020100; - zuc256_eia3_ooo->num_lanes_inuse = 0; - memset(&zuc256_eia3_ooo->state, 0, - sizeof(zuc256_eia3_ooo->state)); - zuc256_eia3_ooo->init_not_done = 0; - zuc256_eia3_ooo->unused_lane_bitmask = 0x0f; + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); /* Init HMAC/SHA1 out-of-order fields */ - hmac_sha_1_ooo->lens[0] = 0; - hmac_sha_1_ooo->lens[1] = 0; - hmac_sha_1_ooo->lens[2] = 0; - hmac_sha_1_ooo->lens[3] = 0; - hmac_sha_1_ooo->lens[4] = 0xFFFF; - hmac_sha_1_ooo->lens[5] = 0xFFFF; - hmac_sha_1_ooo->lens[6] = 0xFFFF; - hmac_sha_1_ooo->lens[7] = 0xFFFF; - hmac_sha_1_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA1_LANES; j++) { - hmac_sha_1_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_1_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_1_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_1_ooo->ldata[j].outer_block; - memset(p + 5*4 + 1, - 0x00, - 64 - 5*4 - 1 - 2); - p[5*4] = 0x80; - p[64-2] = 0x02; - p[64-1] = 0xA0; - } + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, SSE_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ - hmac_sha_224_ooo->lens[0] = 0; - hmac_sha_224_ooo->lens[1] = 0; - hmac_sha_224_ooo->lens[2] = 0; - hmac_sha_224_ooo->lens[3] = 0; - hmac_sha_224_ooo->lens[4] = 0xFFFF; - hmac_sha_224_ooo->lens[5] = 0xFFFF; - hmac_sha_224_ooo->lens[6] = 0xFFFF; - hmac_sha_224_ooo->lens[7] = 0xFFFF; - hmac_sha_224_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_224_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_sha_224_ooo->ldata[j].extra_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_sha_224_ooo->ldata[j].outer_block; - size = sizeof(hmac_sha_224_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[7*4] = 0x80; /* digest 7 words long */ - p[64-2] = 0x02; /* length in little endian = 0x02E0 */ - p[64-1] = 0xE0; - } + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + SSE_NUM_SHA256_LANES); /* Init HMAC/SHA_256 out-of-order fields */ - hmac_sha_256_ooo->lens[0] = 0; - hmac_sha_256_ooo->lens[1] = 0; - hmac_sha_256_ooo->lens[2] = 0; - hmac_sha_256_ooo->lens[3] = 0; - hmac_sha_256_ooo->lens[4] = 0xFFFF; - hmac_sha_256_ooo->lens[5] = 0xFFFF; - hmac_sha_256_ooo->lens[6] = 0xFFFF; - hmac_sha_256_ooo->lens[7] = 0xFFFF; - hmac_sha_256_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) { - hmac_sha_256_ooo->ldata[j].job_in_lane = NULL; - hmac_sha_256_ooo->ldata[j].extra_block[64] = 0x80; - memset(hmac_sha_256_ooo->ldata[j].extra_block + 65, - 0x00, - 64+7); - p = hmac_sha_256_ooo->ldata[j].outer_block; - memset(p + 8*4 + 1, - 0x00, - 64 - 8*4 - 1 - 2); /* digest is 8*4 bytes long */ - p[8*4] = 0x80; - p[64-2] = 0x03; /* length of (opad (64*8) bits + 256 bits) - * in hex is 0x300 */ - p[64-1] = 0x00; - } + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + SSE_NUM_SHA256_LANES); /* Init HMAC/SHA384 out-of-order fields */ - hmac_sha_384_ooo->lens[0] = 0; - hmac_sha_384_ooo->lens[1] = 0; - hmac_sha_384_ooo->lens[2] = 0xFFFF; - hmac_sha_384_ooo->lens[3] = 0xFFFF; - hmac_sha_384_ooo->lens[4] = 0xFFFF; - hmac_sha_384_ooo->lens[5] = 0xFFFF; - hmac_sha_384_ooo->lens[6] = 0xFFFF; - hmac_sha_384_ooo->lens[7] = 0xFFFF; - hmac_sha_384_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_384_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_384_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_384_BLOCK_SIZE + 1), - 0x00, IMB_SHA_384_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA384_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_384_BLOCK_SIZE - - IMB_SHA384_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA384_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 384 bits - * 1408 bits == 0x0580. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_384_BLOCK_SIZE - 2] = 0x05; - p[IMB_SHA_384_BLOCK_SIZE - 1] = 0x80; - } + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + SSE_NUM_SHA512_LANES); /* Init HMAC/SHA512 out-of-order fields */ - hmac_sha_512_ooo->lens[0] = 0; - hmac_sha_512_ooo->lens[1] = 0; - hmac_sha_512_ooo->lens[2] = 0xFFFF; - hmac_sha_512_ooo->lens[3] = 0xFFFF; - hmac_sha_512_ooo->lens[4] = 0xFFFF; - hmac_sha_512_ooo->lens[5] = 0xFFFF; - hmac_sha_512_ooo->lens[6] = 0xFFFF; - hmac_sha_512_ooo->lens[7] = 0xFFFF; - hmac_sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) { - MB_MGR_HMAC_SHA_512_OOO *ctx = hmac_sha_512_ooo; - - ctx->ldata[j].job_in_lane = NULL; - ctx->ldata[j].extra_block[IMB_SHA_512_BLOCK_SIZE] = 0x80; - memset(ctx->ldata[j].extra_block + (IMB_SHA_512_BLOCK_SIZE + 1), - 0x00, IMB_SHA_512_BLOCK_SIZE + 7); - - p = ctx->ldata[j].outer_block; - memset(p + IMB_SHA512_DIGEST_SIZE_IN_BYTES + 1, 0x00, - /* special end point because this length is constant */ - IMB_SHA_512_BLOCK_SIZE - - IMB_SHA512_DIGEST_SIZE_IN_BYTES - 1 - 2); - p[IMB_SHA512_DIGEST_SIZE_IN_BYTES] = 0x80; /* mark the end */ - /* - * hmac outer block length always of fixed size, it is OKey - * length, a whole message block length, 1024 bits, with padding - * plus the length of the inner digest, which is 512 bits - * 1536 bits == 0x600. The input message block needs to be - * converted to big endian within the sha implementation - * before use. - */ - p[IMB_SHA_512_BLOCK_SIZE - 2] = 0x06; - p[IMB_SHA_512_BLOCK_SIZE - 1] = 0x00; - } + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + SSE_NUM_SHA512_LANES); /* Init HMAC/MD5 out-of-order fields */ - hmac_md5_ooo->lens[0] = 0; - hmac_md5_ooo->lens[1] = 0; - hmac_md5_ooo->lens[2] = 0; - hmac_md5_ooo->lens[3] = 0; - hmac_md5_ooo->lens[4] = 0; - hmac_md5_ooo->lens[5] = 0; - hmac_md5_ooo->lens[6] = 0; - hmac_md5_ooo->lens[7] = 0; - hmac_md5_ooo->lens[8] = 0xFFFF; - hmac_md5_ooo->lens[9] = 0xFFFF; - hmac_md5_ooo->lens[10] = 0xFFFF; - hmac_md5_ooo->lens[11] = 0xFFFF; - hmac_md5_ooo->lens[12] = 0xFFFF; - hmac_md5_ooo->lens[13] = 0xFFFF; - hmac_md5_ooo->lens[14] = 0xFFFF; - hmac_md5_ooo->lens[15] = 0xFFFF; - hmac_md5_ooo->unused_lanes = 0xF76543210; - for (j = 0; j < SSE_NUM_MD5_LANES; j++) { - hmac_md5_ooo->ldata[j].job_in_lane = NULL; - - p = hmac_md5_ooo->ldata[j].extra_block; - size = sizeof(hmac_md5_ooo->ldata[j].extra_block); - memset (p, 0x00, size); - p[64] = 0x80; - - p = hmac_md5_ooo->ldata[j].outer_block; - size = sizeof(hmac_md5_ooo->ldata[j].outer_block); - memset(p, 0x00, size); - p[4*4] = 0x80; - p[64-7] = 0x02; - p[64-8] = 0x80; - } + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); /* Init AES/XCBC OOO fields */ - memset(aes_xcbc_ooo->lens, 0xff, - sizeof(aes_xcbc_ooo->lens)); - aes_xcbc_ooo->unused_lanes = 0xFF03020100; - for (j = 0; j < 4; j++) { - aes_xcbc_ooo->lens[j] = 0xFFFF; - aes_xcbc_ooo->ldata[j].job_in_lane = NULL; - aes_xcbc_ooo->ldata[j].final_block[16] = 0x80; - memset(aes_xcbc_ooo->ldata[j].final_block + 17, 0x00, 15); - } - aes_xcbc_ooo->num_lanes_inuse = 0; + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); /* Init AES-CCM auth out-of-order fields */ - memset(aes_ccm_ooo, 0, sizeof(MB_MGR_CCM_OOO)); - for (j = 4; j < 16; j++) - aes_ccm_ooo->lens[j] = 0xFFFF; - aes_ccm_ooo->unused_lanes = 0xF3210; - aes_ccm_ooo->num_lanes_inuse = 0; - - memset(aes256_ccm_ooo, 0, sizeof(MB_MGR_CCM_OOO)); - for (j = 4; j < 16; j++) - aes256_ccm_ooo->lens[j] = 0xFFFF; - aes256_ccm_ooo->unused_lanes = 0xF3210; - aes256_ccm_ooo->num_lanes_inuse = 0; + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 4); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 4); /* Init AES-CMAC auth out-of-order fields */ - aes_cmac_ooo->lens[0] = 0; - aes_cmac_ooo->lens[1] = 0; - aes_cmac_ooo->lens[2] = 0; - aes_cmac_ooo->lens[3] = 0; - aes_cmac_ooo->lens[4] = 0xFFFF; - aes_cmac_ooo->lens[5] = 0xFFFF; - aes_cmac_ooo->lens[6] = 0xFFFF; - aes_cmac_ooo->lens[7] = 0xFFFF; - for (j = 0; j < 4; j++) { - aes_cmac_ooo->init_done[j] = 0; - aes_cmac_ooo->job_in_lane[j] = NULL; - } - aes_cmac_ooo->unused_lanes = 0xF3210; - aes_cmac_ooo->num_lanes_inuse = 0; - - aes256_cmac_ooo->lens[0] = 0; - aes256_cmac_ooo->lens[1] = 0; - aes256_cmac_ooo->lens[2] = 0; - aes256_cmac_ooo->lens[3] = 0; - aes256_cmac_ooo->lens[4] = 0xFFFF; - aes256_cmac_ooo->lens[5] = 0xFFFF; - aes256_cmac_ooo->lens[6] = 0xFFFF; - aes256_cmac_ooo->lens[7] = 0xFFFF; - for (j = 0; j < 4; j++) { - aes256_cmac_ooo->init_done[j] = 0; - aes256_cmac_ooo->job_in_lane[j] = NULL; - } - aes256_cmac_ooo->unused_lanes = 0xF3210; - aes256_cmac_ooo->num_lanes_inuse = 0; + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 4); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 4); /* Init AES-CBCS out-of-order fields */ - memset(aes128_cbcs_ooo->lens, 0xFF, sizeof(aes128_cbcs_ooo->lens)); - memset(aes128_cbcs_ooo->job_in_lane, 0, - sizeof(aes128_cbcs_ooo->job_in_lane)); - aes128_cbcs_ooo->num_lanes_inuse = 0; - aes128_cbcs_ooo->unused_lanes = 0xF3210; + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); /* Init SHA1 out-of-order fields */ - sha_1_ooo->lens[0] = 0; - sha_1_ooo->lens[1] = 0; - sha_1_ooo->lens[2] = 0; - sha_1_ooo->lens[3] = 0; - sha_1_ooo->unused_lanes = 0xF3210; - sha_1_ooo->num_lanes_inuse = 0; - for (j = 0; j < SSE_NUM_SHA1_LANES; j++) - sha_1_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); /* Init SHA224 out-of-order fields */ - sha_224_ooo->lens[0] = 0; - sha_224_ooo->lens[1] = 0; - sha_224_ooo->lens[2] = 0; - sha_224_ooo->lens[3] = 0; - sha_224_ooo->unused_lanes = 0xF3210; - sha_224_ooo->num_lanes_inuse = 0; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) - sha_224_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); /* Init SHA256 out-of-order fields */ - sha_256_ooo->lens[0] = 0; - sha_256_ooo->lens[1] = 0; - sha_256_ooo->lens[2] = 0; - sha_256_ooo->lens[3] = 0; - sha_256_ooo->unused_lanes = 0xF3210; - sha_256_ooo->num_lanes_inuse = 0; - for (j = 0; j < SSE_NUM_SHA256_LANES; j++) - sha_256_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); /* Init SHA512 out-of-order fields */ - sha_512_ooo->lens[0] = 0; - sha_512_ooo->lens[1] = 0; - sha_512_ooo->unused_lanes = 0xFF0100; - for (j = 0; j < SSE_NUM_SHA512_LANES; j++) - sha_512_ooo->ldata[j].job_in_lane = NULL; + ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void -- GitLab From a2197a4ee875b40f12a8fa4076939aa1416509f5 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 15 Jul 2022 14:12:18 +0000 Subject: [PATCH 244/369] lib: [memcpy] reduce number of compare instructions on load and store --- lib/include/memcpy.asm | 50 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/lib/include/memcpy.asm b/lib/include/memcpy.asm index 6c6b95b2..ad85b206 100644 --- a/lib/include/memcpy.asm +++ b/lib/include/memcpy.asm @@ -565,33 +565,26 @@ or %%SIZE, %%SIZE je %%end_load %endif - cmp %%SIZE, 1 - je %%_size_1 cmp %%SIZE, 2 + jb %%_size_1 je %%_size_2 - cmp %%SIZE, 3 - je %%_size_3 cmp %%SIZE, 4 + jb %%_size_3 je %%_size_4 - cmp %%SIZE, 5 - je %%_size_5 cmp %%SIZE, 6 + jb %%_size_5 je %%_size_6 - cmp %%SIZE, 7 - je %%_size_7 cmp %%SIZE, 8 + jb %%_size_7 je %%_size_8 - cmp %%SIZE, 9 - je %%_size_9 cmp %%SIZE, 10 + jb %%_size_9 je %%_size_10 - cmp %%SIZE, 11 - je %%_size_11 cmp %%SIZE, 12 + jb %%_size_11 je %%_size_12 - cmp %%SIZE, 13 - je %%_size_13 cmp %%SIZE, 14 + jb %%_size_13 je %%_size_14 %%_size_15: @@ -654,12 +647,29 @@ sub %%TMP, 16 %%_check_size: -%assign %%I 1 -%rep 16 - cmp %%TMP, %%I - je APPEND(%%_size_, %%I) -%assign %%I (%%I+1) -%endrep + cmp %%TMP, 2 + jb %%_size_1 + je %%_size_2 + cmp %%TMP, 4 + jb %%_size_3 + je %%_size_4 + cmp %%TMP, 6 + jb %%_size_5 + je %%_size_6 + cmp %%TMP, 8 + jb %%_size_7 + je %%_size_8 + cmp %%TMP, 10 + jb %%_size_9 + je %%_size_10 + cmp %%TMP, 12 + jb %%_size_11 + je %%_size_12 + cmp %%TMP, 14 + jb %%_size_13 + je %%_size_14 + cmp %%TMP, 15 + je %%_size_15 %%_size_16: vmovdqu XWORD(%%DST), [%%IDX] -- GitLab From 8541f33ea3443e5582f1b0c19ecc6fa19f8e5aa2 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Fri, 15 Jul 2022 15:32:46 +0000 Subject: [PATCH 245/369] sse: [SHA] reset OOO mgr if SHAEXT not defined Reset SHA OOO mgrs also when SHAEXT is not defined. --- lib/sse_t1/mb_mgr_sse.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 2cf6da48..bb01fa39 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -714,34 +714,31 @@ reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); #ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { + if (state->features & IMB_FEATURE_SHANI) /* Init SHA1 NI out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, 2); - } else { + else +#endif /* HASH_USE_SHAEXT */ /* Init SHA1 out-of-order fields */ ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); - } -#endif /* HASH_USE_SHAEXT */ #ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { + if (state->features & IMB_FEATURE_SHANI) /* Init SHA224 NI out-of-order fields */ ooo_mgr_sha256_reset(state->sha_224_ooo, 2); - } else { + else +#endif /* HASH_USE_SHAEXT */ /* Init SHA224 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); - } -#endif /* HASH_USE_SHAEXT */ #ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { + if (state->features & IMB_FEATURE_SHANI) /* Init SHA256 NI out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, 2); - } else { + else +#endif /* HASH_USE_SHAEXT */ /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); - } -#endif /* HASH_USE_SHAEXT */ /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); -- GitLab From b0133d1ce3c4f8f8521c65e6ed2941c62ccdbed4 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 22 Jun 2022 13:48:57 +0000 Subject: [PATCH 246/369] sse: [SHA384] expose multi-buffer API --- lib/include/arch_sse_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 14 ++++++++++++++ lib/include/sha_mb_mgr.h | 22 ++++++++++++++++++++-- lib/intel-ipsec-mb.h | 1 + lib/no-aesni/mb_mgr_sse_no_aesni.c | 5 +++++ lib/sse_t1/mb_mgr_sse.c | 5 +++++ lib/sse_t1/sha_mb_sse.c | 21 +++++++++++++++++++++ lib/x86_64/alloc.c | 1 + 8 files changed, 72 insertions(+), 2 deletions(-) diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index 3f005be7..941e9991 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -212,6 +212,11 @@ IMB_JOB *submit_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha384_sse(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha384_sse(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha512_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_sse(MB_MGR_SHA_512_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index e65a30e5..ed8d60a3 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,6 +638,9 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; +#endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #if defined (SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; @@ -714,11 +717,15 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: +#ifdef SSE + return SUBMIT_JOB_SHA384(sha_384_ooo, job); +#else IMB_SHA384(state, job->src + job->hash_start_src_offset_in_bytes, job->msg_len_to_hash_in_bytes, job->auth_tag_output); job->status |= IMB_STATUS_COMPLETED_AUTH; return job; +#endif case IMB_AUTH_SHA_512: return SUBMIT_JOB_SHA512(sha_512_ooo, job); case IMB_AUTH_ZUC_EIA3_BITLEN: @@ -843,6 +850,9 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; +#ifdef SSE + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; +#endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #if defined(SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; @@ -891,6 +901,10 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); +#ifdef SSE + case IMB_AUTH_SHA_384: + return FLUSH_JOB_SHA384(sha_384_ooo, job); +#endif case IMB_AUTH_SHA_512: return FLUSH_JOB_SHA512(sha_512_ooo, job); case IMB_AUTH_AES_XCBC: diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index 488e5ed4..fea65d9c 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -182,6 +182,19 @@ void sha256_ni_mb_init_digest(uint32_t *digest, const unsigned lane) digest[8*lane + 7] = SHA256_H7; } +__forceinline +void sha384_mb_init_digest(uint64_t *digest, const unsigned lane) +{ + digest[lane + 0*8] = SHA384_H0; + digest[lane + 1*8] = SHA384_H1; + digest[lane + 2*8] = SHA384_H2; + digest[lane + 3*8] = SHA384_H3; + digest[lane + 4*8] = SHA384_H4; + digest[lane + 5*8] = SHA384_H5; + digest[lane + 6*8] = SHA384_H6; + digest[lane + 7*8] = SHA384_H7; +} + __forceinline void sha512_mb_init_digest(uint64_t *digest, const unsigned lane) { @@ -205,7 +218,9 @@ sha_mb_generic_init(void *digest, const int sha_type, const unsigned lane) sha224_mb_init_digest(digest, lane); else if (sha_type == 256) sha256_mb_init_digest(digest, lane); - else if (sha_type == 512) + else if (sha_type == 384) + sha384_mb_init_digest(digest, lane); + else /* sha_type == 512 */ sha512_mb_init_digest(digest, lane); } @@ -235,7 +250,10 @@ void sha_mb_generic_write_digest(void *dst, const void *src, else if (sha_type == 256) copy_bswap4_array_mb(dst, src, NUM_SHA_256_DIGEST_WORDS, offset, lane); - else if (sha_type == 512) + else if (sha_type == 384) + copy_bswap8_array_mb(dst, src, NUM_SHA_384_DIGEST_WORDS, offset, + lane); + else /* sha_type == 512 */ copy_bswap8_array_mb(dst, src, NUM_SHA_512_DIGEST_WORDS, offset, lane); } diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 4942cc0e..31a0ac8f 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1223,6 +1223,7 @@ typedef struct IMB_MGR { void *sha_1_ooo; void *sha_224_ooo; void *sha_256_ooo; + void *sha_384_ooo; void *sha_512_ooo; void *end_ooo; /* add new out-of-order managers above this line */ } IMB_MGR; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 0e57b945..ca57083c 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -100,6 +100,8 @@ #define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_sse +#define FLUSH_JOB_SHA384 flush_job_sha384_sse #define SUBMIT_JOB_SHA512 submit_job_sha512_sse #define FLUSH_JOB_SHA512 flush_job_sha512_sse @@ -447,6 +449,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); + /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index bb01fa39..bf58083c 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -104,6 +104,8 @@ #define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse #define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse #define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_sse +#define FLUSH_JOB_SHA384 flush_job_sha384_sse #define SUBMIT_JOB_SHA512 submit_job_sha512_sse #define FLUSH_JOB_SHA512 flush_job_sha512_sse @@ -740,6 +742,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); + /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); diff --git a/lib/sse_t1/sha_mb_sse.c b/lib/sse_t1/sha_mb_sse.c index 16c07d44..e528e6ff 100644 --- a/lib/sse_t1/sha_mb_sse.c +++ b/lib/sse_t1/sha_mb_sse.c @@ -91,6 +91,27 @@ IMB_JOB *flush_job_sha256_sse(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) call_sha_256_mult_sse_from_c, 0); } +/* ========================================================================== */ +/* + * SHA384 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha384_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 1, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x2_sse_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha384_sse(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 0, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x2_sse_from_c); +} + /* ========================================================================== */ /* * SHA512 MB API diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 144dc10e..592810c4 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -87,6 +87,7 @@ const struct { OOO_INFO(sha_1_ooo, MB_MGR_SHA_1_OOO), OOO_INFO(sha_224_ooo, MB_MGR_SHA_256_OOO), OOO_INFO(sha_256_ooo, MB_MGR_SHA_256_OOO), + OOO_INFO(sha_384_ooo, MB_MGR_SHA_512_OOO), OOO_INFO(sha_512_ooo, MB_MGR_SHA_512_OOO) }; -- GitLab From ba47c73dcc267e3036f7d2e23a1ed7d92334cfb7 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 22 Jun 2022 15:22:18 +0000 Subject: [PATCH 247/369] avx: [SHA384] expose multi-buffer API --- lib/avx/mb_mgr_avx.c | 5 +++++ lib/avx/sha_mb_avx.c | 24 ++++++++++++++++++++++++ lib/include/arch_avx_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 8 ++++---- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 6f2c2bd8..2757f087 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -127,6 +127,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx #define SUBMIT_JOB_SHA256 submit_job_sha256_avx #define FLUSH_JOB_SHA256 flush_job_sha256_avx +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx +#define FLUSH_JOB_SHA384 flush_job_sha384_avx #define SUBMIT_JOB_SHA512 submit_job_sha512_avx #define FLUSH_JOB_SHA512 flush_job_sha512_avx @@ -442,6 +444,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX_NUM_SHA256_LANES); + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX_NUM_SHA512_LANES); + /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX_NUM_SHA512_LANES); } diff --git a/lib/avx/sha_mb_avx.c b/lib/avx/sha_mb_avx.c index add37ee3..b8026e7d 100644 --- a/lib/avx/sha_mb_avx.c +++ b/lib/avx/sha_mb_avx.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx_type1.h" +IMB_JOB *submit_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); @@ -94,6 +97,27 @@ IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) call_sha_256_mult_avx_from_c, 0); } +/* ========================================================================== */ +/* + * SHA384 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 1, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x2_avx_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha384_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 2, 0, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x2_avx_from_c); +} + /* ========================================================================== */ /* * SHA512 MB API diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index 9c7bb420..c6827b2d 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -159,6 +159,11 @@ IMB_JOB *submit_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha384_avx(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx(MB_MGR_SHA_512_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index ed8d60a3..c85ce808 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,7 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; #endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; @@ -717,7 +717,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: -#ifdef SSE +#if defined(SSE) || defined(AVX) return SUBMIT_JOB_SHA384(sha_384_ooo, job); #else IMB_SHA384(state, @@ -850,7 +850,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#ifdef SSE +#if defined(SSE) || defined(AVX) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; #endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; @@ -901,7 +901,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#ifdef SSE +#if defined(SSE) || defined(AVX) case IMB_AUTH_SHA_384: return FLUSH_JOB_SHA384(sha_384_ooo, job); #endif -- GitLab From 3bff66be133f30fa096ce54ba1bfdfdb72d41d26 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 22 Jun 2022 15:32:15 +0000 Subject: [PATCH 248/369] avx2: [SHA384] expose multi-buffer API --- lib/avx2_t1/mb_mgr_avx2.c | 5 +++++ lib/avx2_t1/sha_mb_avx2.c | 24 ++++++++++++++++++++++++ lib/include/arch_avx2_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 8 ++++---- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 34ea3dab..44a652ef 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -137,6 +137,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx2 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 #define FLUSH_JOB_SHA256 flush_job_sha256_avx2 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx2 #define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 #define FLUSH_JOB_SHA512 flush_job_sha512_avx2 @@ -422,6 +424,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); + /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); } diff --git a/lib/avx2_t1/sha_mb_avx2.c b/lib/avx2_t1/sha_mb_avx2.c index d0813518..fcca3021 100644 --- a/lib/avx2_t1/sha_mb_avx2.c +++ b/lib/avx2_t1/sha_mb_avx2.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx2_type1.h" +IMB_JOB *submit_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); @@ -94,6 +97,27 @@ IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) call_sha256_oct_avx2_from_c, 0); } +/* ========================================================================== */ +/* + * SHA384 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 4, 1, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x4_avx2_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 4, 0, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x4_avx2_from_c); +} + /* ========================================================================== */ /* * SHA512 MB API diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index ccca8414..6df5ef99 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -68,6 +68,11 @@ IMB_JOB *submit_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx2(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx2(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx2(MB_MGR_SHA_512_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index c85ce808..b497ccc7 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,7 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; #endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; @@ -717,7 +717,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA384(sha_384_ooo, job); #else IMB_SHA384(state, @@ -850,7 +850,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; #endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; @@ -901,7 +901,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#if defined(SSE) || defined(AVX) +#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_384: return FLUSH_JOB_SHA384(sha_384_ooo, job); #endif -- GitLab From 0d261f86636b27b57cb57946b645654157565192 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 22 Jun 2022 15:43:19 +0000 Subject: [PATCH 249/369] avx512: [SHA384] expose multi-buffer API --- lib/avx512_t1/mb_mgr_avx512.c | 5 +++++ lib/avx512_t1/sha_mb_avx512.c | 24 ++++++++++++++++++++++++ lib/include/arch_avx512_type1.h | 5 +++++ lib/include/mb_mgr_code.h | 14 -------------- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 179dc45a..d21afb69 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -122,6 +122,8 @@ #define FLUSH_JOB_SHA224 flush_job_sha224_avx512 #define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 #define FLUSH_JOB_SHA256 flush_job_sha256_avx512 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx512 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx512 #define SUBMIT_JOB_SHA512 submit_job_sha512_avx512 #define FLUSH_JOB_SHA512 flush_job_sha512_avx512 @@ -992,6 +994,9 @@ reset_ooo_mgrs(IMB_MGR *state) /* Init SHA256 out-of-order fields */ ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX512_NUM_SHA512_LANES); + /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX512_NUM_SHA512_LANES); } diff --git a/lib/avx512_t1/sha_mb_avx512.c b/lib/avx512_t1/sha_mb_avx512.c index 1c543b36..6b283fc2 100644 --- a/lib/avx512_t1/sha_mb_avx512.c +++ b/lib/avx512_t1/sha_mb_avx512.c @@ -28,6 +28,9 @@ #include "include/sha_mb_mgr.h" #include "include/arch_avx512_type1.h" +IMB_JOB *submit_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); @@ -95,6 +98,27 @@ IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job) call_sha256_x16_avx512_from_c, 0); } +/* ========================================================================== */ +/* + * SHA384 MB API + */ + +IMB_DLL_LOCAL +IMB_JOB *submit_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 8, 1, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x8_avx512_from_c); +} + +IMB_DLL_LOCAL +IMB_JOB *flush_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job) +{ + return submit_flush_job_sha_512(state, job, 8, 0, 384, + IMB_SHA_512_BLOCK_SIZE, SHA384_PAD_SIZE, + call_sha512_x8_avx512_from_c); +} + /* ========================================================================== */ /* * SHA512 MB API diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index 233785d4..fc7a3adb 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -90,6 +90,11 @@ IMB_JOB *submit_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *flush_job_sha256_avx512(MB_MGR_SHA_256_OOO *state, IMB_JOB *job); +IMB_JOB *submit_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); +IMB_JOB *flush_job_sha384_avx512(MB_MGR_SHA_512_OOO *state, + IMB_JOB *job); + IMB_JOB *submit_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_sha512_avx512(MB_MGR_SHA_512_OOO *state, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index b497ccc7..30d38acf 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -638,9 +638,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; -#endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #if defined (SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; @@ -717,15 +715,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) #endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: -#if defined(SSE) || defined(AVX) || defined(AVX2) return SUBMIT_JOB_SHA384(sha_384_ooo, job); -#else - IMB_SHA384(state, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bytes, job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif case IMB_AUTH_SHA_512: return SUBMIT_JOB_SHA512(sha_512_ooo, job); case IMB_AUTH_ZUC_EIA3_BITLEN: @@ -850,9 +840,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; -#if defined(SSE) || defined(AVX) || defined(AVX2) MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; -#endif MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; #if defined(SSE) || defined (AVX512) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; @@ -901,10 +889,8 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); #endif return FLUSH_JOB_SHA256(sha_256_ooo, job); -#if defined(SSE) || defined(AVX) || defined(AVX2) case IMB_AUTH_SHA_384: return FLUSH_JOB_SHA384(sha_384_ooo, job); -#endif case IMB_AUTH_SHA_512: return FLUSH_JOB_SHA512(sha_512_ooo, job); case IMB_AUTH_AES_XCBC: -- GitLab From ac831e919db83f9451d376a825f8da55f987f2c6 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Wed, 22 Jun 2022 15:46:23 +0000 Subject: [PATCH 250/369] lib: update release notes and readme --- README | 1 + README.md | 1 + ReleaseNotes.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index 71704eef..2a7330dc 100644 --- a/README +++ b/README @@ -138,6 +138,7 @@ implementations. | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | + | SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/README.md b/README.md index e7c2657f..d7780a7d 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Table 2. List of supported integrity algorithms and their implementations. | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | +| SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index d65157a1..b35dedd7 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -10,7 +10,7 @@ Library - JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added -- SHA1/224/256/512 multi-buffer implementation added +- SHA1/224/256/384/512 multi-buffer implementation added - Burst API added - Cipher-only burst API added (AES-CBC/CTR support only) - Hash-only burst API added (HMAC-SHA1/224/256/384/512 support only) -- GitLab From 47558769ec8be1eed15da7ae49e2057a158a8e53 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Mon, 18 Jul 2022 14:32:34 +0000 Subject: [PATCH 251/369] test: update cross-validation tool --- test/ipsec_xvalid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 0844463a..c44c1d4b 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -1701,7 +1701,7 @@ perform_safe_checks(IMB_MGR *mgr, const IMB_ARCH arch, const char *dir) "aes256_ccm_ooo", "aes256_cmac_ooo", "snow3g_uea2_ooo", "snow3g_uia2_ooo", "sha_1_ooo", "sha_224_ooo", "sha_256_ooo", - "sha_512_ooo", + "sha_384_ooo", "sha_512_ooo", "end_ooo" /* add new ooo manager above this line */ }; void *ooo_mgr_p = *ooo_ptr; -- GitLab From f7bcc35aa1dcced13d7895798687c0ac1d751651 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 22 Jul 2022 14:41:19 +0000 Subject: [PATCH 252/369] CI: [BSD] update macos version for github actions --- .github/workflows/freebsd_build_shared_clang.yml | 4 ++-- .github/workflows/freebsd_build_shared_gcc.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/freebsd_build_shared_clang.yml b/.github/workflows/freebsd_build_shared_clang.yml index 3ecc0e46..6104ca5a 100644 --- a/.github/workflows/freebsd_build_shared_clang.yml +++ b/.github/workflows/freebsd_build_shared_clang.yml @@ -8,7 +8,7 @@ on: jobs: testfreebsd: - runs-on: macos-10.15 + runs-on: macos-12 name: FreeBSD run steps: - name: Checkout @@ -17,7 +17,7 @@ jobs: repository: intel/intel-ipsec-mb - name: Test in FreeBSD id: tests - uses: vmactions/freebsd-vm@v0.1.5 + uses: vmactions/freebsd-vm@v0.2.0 with: usesh: true mem: 8192 diff --git a/.github/workflows/freebsd_build_shared_gcc.yml b/.github/workflows/freebsd_build_shared_gcc.yml index 4ae7e405..c7824d73 100644 --- a/.github/workflows/freebsd_build_shared_gcc.yml +++ b/.github/workflows/freebsd_build_shared_gcc.yml @@ -8,7 +8,7 @@ on: jobs: testfreebsd: - runs-on: macos-10.15 + runs-on: macos-12 name: FreeBSD GCC run steps: - name: Checkout @@ -17,7 +17,7 @@ jobs: repository: intel/intel-ipsec-mb - name: Test FreeBSD GCC build id: tests - uses: vmactions/freebsd-vm@v0.1.5 + uses: vmactions/freebsd-vm@v0.2.0 with: usesh: true mem: 8192 -- GitLab From e5c2a1f11b39a2ad4ea11747e2c3b72b3aa4ee88 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 21 Jul 2022 11:58:46 +0100 Subject: [PATCH 253/369] sse_t1: Fix AES-CMAC-256 GP reg containing PT data --- lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm index 2b597f07..8aecdc2d 100644 --- a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm @@ -240,7 +240,7 @@ endstruc shl tmp3, 4 add tmp, tmp3 - memcpy_sse_16 m_last, tmp, r, tmp4, iv + memcpy_sse_16 m_last, tmp, r, tmp4, tmp3 ;; src + n + r mov tmp3, [job + _skey2] -- GitLab From cdce83b2e6df23cf0a6647415e7ad6a23ed53eb1 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 21 Jul 2022 12:03:04 +0100 Subject: [PATCH 254/369] lib: [KASUMI] Fix for Windows storing PT data on then stack --- lib/include/kasumi_internal.h | 5 +++-- lib/include/memcpy.h | 3 +++ lib/sse_t1/memcpy_sse.asm | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/include/kasumi_internal.h b/lib/include/kasumi_internal.h index 96e98f58..bb9e76ab 100755 --- a/lib/include/kasumi_internal.h +++ b/lib/include/kasumi_internal.h @@ -42,6 +42,7 @@ #include "wireless_common.h" #include "include/clear_regs_mem.h" #include "include/constant_lookup.h" +#include "memcpy.h" #include "error.h" /*--------------------------------------------------------------------- @@ -1680,7 +1681,7 @@ kasumi_f9_1_buffer(const kasumi_key_sched_t *pCtx, const void *dataIn, /* Not a whole 8 byte block remaining */ mask.b64[0] = ~(mask.b64[0] >> (BYTESIZE * lengthInBytes)); - memcpy(&safeBuf.b64, pIn, lengthInBytes); + safe_memcpy(&safeBuf.b64, pIn, lengthInBytes); mask.b64[0] &= BSWAP64(safeBuf.b64); a.b64[0] ^= mask.b64[0]; @@ -1768,7 +1769,7 @@ kasumi_f9_1_buffer_user(const kasumi_key_sched_t *pCtx, const uint64_t IV, message.b64[0] = 0; mask.b64[0] = ~(mask.b64[0] >> lengthInBits); /*round up and copy last lengthInBits */ - memcpy(&safebuff.b64[0], pIn, (lengthInBits + 7) / 8); + safe_memcpy(&safebuff.b64[0], pIn, (lengthInBits + 7) / 8); message.b64[0] = BSWAP64(safebuff.b64[0]); temp.b64[0] = mask.b64[0] & message.b64[0]; temp.b64[0] |= diff --git a/lib/include/memcpy.h b/lib/include/memcpy.h index 92cd2c56..20e9ac35 100644 --- a/lib/include/memcpy.h +++ b/lib/include/memcpy.h @@ -37,4 +37,7 @@ void memcpy_fn_avx_16(void *dst, const void *src, const size_t size); /* Memcpy 128 bytes with SSE instructions */ void memcpy_fn_sse_128(void *dst, const void *src); +/* Basic memcpy that doesn't use stack */ +void safe_memcpy(void *dst, const void *src, const size_t size); + #endif /* MEMCPY_H */ diff --git a/lib/sse_t1/memcpy_sse.asm b/lib/sse_t1/memcpy_sse.asm index 22a35b83..b2433175 100644 --- a/lib/sse_t1/memcpy_sse.asm +++ b/lib/sse_t1/memcpy_sse.asm @@ -68,4 +68,24 @@ memcpy_fn_sse_128: ret +MKGLOBAL(safe_memcpy,function,internal) +safe_memcpy: +%ifndef LINUX + ;; save rdi and rsi + mov rax, rdi + mov r9, rsi + + mov rdi, arg1 + mov rsi, arg2 +%endif + mov rcx, arg3 + rep movsb + +%ifndef LINUX + ;; restore rdi and rsi + mov rdi, rax + mov rsi, r9 +%endif + ret + mksection stack-noexec -- GitLab From a3d9c04fac7391d9a1416de57a5f8a2afc30ca03 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 09:51:50 +0000 Subject: [PATCH 255/369] test: [misc] add API's to detect XGETBV and use XGETBV to detected AVX-SSE transition problems --- test/misc.asm | 12 +++++++ test/misc.h | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/test/misc.asm b/test/misc.asm index 2b62dec7..f5e5b54f 100644 --- a/test/misc.asm +++ b/test/misc.asm @@ -60,6 +60,7 @@ section .bss default rel MKGLOBAL(gps,data,) +align 8 gps: resq 14 MKGLOBAL(simd_regs,data,) @@ -68,6 +69,17 @@ simd_regs: resb 32*64 section .text +;; uint32_t avx_sse_transition_check(void) +MKGLOBAL(avx_sse_transition_check,function,) +align 16 +avx_sse_transition_check: + mov ecx, 1 + xgetbv + ;; result goes to edx:eax + ;; we care about bits 2 and 6 only + and eax, (1 << 2) | (1 << 6) + ret + ;; void *nosimd_memcpy(void *dst, const void *src, size_t n) MKGLOBAL(nosimd_memcpy,function,) align 16 diff --git a/test/misc.h b/test/misc.h index 998e62b2..ba622ea3 100644 --- a/test/misc.h +++ b/test/misc.h @@ -25,6 +25,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *****************************************************************************/ +#ifdef __WIN32 +#include +#endif + #ifndef XVALIDAPP_MISC_H #define XVALIDAPP_MISC_H @@ -61,4 +65,86 @@ void *nosimd_memset(void *p, int c, size_t n); /* custom replacement for memcpy() */ void *nosimd_memcpy(void *dst, const void *src, size_t n); +/* + * Detects if SIMD registers are in the state that + * can cause AVX-SSE transition penalty + */ +uint32_t avx_sse_transition_check(void); + +#define MISC_AVX_SSE_YMM0_15_ISSUE (1 << 2) +#define MISC_AVX_SSE_ZMM0_15_ISSUE (1 << 6) +#define MISC_AVX_SSE_ISSUE (MISC_AVX_SSE_YMM0_15_ISSUE | \ + MISC_AVX_SSE_ZMM0_15_ISSUE) + +/* CPUID feature detection code follows here */ + +struct misc_cpuid_regs { + uint32_t eax; + uint32_t ebx; + uint32_t ecx; + uint32_t edx; +}; + +/** + * @brief C wrapper for CPUID opcode + * + * @param leaf[in] CPUID leaf number (EAX) + * @param subleaf[in] CPUID sub-leaf number (ECX) + * @param out[out] registers structure to store results of CPUID into + */ +static void +misc_cpuid(const unsigned leaf, const unsigned subleaf, + struct misc_cpuid_regs *out) +{ +#ifdef _WIN32 + /* Windows */ + int regs[4]; + + __cpuidex(regs, leaf, subleaf); + out->eax = regs[0]; + out->ebx = regs[1]; + out->ecx = regs[2]; + out->edx = regs[3]; +#else + /* Linux */ + asm volatile("mov %4, %%eax\n\t" + "mov %5, %%ecx\n\t" + "cpuid\n\t" + "mov %%eax, %0\n\t" + "mov %%ebx, %1\n\t" + "mov %%ecx, %2\n\t" + "mov %%edx, %3\n\t" + : "=g" (out->eax), "=g" (out->ebx), "=g" (out->ecx), + "=g" (out->edx) + : "g" (leaf), "g" (subleaf) + : "%eax", "%ebx", "%ecx", "%edx"); +#endif /* Linux */ +} + +/** + * @brief Detects if XGETBV instruction is available to use. + * Call it before calling avx_sse_transition_check(). + * + * @retval 0 XGETBV NOT available + * @retval 1 XGETBV available + */ +static int avx_sse_detectability(void) +{ + struct misc_cpuid_regs r; + + /* Get highest supported CPUID leaf number */ + misc_cpuid(0x0, 0x0, &r); + + const unsigned hi_leaf_number = r.eax; + + if (hi_leaf_number < 0xd) + return 0; + + /* Get CPUID leaf 0xd subleaf 0x1 */ + misc_cpuid(0xd, 0x1, &r); + + /* return bit 2 from EAX */ + return (r.eax >> 2) & 1; +} + #endif /* XVALIDAPP_MISC_H */ -- GitLab From 811dafa8d082d3e3d900818ce9991500362f7019 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 09:52:50 +0000 Subject: [PATCH 256/369] test: [xvalid] add AVX-SSE transition check to the xvalid tool --- test/ipsec_xvalid.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index c44c1d4b..ce7a0f55 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -690,6 +690,31 @@ uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1}; uint64_t flags = 0; /* flags passed to alloc_mb_mgr() */ +int is_avx_sse_check_possible = -1; /* -1 => detect, 0 => not possible, 1 => possible */ + +static void +avx_sse_check(const char *ctx_str, + const unsigned hash_alg, + const unsigned cipher_mode) +{ + if (!is_avx_sse_check_possible) + return; + + if (is_avx_sse_check_possible == -1) + is_avx_sse_check_possible = avx_sse_detectability(); + + const uint32_t avx_sse_flag = avx_sse_transition_check(); + + if (avx_sse_flag & MISC_AVX_SSE_ZMM0_15_ISSUE) + printf("AVX-SSE transition after %s in ZMM0-ZMM15: " + "hash_alg = %u, cipher_mode = %u, flag 0x%x\n", + ctx_str, hash_alg, cipher_mode, avx_sse_flag); + else if (avx_sse_flag & MISC_AVX_SSE_YMM0_15_ISSUE) + printf("AVX-SSE transition after %s in YMM0-YMM15: " + "hash_alg = %u, cipher_mode = %u, flag 0x%x\n", + ctx_str, hash_alg, cipher_mode, avx_sse_flag); +} + static void clear_data(struct data *data) { @@ -1960,6 +1985,9 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, * other functions from storing sensitive data in stack */ job = IMB_SUBMIT_JOB(enc_mb_mgr); + avx_sse_check("enc-submit", (unsigned) params->hash_alg, + (unsigned) params->cipher_mode); + if (job) { unsigned idx = (unsigned)((uintptr_t) job->user_data); @@ -1994,6 +2022,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Flush rest of the jobs, if there are outstanding jobs */ while (num_processed_jobs != num_jobs) { job = IMB_FLUSH_JOB(enc_mb_mgr); + + avx_sse_check("enc-flush", (unsigned) params->hash_alg, + (unsigned) params->cipher_mode); + while (job != NULL) { unsigned idx = (unsigned)((uintptr_t) job->user_data); @@ -2079,6 +2111,9 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, * other functions from storing sensitive data in stack */ job = IMB_SUBMIT_JOB(dec_mb_mgr); + avx_sse_check("dec-submit", (unsigned) params->hash_alg, + (unsigned) params->cipher_mode); + if (job != NULL) { unsigned idx = (unsigned)((uintptr_t) job->user_data); @@ -2104,6 +2139,10 @@ do_test(IMB_MGR *enc_mb_mgr, const IMB_ARCH enc_arch, /* Flush rest of the jobs, if there are outstanding jobs */ while (num_processed_jobs != num_jobs) { job = IMB_FLUSH_JOB(dec_mb_mgr); + + avx_sse_check("dec-flush", (unsigned) params->hash_alg, + (unsigned) params->cipher_mode); + while (job != NULL) { unsigned idx = (unsigned)((uintptr_t) job->user_data); -- GitLab From facc8b8f6556b869dbd973fe2894649e20aeb7bf Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 10:38:15 +0000 Subject: [PATCH 257/369] test: [xvalid] decode cipher and hash algorithms as strings --- test/ipsec_xvalid.c | 25 ++++--- test/misc.h | 171 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 9 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index ce7a0f55..035e721b 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -690,12 +690,13 @@ uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1}; uint64_t flags = 0; /* flags passed to alloc_mb_mgr() */ -int is_avx_sse_check_possible = -1; /* -1 => detect, 0 => not possible, 1 => possible */ +/* -1 => detect, 0 => not possible, 1 => possible */ +int is_avx_sse_check_possible = -1; static void avx_sse_check(const char *ctx_str, - const unsigned hash_alg, - const unsigned cipher_mode) + const IMB_HASH_ALG hash_alg, + const IMB_CIPHER_MODE cipher_mode) { if (!is_avx_sse_check_possible) return; @@ -705,14 +706,20 @@ avx_sse_check(const char *ctx_str, const uint32_t avx_sse_flag = avx_sse_transition_check(); + if (!avx_sse_flag) + return; + + const char *hash_str = misc_hash_alg_to_str(hash_alg); + const char *cipher_str = misc_cipher_mode_to_str(cipher_mode); + if (avx_sse_flag & MISC_AVX_SSE_ZMM0_15_ISSUE) - printf("AVX-SSE transition after %s in ZMM0-ZMM15: " - "hash_alg = %u, cipher_mode = %u, flag 0x%x\n", - ctx_str, hash_alg, cipher_mode, avx_sse_flag); + printf("ERROR: AVX-SSE transition after %s in ZMM0-ZMM15: " + "hash: %s, cipher: %s\n", + ctx_str, hash_str, cipher_str); else if (avx_sse_flag & MISC_AVX_SSE_YMM0_15_ISSUE) - printf("AVX-SSE transition after %s in YMM0-YMM15: " - "hash_alg = %u, cipher_mode = %u, flag 0x%x\n", - ctx_str, hash_alg, cipher_mode, avx_sse_flag); + printf("ERROR: AVX-SSE transition after %s in YMM0-YMM15: " + "hash: %s, cipher: %s\n", + ctx_str, hash_str, cipher_str); } static void diff --git a/test/misc.h b/test/misc.h index ba622ea3..99539620 100644 --- a/test/misc.h +++ b/test/misc.h @@ -29,6 +29,8 @@ #include #endif +#include + #ifndef XVALIDAPP_MISC_H #define XVALIDAPP_MISC_H @@ -147,4 +149,173 @@ static int avx_sse_detectability(void) return (r.eax >> 2) & 1; } +/* decodes cipher mode to string */ +static const char *misc_cipher_mode_to_str(const IMB_CIPHER_MODE mode) +{ + static char cb[64]; + + switch (mode) { + case IMB_CIPHER_CBC: + return "aes-cbc"; + case IMB_CIPHER_CNTR: + return "aes-ctr"; + case IMB_CIPHER_NULL: + return "null"; + case IMB_CIPHER_DOCSIS_SEC_BPI: + return "aes-docsis"; + case IMB_CIPHER_GCM: + return "aead-aes-gcm"; + case IMB_CIPHER_CUSTOM: + return "custom"; + case IMB_CIPHER_DES: + return "des-cbc"; + case IMB_CIPHER_DOCSIS_DES: + return "des-docsis"; + case IMB_CIPHER_CCM: + return "aes-ccm"; + case IMB_CIPHER_DES3: + return "3des-cbc"; + case IMB_CIPHER_PON_AES_CNTR: + return "pon-aes-ctr"; + case IMB_CIPHER_ECB: + return "aes-ecb"; + case IMB_CIPHER_CNTR_BITLEN: + return "aes-ctr (bitlen)"; + case IMB_CIPHER_ZUC_EEA3: + return "zuc-eea3"; + case IMB_CIPHER_SNOW3G_UEA2_BITLEN: + return "snow3g-uea2"; + case IMB_CIPHER_KASUMI_UEA1_BITLEN: + return "kasumi-uea1"; + case IMB_CIPHER_CBCS_1_9: + return "aes-cbcs-1-9"; + case IMB_CIPHER_CHACHA20: + return "chacha20"; + case IMB_CIPHER_CHACHA20_POLY1305: + return "aead-chacha20-poly1305"; + case IMB_CIPHER_CHACHA20_POLY1305_SGL: + return "aead-chacha20-poly1305-sgl"; + case IMB_CIPHER_SNOW_V: + return "snow-v"; + case IMB_CIPHER_SNOW_V_AEAD: + return "aead-snow-v"; + case IMB_CIPHER_GCM_SGL: + return "aead-aes-gcm-sgl"; + case IMB_CIPHER_NUM: + default: + break; + } + + memset(cb, 0, sizeof(cb)); + snprintf(cb, sizeof(cb) - 1, "unknown<%u>", (unsigned) mode); + return cb; +} + +/* decodes hash algorithm to string */ +static const char *misc_hash_alg_to_str(const IMB_HASH_ALG mode) +{ + static char cb[64]; + + switch (mode) { + case IMB_AUTH_HMAC_SHA_1: + return "hmac-sha1"; + case IMB_AUTH_HMAC_SHA_224: + return "hmac-sha224"; + case IMB_AUTH_HMAC_SHA_256: + return "hmac-sha256"; + case IMB_AUTH_HMAC_SHA_384: + return "hmac-sha384"; + case IMB_AUTH_HMAC_SHA_512: + return "hmac-sha512"; + case IMB_AUTH_AES_XCBC: + return "aes-xcbc"; + case IMB_AUTH_MD5: + return "hmac-md5"; + case IMB_AUTH_NULL: + return "null"; + case IMB_AUTH_AES_GMAC: + return "aead-aes-gcm"; + case IMB_AUTH_CUSTOM: + return "custom"; + case IMB_AUTH_AES_CCM: + return "aes-ccm"; + case IMB_AUTH_AES_CMAC: + return "aes-cmac-128"; + case IMB_AUTH_SHA_1: + return "sha1"; + case IMB_AUTH_SHA_224: + return "sha224"; + case IMB_AUTH_SHA_256: + return "sha256"; + case IMB_AUTH_SHA_384: + return "sha384"; + case IMB_AUTH_SHA_512: + return "sha512"; + case IMB_AUTH_AES_CMAC_BITLEN: + return "aes-cmac (bitlen)"; + case IMB_AUTH_PON_CRC_BIP: + return "pon-crc-bip"; + case IMB_AUTH_ZUC_EIA3_BITLEN: + return "zuc-eia3"; + case IMB_AUTH_DOCSIS_CRC32: + return "docsis-crc32"; + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + return "snow3g-uia2"; + case IMB_AUTH_KASUMI_UIA1: + return "kasumi-uia1"; + case IMB_AUTH_AES_GMAC_128: + return "aes-gmac-128"; + case IMB_AUTH_AES_GMAC_192: + return "aes-gmac-192"; + case IMB_AUTH_AES_GMAC_256: + return "aes-gmac-256"; + case IMB_AUTH_AES_CMAC_256: + return "aes-cmac-256"; + case IMB_AUTH_POLY1305: + return "poly1305"; + case IMB_AUTH_CHACHA20_POLY1305: + return "aead-chacha20-poly1305"; + case IMB_AUTH_CHACHA20_POLY1305_SGL: + return "aead-chacha20-poly1305-sgl"; + case IMB_AUTH_ZUC256_EIA3_BITLEN: + return "zuc256-eia3"; + case IMB_AUTH_SNOW_V_AEAD: + return "aead-snow-v"; + case IMB_AUTH_GCM_SGL: + return "aead-aes-gcm-sgl"; + case IMB_AUTH_CRC32_ETHERNET_FCS: + return "crc32-ethernet-fcs"; + case IMB_AUTH_CRC32_SCTP: + return "crc32-sctp"; + case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: + return "crc32-wimax-ofdma-data"; + case IMB_AUTH_CRC24_LTE_A: + return "crc24-lte-a"; + case IMB_AUTH_CRC24_LTE_B: + return "crc24-lte-b"; + case IMB_AUTH_CRC16_X25: + return "crc16-x25"; + case IMB_AUTH_CRC16_FP_DATA: + return "crc16-fp-data"; + case IMB_AUTH_CRC11_FP_HEADER: + return "crc11-fp-header"; + case IMB_AUTH_CRC10_IUUP_DATA: + return "crc10-iuup-data"; + case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: + return "crc8-wimax-ofdma-hcs"; + case IMB_AUTH_CRC7_FP_HEADER: + return "crc7-fp-header"; + case IMB_AUTH_CRC6_IUUP_HEADER: + return "crc6-iuup-header"; + case IMB_AUTH_GHASH: + return "ghash"; + case IMB_AUTH_NUM: + default: + break; + } + + memset(cb, 0, sizeof(cb)); + snprintf(cb, sizeof(cb) - 1, "unknown<%u>", (unsigned) mode); + return cb; +} #endif /* XVALIDAPP_MISC_H */ -- GitLab From 7b6ec5d53f3c82a9b29f82c04eadb526577075c9 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 10:54:07 +0000 Subject: [PATCH 258/369] test: [xvalid] AVX-SSE check is disabled by default, command line option added to enable it --- test/ipsec_xvalid.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 035e721b..331a9dcb 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -690,8 +690,8 @@ uint8_t dec_archs[IMB_ARCH_NUM] = {0, 0, 1, 1, 1, 1}; uint64_t flags = 0; /* flags passed to alloc_mb_mgr() */ -/* -1 => detect, 0 => not possible, 1 => possible */ -int is_avx_sse_check_possible = -1; +/* 0 => not possible, 1 => possible */ +int is_avx_sse_check_possible = 0; static void avx_sse_check(const char *ctx_str, @@ -701,9 +701,6 @@ avx_sse_check(const char *ctx_str, if (!is_avx_sse_check_possible) return; - if (is_avx_sse_check_possible == -1) - is_avx_sse_check_possible = avx_sse_detectability(); - const uint32_t avx_sse_flag = avx_sse_transition_check(); if (!avx_sse_flag) @@ -714,11 +711,11 @@ avx_sse_check(const char *ctx_str, if (avx_sse_flag & MISC_AVX_SSE_ZMM0_15_ISSUE) printf("ERROR: AVX-SSE transition after %s in ZMM0-ZMM15: " - "hash: %s, cipher: %s\n", + "HASH=%s, CIPHER=%s\n", ctx_str, hash_str, cipher_str); else if (avx_sse_flag & MISC_AVX_SSE_YMM0_15_ISSUE) printf("ERROR: AVX-SSE transition after %s in YMM0-YMM15: " - "hash: %s, cipher: %s\n", + "HASH=%s, CIPHER=%s\n", ctx_str, hash_str, cipher_str); } @@ -2716,7 +2713,9 @@ static void usage(const char *app_name) "(maximum = %d)\n" "--safe-check: check if keys, IVs, plaintext or tags " "get cleared from IMB_MGR upon job completion (off by default; " - "requires library compiled with SAFE_DATA)\n", + "requires library compiled with SAFE_DATA)\n" + "--avx-sse: if XGETBV is available then check for potential " + "AVX-SSE transition problems\n", app_name, MAX_NUM_JOBS); } @@ -3034,6 +3033,10 @@ int main(int argc, char *argv[]) } } else if (strcmp(argv[i], "--imix") == 0) { imix_enabled = 1; + } else if (strcmp(argv[i], "--avx-sse") == 0) { + is_avx_sse_check_possible = avx_sse_detectability(); + if (!is_avx_sse_check_possible) + fprintf(stderr, "XGETBV not available\n"); } else { usage(argv[0]); return EXIT_FAILURE; -- GitLab From 28ad4722145a97a21292ab6239108641616dafe6 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 12:43:22 +0000 Subject: [PATCH 259/369] avx512: [hmac-sha1] fix vzeroupper for SAFE_OPTIONS=n --- lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm index f7fc8bfb..fb2ca565 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm @@ -51,6 +51,8 @@ ;; %define DO_DBGPRINT %include "include/dbgprint.asm" +%use smartalign + extern sha1_x16_avx512 mksection .rodata @@ -213,7 +215,6 @@ ge64_bytes: mov DWORD(num_lanes_inuse), [state + _num_lanes_inuse_sha1] cmp num_lanes_inuse, 0x10 ; all 16 lanes used? jne return_null - jmp start_loop align 16 start_loop: @@ -381,14 +382,13 @@ clear_ret: mov dword [lane_data + _outer_block + 16], 0 %endif +return: %ifdef SAFE_DATA clear_all_zmms_asm %else vzeroupper %endif -return: - DBGPRINTL "---------- exit sha1 submit -----------" mov rbp, [rsp + _gpr_save + 8*0] mov r12, [rsp + _gpr_save + 8*1] -- GitLab From 3f383da925691e1bf54ed03c200c26877adc2e05 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 12:47:27 +0000 Subject: [PATCH 260/369] avx512: [hmac-sha224/256] fix vzeroupper with SAFE_OPTIONS=n --- lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm index a54984cc..0ebbb582 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm @@ -51,6 +51,8 @@ ;; %define DO_DBGPRINT %include "include/dbgprint.asm" +%use smartalign + extern sha256_x16_avx512 mksection .rodata @@ -213,7 +215,6 @@ lt64_bytes: ge64_bytes: cmp dword [state + _num_lanes_inuse_sha256], 0x10 ; all 16 lanes used? jne return_null - jmp start_loop align 16 start_loop: @@ -427,13 +428,13 @@ clear_ret: %endif %endif ;; SAFE_DATA +return: %ifdef SAFE_DATA clear_all_zmms_asm %else vzeroupper %endif -return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] -- GitLab From ab3f3119cc71e58cb2358dedbf649f5aeb930992 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 12:49:16 +0000 Subject: [PATCH 261/369] avx512: [hmac-sha384/512] fix vzeroupper with SAFE_OPTIONS=n --- lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm index b7a9d4be..1166f08c 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm @@ -41,6 +41,8 @@ %include "include/cet.inc" %include "include/clear_regs.asm" +%use smartalign + extern sha512_x8_avx512 mksection .rodata @@ -208,7 +210,6 @@ lt128_bytes: ge128_bytes: cmp unused_lanes, 0xf jne return_null - jmp start_loop align 32 start_loop: @@ -396,13 +397,13 @@ clear_ret: %endif %endif ;; SAFE_DATA +return: %ifdef SAFE_DATA clear_all_zmms_asm %else vzeroupper %endif -return: mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] -- GitLab From 3e977b30da886c47c87f899dd6ce6a5194ee41de Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 13:22:34 +0000 Subject: [PATCH 262/369] avx512: [aes-cbcs-1-9] fix vzeroupper with SAFE_OPTIONS=n --- lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm | 16 ++-------------- .../mb_mgr_aes128_cbcs_1_9_flush_avx512.asm | 5 +++++ .../mb_mgr_aes128_cbcs_1_9_submit_avx512.asm | 5 +++++ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm index 82015d45..5fe71208 100644 --- a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm @@ -32,9 +32,6 @@ %include "include/reg_sizes.asm" %include "include/clear_regs.asm" %include "include/cet.inc" -struc STACK -_gpr_save: resq 1 -endstruc %define GPR_SAVE_AREA rsp + _gpr_save @@ -104,16 +101,13 @@ endstruc ;; Save registers states %macro FUNC_SAVE 0 - sub rsp, STACK_size - mov [GPR_SAVE_AREA + 8*0], rbp + push rbp %endmacro ;; Restore register states %macro FUNC_RESTORE 0 ;; XMMs are saved at a higher level - mov rbp, [GPR_SAVE_AREA + 8*0] - add rsp, STACK_size - vzeroupper + pop rbp %endmacro %macro LOAD_STORE_4x1 10 @@ -454,15 +448,9 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbcs_1_9_enc_128_vaes_avx512,function,internal) aes_cbcs_1_9_enc_128_vaes_avx512: - endbranch64 FUNC_SAVE CBCS_ENC 9, 160 FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%endif ;; SAFE_DATA - ret mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm index ab709c2e..de6963ee 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm @@ -198,6 +198,11 @@ len_is_0: %endif return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm index ec25e86c..c34d977c 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm @@ -197,6 +197,11 @@ len_is_0: %endif return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] -- GitLab From 7966b71ea96511a29e16cabe35d1a07f00fa3a5a Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 13:29:06 +0000 Subject: [PATCH 263/369] avx512: [aes-cmac-128/256] fix vzeroupper with SAFE_OPTIONS=n --- lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm | 28 ------------------- ...s128_cmac_submit_flush_x16_vaes_avx512.asm | 9 ++++-- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index b028ec90..f6fb76e0 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -1095,13 +1095,6 @@ aes128_cbc_mac_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1113,13 +1106,6 @@ aes256_cbc_mac_vaes_avx512: FUNC_SAVE CBC_ENC 13, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1211,13 +1197,6 @@ aes128_cbc_mac_flush_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_CBC, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1231,13 +1210,6 @@ aes256_cbc_mac_flush_vaes_avx512: FUNC_SAVE CBC_ENC 13, MAC_TYPE_CBC, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm index f0e51738..abbd81ed 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm @@ -31,6 +31,7 @@ %include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" +%include "include/clear_regs.asm" %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_vaes_avx512 @@ -479,12 +480,16 @@ endstruc %assign round (round + 1) %endrep +%endif ;; SAFE_DATA + +%%_return: +%ifdef SAFE_DATA + clear_all_zmms_asm %else vzeroupper %endif ;; SAFE_DATA -%%_return: - mov rbx, [rsp + _gpr_save + 8*0] + mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] mov r13, [rsp + _gpr_save + 8*3] -- GitLab From 923e0ffc68df12d15ac5c18c966bf4eafee8fb25 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 13:39:59 +0000 Subject: [PATCH 264/369] avx512: [aes-xcbc] fix vzeroupper with SAFE_OPTIONS=n --- lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm | 14 -------------- ...gr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm | 12 ++++++++---- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index f6fb76e0..b0475c63 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -1117,13 +1117,6 @@ aes_xcbc_mac_128_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_XCBC, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1223,13 +1216,6 @@ aes_xcbc_mac_128_flush_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_XCBC, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm index ba09c887..8703865f 100644 --- a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm @@ -32,6 +32,7 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" +%include "include/clear_regs.asm" %ifndef AES_XCBC_X16 %define AES_XCBC_X16 aes_xcbc_mac_128_vaes_avx512 @@ -370,7 +371,7 @@ endstruc vmovdqu8 [icv]{k1}, xmm0 %ifdef SAFE_DATA - vpxor ymm0, ymm0 + vpxor xmm0, xmm0, xmm0 ;; Clear final block (32 bytes) vmovdqa [lane_data + _xcbc_final_block], ymm0 @@ -380,13 +381,16 @@ endstruc vmovdqa [state + _aes_xcbc_args_key_tab + round * (16*16) + idx], xmm0 %assign round (round + 1) %endrep - -%else - vzeroupper %endif %%_return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] -- GitLab From 29fbfd356704043ce6b25d2785fce4921343a474 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 14:03:29 +0000 Subject: [PATCH 265/369] avx512: [aes-cbc] move handling of safe_data/vzeroupper from the algorithmic code to the manager code --- lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm | 42 ------------------- .../mb_mgr_aes128_cbc_enc_flush_avx512.asm | 9 +++- .../mb_mgr_aes128_cbc_enc_submit_avx512.asm | 9 ++-- 3 files changed, 14 insertions(+), 46 deletions(-) diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index b0475c63..c7d0ce77 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -1041,13 +1041,6 @@ aes_cbc_enc_128_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1059,13 +1052,6 @@ aes_cbc_enc_192_vaes_avx512: FUNC_SAVE CBC_ENC 11, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1077,13 +1063,6 @@ aes_cbc_enc_256_vaes_avx512: FUNC_SAVE CBC_ENC 13, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1130,13 +1109,6 @@ aes_cbc_enc_128_flush_vaes_avx512: FUNC_SAVE CBC_ENC 9, MAC_TYPE_NONE, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1150,13 +1122,6 @@ aes_cbc_enc_192_flush_vaes_avx512: FUNC_SAVE CBC_ENC 11, MAC_TYPE_NONE, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1170,13 +1135,6 @@ aes_cbc_enc_256_flush_vaes_avx512: FUNC_SAVE CBC_ENC 13, MAC_TYPE_NONE, FLUSH FUNC_RESTORE - -%ifdef SAFE_DATA - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - ret ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm index 33ee6cbb..48dd36bf 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -31,6 +31,8 @@ %include "include/constants.asm" %include "include/reg_sizes.asm" %include "include/cet.inc" +%include "include/clear_regs.asm" + %ifndef AES_CBC_ENC_X16 %define AES_CBC_ENC_X16 aes_cbc_enc_128_flush_vaes_avx512 %define FLUSH_JOB_AES_ENC flush_job_aes128_enc_vaes_avx512 @@ -258,10 +260,15 @@ len_is_0: vmovdqa [state + _aesarg_key_tab + round * (16*16) + idx], xmm0 %assign round (round + 1) %endrep + %endif - vzeroupper return: +%ifdef SAFE_DATA + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm index 6b8b3465..98b3952d 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm @@ -31,6 +31,8 @@ %include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" +%include "include/clear_regs.asm" + %ifndef AES_CBC_ENC_X16 %define AES_CBC_ENC_X16 aes_cbc_enc_128_vaes_avx512 %define NUM_KEYS 11 @@ -252,13 +254,14 @@ len_is_0: vmovdqa [state + _aesarg_key_tab + round * (16*16) + idx], xmm0 %assign round (round + 1) %endrep - %endif return: -%ifndef SAFE_DATA +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper -%endif +%endif ;; SAFE_DATA mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] -- GitLab From adebe6ae0f859f69815dee340f1718da2d10408e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 16:51:18 +0000 Subject: [PATCH 266/369] avx512: [aes-ccm] fix AVX-SSE transition with SAFE_DATA option on --- .../mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm index 23532fad..52f2efa0 100644 --- a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -32,6 +32,7 @@ %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/memcpy.asm" +%include "include/clear_regs.asm" %ifndef AES_CBC_MAC %define AES_CBC_MAC aes128_cbc_mac_vaes_avx512 @@ -602,7 +603,9 @@ endstruc %endif ;; SAFE_DATA %%_return: -%ifndef SAFE_DATA +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper %endif mov rbx, [rsp + _gpr_save + 8*0] -- GitLab From 9ef6aa5fb9d725c22f6e305b1be84a46303d3ed4 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 16:52:37 +0000 Subject: [PATCH 267/369] avx512: [snow3g-uea2] fix for AVX-SSE transition with SAFE_DATA option on --- lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm b/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm index ee406950..d295d9ab 100644 --- a/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm +++ b/lib/avx512_t2/snow3g_uea2_by16_vaes_avx512.asm @@ -332,7 +332,9 @@ endstruc ;; Restores register contents and removes the stack frame ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %macro SNOW3G_FUNC_END 0 -%ifndef SAFE_DATA +%ifdef SAFE_DATA + clear_all_zmms_asm +%else vzeroupper %endif mov rbx, [rsp + _gpr_save + 8 * 0] -- GitLab From 09691bb4794bb5c4e887be0381431411a32a6c30 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 16:53:27 +0000 Subject: [PATCH 268/369] avx512: [aead-chacha20-poly1305] fix for AVX-SSE transition with SAFE_DATA option on --- lib/avx512_t1/poly_avx512.asm | 5 ++++- lib/include/clear_regs.asm | 2 +- lib/x86_64/chacha20_poly1305.c | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/avx512_t1/poly_avx512.asm b/lib/avx512_t1/poly_avx512.asm index ed58d554..f5c5f3ff 100644 --- a/lib/avx512_t1/poly_avx512.asm +++ b/lib/avx512_t1/poly_avx512.asm @@ -1321,7 +1321,10 @@ APPEND(%%_shuffle_blocks_, i): mov rsp, [rsp + _rsp_save] %ifdef SAFE_DATA - clear_scratch_gps_asm + clear_scratch_gps_asm + clear_all_zmms_asm +%else + vzeroupper %endif ;; SAFE_DATA %endmacro diff --git a/lib/include/clear_regs.asm b/lib/include/clear_regs.asm index 8461aa03..3df53f47 100644 --- a/lib/include/clear_regs.asm +++ b/lib/include/clear_regs.asm @@ -170,7 +170,7 @@ %assign i (i+1) %endrep %endif ; LINUX - vzeroupper + vzeroupper %endmacro ; diff --git a/lib/x86_64/chacha20_poly1305.c b/lib/x86_64/chacha20_poly1305.c index c462392d..3e52e809 100644 --- a/lib/x86_64/chacha20_poly1305.c +++ b/lib/x86_64/chacha20_poly1305.c @@ -662,6 +662,7 @@ IMB_JOB *aead_chacha20_poly1305(IMB_JOB *job, const IMB_ARCH arch, for Poly key */ submit_job_chacha20_poly_dec_avx512(job, ks + 64, len_to_gen - 64); + break; } } -- GitLab From 2f7e890c0a7ed66b019189450e8312ecd850b907 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 22 Jul 2022 17:19:07 +0000 Subject: [PATCH 269/369] build: [windows] do not generate endbranch64 (CET) and disable guard compiler settings --- lib/include/cet.inc | 5 ++++- lib/win_x64.mak | 6 ------ perf/win_x64.mak | 6 ------ test/win_x64.mak | 6 ------ 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/lib/include/cet.inc b/lib/include/cet.inc index c29730d1..b8e9dde8 100644 --- a/lib/include/cet.inc +++ b/lib/include/cet.inc @@ -29,7 +29,10 @@ %define _CET_INC_ %macro endbranch64 0 -db 0xf3, 0x0f, 0x1e, 0xfa + ;; generate enbranch64 opcode only for Linux/FreeBSD +%ifdef LINUX + db 0xf3, 0x0f, 0x1e, 0xfa +%endif %endmacro %endif ; _CET_INC_ diff --git a/lib/win_x64.mak b/lib/win_x64.mak index b31233ce..66348269 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -112,12 +112,6 @@ DAFLAGS = $(DAFLAGS) -DAESNI_EMU CC = cl -# check for CET support -!if ([$(CC) /? 2>&1 | findstr /C:"guard:cf" > nul] == 0) -DCFLAGS = $(DCFLAGS) /guard:cf -DLFLAGS = $(DLFLAGS) /CETCOMPAT /GUARD:CF /DYNAMICBASE -!endif - CFLAGS_ALL = $(EXTRA_CFLAGS) /DNO_COMPAT_IMB_API_053 /I. /Iinclude /Ino-aesni \ /nologo /Y- /W3 /WX- /Gm- /fp:precise /EHsc /Z7 diff --git a/perf/win_x64.mak b/perf/win_x64.mak index b0b864db..99d7847a 100644 --- a/perf/win_x64.mak +++ b/perf/win_x64.mak @@ -62,12 +62,6 @@ DLFLAGS = CC = cl -# check for CET support -!if ([$(CC) /? 2>&1 | findstr /C:"guard:cf" > nul] == 0) -DCFLAGS = $(DCFLAGS) /guard:cf -DLFLAGS = $(DLFLAGS) /CETCOMPAT /GUARD:CF /DYNAMICBASE -!endif - # _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure strtok() being used CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) diff --git a/test/win_x64.mak b/test/win_x64.mak index a3bd0b00..3d075b66 100644 --- a/test/win_x64.mak +++ b/test/win_x64.mak @@ -59,12 +59,6 @@ DLFLAGS = # compiler CC = cl -# check for CET support -!if ([$(CC) /? 2>&1 | findstr /C:"guard:cf" > nul] == 0) -DCFLAGS = $(DCFLAGS) /guard:cf -DLFLAGS = $(DLFLAGS) /CETCOMPAT /GUARD:CF /DYNAMICBASE -!endif - # _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure snprintf() being used CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) -- GitLab From 7a26fd18de5cfc65da4169189331b72b31f67a79 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Thu, 21 Jul 2022 08:59:23 +0000 Subject: [PATCH 270/369] update doxygen config file --- lib/api_doxygen.conf | 80 +++----------------------------------------- 1 file changed, 5 insertions(+), 75 deletions(-) diff --git a/lib/api_doxygen.conf b/lib/api_doxygen.conf index 18fb84ad..5443fd20 100644 --- a/lib/api_doxygen.conf +++ b/lib/api_doxygen.conf @@ -114,7 +114,7 @@ INLINE_INHERITED_MEMB = NO # shortest path that makes the file name unique will be used # The default value is: YES. -FULL_PATH_NAMES = YES +FULL_PATH_NAMES = NO # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand @@ -204,12 +204,6 @@ TAB_SIZE = 8 ALIASES = -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For # instance, some of the names that are used will be different. The list of all @@ -745,7 +739,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ./intel-ipsec-mb.h +INPUT = ./intel-ipsec-mb.h ../README.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -894,7 +888,7 @@ FILTER_SOURCE_PATTERNS = # (index.html). This can be useful if you have a project on for instance GitHub # and want to reuse the introduction page also for the doxygen output. -USE_MDFILE_AS_MAINPAGE = +USE_MDFILE_AS_MAINPAGE = README.md #--------------------------------------------------------------------------- # Configuration options related to source browsing @@ -982,25 +976,6 @@ USE_HTAGS = NO VERBATIM_HEADERS = YES -# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. -# Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse-libclang=ON option for CMake. -# The default value is: NO. - -CLANG_ASSISTED_PARSING = NO - -# If clang assisted parsing is enabled you can provide the compiler with command -# line options that you would normally use when invoking the compiler. Note that -# the include paths will already be set by doxygen for the files and directories -# specified with INPUT and INCLUDE_PATH. -# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. - -CLANG_OPTIONS = - #--------------------------------------------------------------------------- # Configuration options related to the alphabetical class index #--------------------------------------------------------------------------- @@ -1012,13 +987,6 @@ CLANG_OPTIONS = ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored @@ -1373,7 +1341,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -DISABLE_INDEX = NO +DISABLE_INDEX = YES # The GENERATE_TREEVIEW tag is used to specify whether a tree-like index # structure should be generated to display hierarchical information. If the tag @@ -1390,7 +1358,7 @@ DISABLE_INDEX = NO # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. -GENERATE_TREEVIEW = NO +GENERATE_TREEVIEW = YES # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. @@ -1714,16 +1682,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # http://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -1796,16 +1754,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -1895,15 +1843,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2082,15 +2021,6 @@ EXTERNAL_PAGES = YES # Configuration options related to the dot tool #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - # You can include diagrams made with dia in doxygen documentation. Doxygen will # then run dia to produce the diagram and insert it in the documentation. The # DIA_PATH tag allows you to specify the directory where the dia binary resides. -- GitLab From 6b95b1f4c557e95bf8301f4fc08647052d0595bc Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 22 Jul 2022 14:34:34 +0000 Subject: [PATCH 271/369] add links between web page and README --- README | 27 +++++++++++++++++++-------- README.md | 24 ++++++++++++++++-------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README b/README index 2a7330dc..fe655f6c 100644 --- a/README +++ b/README @@ -12,8 +12,8 @@ The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. -The library is used as software crypto provider in DPDK, Intel(R) QAT -Engine and FD.io. +The library is is hosted on GitHub and is used as software crypto +provider in DPDK, Intel(R) QAT Engine and FD.io. Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The @@ -312,7 +312,18 @@ https://github.com/intel/intel-ipsec-mb/releases -5. COMPILATION +5. DOCUMENTATION + + +Full documentation can be found at: +https://intel.github.io/intel-ipsec-mb + +To generate documentation locally, run: +> make doxy + + + +6. COMPILATION Linux (64-bit only) @@ -451,7 +462,7 @@ For more build options and their explanation run: -6. SECURITY CONSIDERATIONS & OPTIONS FOR INCREASED SECURITY +7. SECURITY CONSIDERATIONS & OPTIONS FOR INCREASED SECURITY Security Considerations @@ -558,7 +569,7 @@ details, trade offs and mitigations of using shorter tag sizes. -7. INSTALLATION +8. INSTALLATION Linux (64-bit only) @@ -623,7 +634,7 @@ install it using SHARED=n option: -8. BACKWARDS COMPATIBILITY +9. BACKWARDS COMPATIBILITY In version 0.54, some symbols have been renamed to avoid too generic @@ -643,7 +654,7 @@ which will not export the old symbols. -9. DISCLAIMER (ZUC, KASUMI, SNOW3G) +10. DISCLAIMER (ZUC, KASUMI, SNOW3G) Please note that cryptographic material, such as ciphering algorithms, @@ -658,7 +669,7 @@ https://www.etsi.org/security-algorithms-and-codes/cellular-algorithm-licences -10. LEGAL DISCLAIMER +11. LEGAL DISCLAIMER THIS SOFTWARE IS PROVIDED BY INTEL“AS IS”. NO LICENSE, EXPRESS OR diff --git a/README.md b/README.md index d7780a7d..c9861f99 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. -The library is used as software crypto provider in [DPDK](https://www.dpdk.org/), -[Intel(R) QAT Engine](https://github.com/intel/QAT_Engine) and [FD.io](https://fd.io/). +The library is is hosted on [GitHub](https://github.com/intel/intel-ipsec-mb) and is used as software crypto +provider in [DPDK](https://www.dpdk.org/), [Intel(R) QAT Engine](https://github.com/intel/QAT_Engine) +and [FD.io](https://fd.io/). Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The library can also be used directly @@ -299,8 +300,15 @@ Code taken from the tip of the master branch should not be considered fit for pr Refer to the releases tab for stable code versions: https://github.com/intel/intel-ipsec-mb/releases +5\. Documentation +=============== + +Full documentation can be found at: https://intel.github.io/intel-ipsec-mb + +To generate documentation locally, run: +`> make doxy` -5\. Compilation +6\. Compilation =============== Linux (64-bit only) @@ -431,7 +439,7 @@ Build with debugging information: For more build options and their explanation run: `> gmake help` -6\. Security Considerations & Options for Increased Security +7\. Security Considerations & Options for Increased Security ============================================================ ### Security Considerations @@ -521,7 +529,7 @@ The library GCM and GMAC implementation provides flexibility as to tag size sele As explained in [NIST Special Publication 800-38D](https://csrc.nist.gov/publications/detail/sp/800-38d/final) section 5.2.1.2 and Appendix C, using tag sizes shorter than 96 bits can be insecure. Please refer to the aforementioned sections to understand the details, trade offs and mitigations of using shorter tag sizes. -7\. Installation +8\. Installation ================ Linux (64-bit only) @@ -578,7 +586,7 @@ If there is no need to run ldconfig at install stage please use NOLDCONFIG=y opt If library was compiled as an archive (not a default option) then install it using SHARED=n option: `> sudo gmake install SHARED=n` -8\. Backwards compatibility +9\. Backwards compatibility =========================== In version 0.54, some symbols have been renamed to avoid too generic names (such as cipher modes or @@ -594,7 +602,7 @@ For applications which face symbol conflicts due to these old generic names, they should be compiled with the flag -DNO_COMPAT_IMB_API_053, which will not export the old symbols. -9\. Disclaimer (ZUC, KASUMI, SNOW3G) +10\. Disclaimer (ZUC, KASUMI, SNOW3G) ==================================== Please note that cryptographic material, such as ciphering algorithms, may be @@ -606,7 +614,7 @@ For more details please see: - GSMA https://www.gsma.com/security/security-algorithms/ - ETSI https://www.etsi.org/security-algorithms-and-codes/cellular-algorithm-licences -10\. Legal Disclaimer +11\. Legal Disclaimer ==================== THIS SOFTWARE IS PROVIDED BY INTEL"AS IS". NO LICENSE, EXPRESS OR -- GitLab From 2844a737a7024885d11db0da1e3e862ac48b4182 Mon Sep 17 00:00:00 2001 From: Emma Urquhart Date: Fri, 22 Jul 2022 14:38:15 +0000 Subject: [PATCH 272/369] move doxygen files to docs directory --- README | 17 +- README.md | 15 +- docs/README_8md.html | 108 + docs/annotated.html | 120 + docs/annotated_dup.js | 12 + docs/bc_s.png | Bin 0 -> 676 bytes docs/bdwn.png | Bin 0 -> 147 bytes docs/classes.html | 125 + docs/closed.png | Bin 0 -> 132 bytes docs/doc.png | Bin 0 -> 746 bytes docs/doxygen.css | 1841 ++ docs/doxygen.png | Bin 0 -> 3779 bytes docs/doxygen.svg | 26 + docs/dynsections.js | 121 + docs/files.html | 112 + docs/files_dup.js | 4 + docs/folderclosed.png | Bin 0 -> 616 bytes docs/folderopen.png | Bin 0 -> 597 bytes docs/functions.html | 120 + docs/functions_a.html | 126 + docs/functions_c.html | 140 + docs/functions_d.html | 121 + docs/functions_dup.js | 26 + docs/functions_e.html | 117 + docs/functions_f.html | 119 + docs/functions_g.html | 155 + docs/functions_h.html | 123 + docs/functions_i.html | 114 + docs/functions_j.html | 109 + docs/functions_k.html | 117 + docs/functions_l.html | 112 + docs/functions_m.html | 114 + docs/functions_n.html | 111 + docs/functions_o.html | 110 + docs/functions_p.html | 113 + docs/functions_q.html | 109 + docs/functions_r.html | 111 + docs/functions_s.html | 154 + docs/functions_u.html | 112 + docs/functions_v.html | 109 + docs/functions_vars.html | 120 + docs/functions_vars.js | 26 + docs/functions_vars_a.html | 126 + docs/functions_vars_c.html | 140 + docs/functions_vars_d.html | 121 + docs/functions_vars_e.html | 117 + docs/functions_vars_f.html | 119 + docs/functions_vars_g.html | 155 + docs/functions_vars_h.html | 123 + docs/functions_vars_i.html | 114 + docs/functions_vars_j.html | 109 + docs/functions_vars_k.html | 117 + docs/functions_vars_l.html | 112 + docs/functions_vars_m.html | 114 + docs/functions_vars_n.html | 111 + docs/functions_vars_o.html | 110 + docs/functions_vars_p.html | 113 + docs/functions_vars_q.html | 109 + docs/functions_vars_r.html | 111 + docs/functions_vars_s.html | 154 + docs/functions_vars_u.html | 112 + docs/functions_vars_v.html | 109 + docs/functions_vars_x.html | 110 + docs/functions_vars_z.html | 113 + docs/functions_x.html | 110 + docs/functions_z.html | 113 + docs/globals.html | 109 + docs/globals_a.html | 238 + docs/globals_c.html | 113 + docs/globals_d.html | 111 + docs/globals_defs.html | 303 + docs/globals_dup.js | 17 + docs/globals_enum.html | 115 + docs/globals_eval.html | 253 + docs/globals_f.html | 114 + docs/globals_func.html | 226 + docs/globals_func.js | 13 + docs/globals_func_d.html | 109 + docs/globals_func_f.html | 113 + docs/globals_func_g.html | 116 + docs/globals_func_i.html | 121 + docs/globals_func_k.html | 110 + docs/globals_func_m.html | 112 + docs/globals_func_q.html | 112 + docs/globals_func_s.html | 158 + docs/globals_func_z.html | 110 + docs/globals_g.html | 119 + docs/globals_h.html | 112 + docs/globals_i.html | 458 + docs/globals_k.html | 124 + docs/globals_m.html | 112 + docs/globals_q.html | 113 + docs/globals_s.html | 174 + docs/globals_type.html | 217 + docs/globals_x.html | 109 + docs/globals_z.html | 115 + docs/index.html | 618 +- docs/intel-ipsec-mb_8h.html | 19646 ++++++++++++++++ docs/intel-ipsec-mb_8h.js | 630 + docs/intel-ipsec-mb_8h_source.html | 2867 +++ docs/jquery.js | 35 + docs/md__home_eurquhar_ipsec-mb_README.html | 566 + docs/menu.js | 50 + docs/menudata.js | 126 + docs/nav_f.png | Bin 0 -> 153 bytes docs/nav_g.png | Bin 0 -> 95 bytes docs/nav_h.png | Bin 0 -> 98 bytes docs/navtree.css | 147 + docs/navtree.js | 549 + docs/navtreedata.js | 81 + docs/navtreeindex0.js | 253 + docs/navtreeindex1.js | 253 + docs/navtreeindex2.js | 253 + docs/navtreeindex3.js | 236 + docs/open.png | Bin 0 -> 123 bytes docs/pages.html | 81 + docs/resize.js | 156 + docs/search/all_0.html | 37 + docs/search/all_0.js | 16 + docs/search/all_1.html | 37 + docs/search/all_1.js | 151 + docs/search/all_10.html | 37 + docs/search/all_10.js | 9 + docs/search/all_11.html | 37 + docs/search/all_11.js | 7 + docs/search/all_12.html | 37 + docs/search/all_12.js | 116 + docs/search/all_13.html | 37 + docs/search/all_13.js | 7 + docs/search/all_14.html | 37 + docs/search/all_14.js | 4 + docs/search/all_15.html | 37 + docs/search/all_15.js | 6 + docs/search/all_16.html | 37 + docs/search/all_16.js | 15 + docs/search/all_2.html | 37 + docs/search/all_2.js | 41 + docs/search/all_3.html | 37 + docs/search/all_3.js | 19 + docs/search/all_4.html | 37 + docs/search/all_4.js | 12 + docs/search/all_5.html | 37 + docs/search/all_5.js | 20 + docs/search/all_6.html | 37 + docs/search/all_6.js | 63 + docs/search/all_7.html | 37 + docs/search/all_7.js | 22 + docs/search/all_8.html | 37 + docs/search/all_8.js | 363 + docs/search/all_9.html | 37 + docs/search/all_9.js | 4 + docs/search/all_a.html | 37 + docs/search/all_a.js | 29 + docs/search/all_b.html | 37 + docs/search/all_b.js | 7 + docs/search/all_c.html | 37 + docs/search/all_c.js | 13 + docs/search/all_d.html | 37 + docs/search/all_d.js | 6 + docs/search/all_e.html | 37 + docs/search/all_e.js | 5 + docs/search/all_f.html | 37 + docs/search/all_f.js | 8 + docs/search/classes_0.html | 37 + docs/search/classes_0.js | 4 + docs/search/classes_1.html | 37 + docs/search/classes_1.js | 5 + docs/search/classes_2.html | 37 + docs/search/classes_2.js | 7 + docs/search/classes_3.html | 37 + docs/search/classes_3.js | 4 + docs/search/classes_4.html | 37 + docs/search/classes_4.js | 4 + docs/search/close.png | Bin 0 -> 273 bytes docs/search/close.svg | 31 + docs/search/defines_0.html | 37 + docs/search/defines_0.js | 4 + docs/search/defines_1.html | 37 + docs/search/defines_1.js | 4 + docs/search/defines_2.html | 37 + docs/search/defines_2.js | 183 + docs/search/defines_3.html | 37 + docs/search/defines_3.js | 4 + docs/search/enums_0.html | 37 + docs/search/enums_0.js | 12 + docs/search/enumvalues_0.html | 37 + docs/search/enumvalues_0.js | 148 + docs/search/files_0.html | 37 + docs/search/files_0.js | 4 + docs/search/files_1.html | 37 + docs/search/files_1.js | 4 + docs/search/functions_0.html | 37 + docs/search/functions_0.js | 121 + docs/search/functions_1.html | 37 + docs/search/functions_1.js | 4 + docs/search/functions_2.html | 37 + docs/search/functions_2.js | 8 + docs/search/functions_3.html | 37 + docs/search/functions_3.js | 11 + docs/search/functions_4.html | 37 + docs/search/functions_4.js | 16 + docs/search/functions_5.html | 37 + docs/search/functions_5.js | 5 + docs/search/functions_6.html | 37 + docs/search/functions_6.js | 7 + docs/search/functions_7.html | 37 + docs/search/functions_7.js | 7 + docs/search/functions_8.html | 37 + docs/search/functions_8.js | 53 + docs/search/functions_9.html | 37 + docs/search/functions_9.js | 5 + docs/search/mag_sel.png | Bin 0 -> 465 bytes docs/search/mag_sel.svg | 74 + docs/search/nomatches.html | 13 + docs/search/pages_0.html | 37 + docs/search/pages_0.js | 4 + docs/search/search.css | 274 + docs/search/search.js | 793 + docs/search/search_l.png | Bin 0 -> 567 bytes docs/search/search_m.png | Bin 0 -> 158 bytes docs/search/search_r.png | Bin 0 -> 553 bytes docs/search/searchdata.js | 42 + docs/search/typedefs_0.html | 37 + docs/search/typedefs_0.js | 15 + docs/search/typedefs_1.html | 37 + docs/search/typedefs_1.js | 8 + docs/search/typedefs_2.html | 37 + docs/search/typedefs_2.js | 4 + docs/search/typedefs_3.html | 37 + docs/search/typedefs_3.js | 4 + docs/search/typedefs_4.html | 37 + docs/search/typedefs_4.js | 6 + docs/search/typedefs_5.html | 37 + docs/search/typedefs_5.js | 7 + docs/search/typedefs_6.html | 37 + docs/search/typedefs_6.js | 6 + docs/search/typedefs_7.html | 37 + docs/search/typedefs_7.js | 16 + docs/search/typedefs_8.html | 37 + docs/search/typedefs_8.js | 4 + docs/search/typedefs_9.html | 37 + docs/search/typedefs_9.js | 19 + docs/search/typedefs_a.html | 37 + docs/search/typedefs_a.js | 4 + docs/search/typedefs_b.html | 37 + docs/search/typedefs_b.js | 8 + docs/search/variables_0.html | 37 + docs/search/variables_0.js | 15 + docs/search/variables_1.html | 37 + docs/search/variables_1.js | 21 + docs/search/variables_10.html | 37 + docs/search/variables_10.js | 4 + docs/search/variables_11.html | 37 + docs/search/variables_11.js | 6 + docs/search/variables_12.html | 37 + docs/search/variables_12.js | 49 + docs/search/variables_13.html | 37 + docs/search/variables_13.js | 7 + docs/search/variables_14.html | 37 + docs/search/variables_14.js | 4 + docs/search/variables_15.html | 37 + docs/search/variables_15.js | 5 + docs/search/variables_16.html | 37 + docs/search/variables_16.js | 8 + docs/search/variables_2.html | 37 + docs/search/variables_2.js | 35 + docs/search/variables_3.html | 37 + docs/search/variables_3.js | 16 + docs/search/variables_4.html | 37 + docs/search/variables_4.js | 12 + docs/search/variables_5.html | 37 + docs/search/variables_5.js | 14 + docs/search/variables_6.html | 37 + docs/search/variables_6.js | 50 + docs/search/variables_7.html | 37 + docs/search/variables_7.js | 18 + docs/search/variables_8.html | 37 + docs/search/variables_8.js | 9 + docs/search/variables_9.html | 37 + docs/search/variables_9.js | 4 + docs/search/variables_a.html | 37 + docs/search/variables_a.js | 12 + docs/search/variables_b.html | 37 + docs/search/variables_b.js | 7 + docs/search/variables_c.html | 37 + docs/search/variables_c.js | 9 + docs/search/variables_d.html | 37 + docs/search/variables_d.js | 6 + docs/search/variables_e.html | 37 + docs/search/variables_e.js | 5 + docs/search/variables_f.html | 37 + docs/search/variables_f.js | 8 + docs/splitbar.png | Bin 0 -> 314 bytes docs/structIMB__JOB.html | 1383 ++ docs/structIMB__JOB.js | 69 + docs/structIMB__MGR.html | 2720 +++ docs/structIMB__MGR.js | 165 + docs/structIMB__SGL__IOV.html | 172 + docs/structIMB__SGL__IOV.js | 6 + ...ructchacha20__poly1305__context__data.html | 295 + ...structchacha20__poly1305__context__data.js | 13 + docs/structgcm__context__data.html | 238 + docs/structgcm__context__data.js | 10 + docs/structgcm__key__data.html | 251 + docs/structgcm__key__data.js | 10 + docs/structimb__uint128__t.html | 152 + docs/structimb__uint128__t.js | 5 + docs/structkasumi__key__sched__s.html | 156 + docs/structkasumi__key__sched__s.js | 5 + docs/structsnow3g__key__schedule__s.html | 138 + docs/structsnow3g__key__schedule__s.js | 4 + docs/sync_off.png | Bin 0 -> 853 bytes docs/sync_on.png | Bin 0 -> 845 bytes docs/tab_a.png | Bin 0 -> 142 bytes docs/tab_b.png | Bin 0 -> 169 bytes docs/tab_h.png | Bin 0 -> 177 bytes docs/tab_s.png | Bin 0 -> 184 bytes docs/tabs.css | 1 + lib/api_doxygen.conf | 2 +- 319 files changed, 50923 insertions(+), 33 deletions(-) create mode 100644 docs/README_8md.html create mode 100644 docs/annotated.html create mode 100644 docs/annotated_dup.js create mode 100644 docs/bc_s.png create mode 100644 docs/bdwn.png create mode 100644 docs/classes.html create mode 100644 docs/closed.png create mode 100644 docs/doc.png create mode 100644 docs/doxygen.css create mode 100644 docs/doxygen.png create mode 100644 docs/doxygen.svg create mode 100644 docs/dynsections.js create mode 100644 docs/files.html create mode 100644 docs/files_dup.js create mode 100644 docs/folderclosed.png create mode 100644 docs/folderopen.png create mode 100644 docs/functions.html create mode 100644 docs/functions_a.html create mode 100644 docs/functions_c.html create mode 100644 docs/functions_d.html create mode 100644 docs/functions_dup.js create mode 100644 docs/functions_e.html create mode 100644 docs/functions_f.html create mode 100644 docs/functions_g.html create mode 100644 docs/functions_h.html create mode 100644 docs/functions_i.html create mode 100644 docs/functions_j.html create mode 100644 docs/functions_k.html create mode 100644 docs/functions_l.html create mode 100644 docs/functions_m.html create mode 100644 docs/functions_n.html create mode 100644 docs/functions_o.html create mode 100644 docs/functions_p.html create mode 100644 docs/functions_q.html create mode 100644 docs/functions_r.html create mode 100644 docs/functions_s.html create mode 100644 docs/functions_u.html create mode 100644 docs/functions_v.html create mode 100644 docs/functions_vars.html create mode 100644 docs/functions_vars.js create mode 100644 docs/functions_vars_a.html create mode 100644 docs/functions_vars_c.html create mode 100644 docs/functions_vars_d.html create mode 100644 docs/functions_vars_e.html create mode 100644 docs/functions_vars_f.html create mode 100644 docs/functions_vars_g.html create mode 100644 docs/functions_vars_h.html create mode 100644 docs/functions_vars_i.html create mode 100644 docs/functions_vars_j.html create mode 100644 docs/functions_vars_k.html create mode 100644 docs/functions_vars_l.html create mode 100644 docs/functions_vars_m.html create mode 100644 docs/functions_vars_n.html create mode 100644 docs/functions_vars_o.html create mode 100644 docs/functions_vars_p.html create mode 100644 docs/functions_vars_q.html create mode 100644 docs/functions_vars_r.html create mode 100644 docs/functions_vars_s.html create mode 100644 docs/functions_vars_u.html create mode 100644 docs/functions_vars_v.html create mode 100644 docs/functions_vars_x.html create mode 100644 docs/functions_vars_z.html create mode 100644 docs/functions_x.html create mode 100644 docs/functions_z.html create mode 100644 docs/globals.html create mode 100644 docs/globals_a.html create mode 100644 docs/globals_c.html create mode 100644 docs/globals_d.html create mode 100644 docs/globals_defs.html create mode 100644 docs/globals_dup.js create mode 100644 docs/globals_enum.html create mode 100644 docs/globals_eval.html create mode 100644 docs/globals_f.html create mode 100644 docs/globals_func.html create mode 100644 docs/globals_func.js create mode 100644 docs/globals_func_d.html create mode 100644 docs/globals_func_f.html create mode 100644 docs/globals_func_g.html create mode 100644 docs/globals_func_i.html create mode 100644 docs/globals_func_k.html create mode 100644 docs/globals_func_m.html create mode 100644 docs/globals_func_q.html create mode 100644 docs/globals_func_s.html create mode 100644 docs/globals_func_z.html create mode 100644 docs/globals_g.html create mode 100644 docs/globals_h.html create mode 100644 docs/globals_i.html create mode 100644 docs/globals_k.html create mode 100644 docs/globals_m.html create mode 100644 docs/globals_q.html create mode 100644 docs/globals_s.html create mode 100644 docs/globals_type.html create mode 100644 docs/globals_x.html create mode 100644 docs/globals_z.html create mode 100644 docs/intel-ipsec-mb_8h.html create mode 100644 docs/intel-ipsec-mb_8h.js create mode 100644 docs/intel-ipsec-mb_8h_source.html create mode 100644 docs/jquery.js create mode 100644 docs/md__home_eurquhar_ipsec-mb_README.html create mode 100644 docs/menu.js create mode 100644 docs/menudata.js create mode 100644 docs/nav_f.png create mode 100644 docs/nav_g.png create mode 100644 docs/nav_h.png create mode 100644 docs/navtree.css create mode 100644 docs/navtree.js create mode 100644 docs/navtreedata.js create mode 100644 docs/navtreeindex0.js create mode 100644 docs/navtreeindex1.js create mode 100644 docs/navtreeindex2.js create mode 100644 docs/navtreeindex3.js create mode 100644 docs/open.png create mode 100644 docs/pages.html create mode 100644 docs/resize.js create mode 100644 docs/search/all_0.html create mode 100644 docs/search/all_0.js create mode 100644 docs/search/all_1.html create mode 100644 docs/search/all_1.js create mode 100644 docs/search/all_10.html create mode 100644 docs/search/all_10.js create mode 100644 docs/search/all_11.html create mode 100644 docs/search/all_11.js create mode 100644 docs/search/all_12.html create mode 100644 docs/search/all_12.js create mode 100644 docs/search/all_13.html create mode 100644 docs/search/all_13.js create mode 100644 docs/search/all_14.html create mode 100644 docs/search/all_14.js create mode 100644 docs/search/all_15.html create mode 100644 docs/search/all_15.js create mode 100644 docs/search/all_16.html create mode 100644 docs/search/all_16.js create mode 100644 docs/search/all_2.html create mode 100644 docs/search/all_2.js create mode 100644 docs/search/all_3.html create mode 100644 docs/search/all_3.js create mode 100644 docs/search/all_4.html create mode 100644 docs/search/all_4.js create mode 100644 docs/search/all_5.html create mode 100644 docs/search/all_5.js create mode 100644 docs/search/all_6.html create mode 100644 docs/search/all_6.js create mode 100644 docs/search/all_7.html create mode 100644 docs/search/all_7.js create mode 100644 docs/search/all_8.html create mode 100644 docs/search/all_8.js create mode 100644 docs/search/all_9.html create mode 100644 docs/search/all_9.js create mode 100644 docs/search/all_a.html create mode 100644 docs/search/all_a.js create mode 100644 docs/search/all_b.html create mode 100644 docs/search/all_b.js create mode 100644 docs/search/all_c.html create mode 100644 docs/search/all_c.js create mode 100644 docs/search/all_d.html create mode 100644 docs/search/all_d.js create mode 100644 docs/search/all_e.html create mode 100644 docs/search/all_e.js create mode 100644 docs/search/all_f.html create mode 100644 docs/search/all_f.js create mode 100644 docs/search/classes_0.html create mode 100644 docs/search/classes_0.js create mode 100644 docs/search/classes_1.html create mode 100644 docs/search/classes_1.js create mode 100644 docs/search/classes_2.html create mode 100644 docs/search/classes_2.js create mode 100644 docs/search/classes_3.html create mode 100644 docs/search/classes_3.js create mode 100644 docs/search/classes_4.html create mode 100644 docs/search/classes_4.js create mode 100644 docs/search/close.png create mode 100644 docs/search/close.svg create mode 100644 docs/search/defines_0.html create mode 100644 docs/search/defines_0.js create mode 100644 docs/search/defines_1.html create mode 100644 docs/search/defines_1.js create mode 100644 docs/search/defines_2.html create mode 100644 docs/search/defines_2.js create mode 100644 docs/search/defines_3.html create mode 100644 docs/search/defines_3.js create mode 100644 docs/search/enums_0.html create mode 100644 docs/search/enums_0.js create mode 100644 docs/search/enumvalues_0.html create mode 100644 docs/search/enumvalues_0.js create mode 100644 docs/search/files_0.html create mode 100644 docs/search/files_0.js create mode 100644 docs/search/files_1.html create mode 100644 docs/search/files_1.js create mode 100644 docs/search/functions_0.html create mode 100644 docs/search/functions_0.js create mode 100644 docs/search/functions_1.html create mode 100644 docs/search/functions_1.js create mode 100644 docs/search/functions_2.html create mode 100644 docs/search/functions_2.js create mode 100644 docs/search/functions_3.html create mode 100644 docs/search/functions_3.js create mode 100644 docs/search/functions_4.html create mode 100644 docs/search/functions_4.js create mode 100644 docs/search/functions_5.html create mode 100644 docs/search/functions_5.js create mode 100644 docs/search/functions_6.html create mode 100644 docs/search/functions_6.js create mode 100644 docs/search/functions_7.html create mode 100644 docs/search/functions_7.js create mode 100644 docs/search/functions_8.html create mode 100644 docs/search/functions_8.js create mode 100644 docs/search/functions_9.html create mode 100644 docs/search/functions_9.js create mode 100644 docs/search/mag_sel.png create mode 100644 docs/search/mag_sel.svg create mode 100644 docs/search/nomatches.html create mode 100644 docs/search/pages_0.html create mode 100644 docs/search/pages_0.js create mode 100644 docs/search/search.css create mode 100644 docs/search/search.js create mode 100644 docs/search/search_l.png create mode 100644 docs/search/search_m.png create mode 100644 docs/search/search_r.png create mode 100644 docs/search/searchdata.js create mode 100644 docs/search/typedefs_0.html create mode 100644 docs/search/typedefs_0.js create mode 100644 docs/search/typedefs_1.html create mode 100644 docs/search/typedefs_1.js create mode 100644 docs/search/typedefs_2.html create mode 100644 docs/search/typedefs_2.js create mode 100644 docs/search/typedefs_3.html create mode 100644 docs/search/typedefs_3.js create mode 100644 docs/search/typedefs_4.html create mode 100644 docs/search/typedefs_4.js create mode 100644 docs/search/typedefs_5.html create mode 100644 docs/search/typedefs_5.js create mode 100644 docs/search/typedefs_6.html create mode 100644 docs/search/typedefs_6.js create mode 100644 docs/search/typedefs_7.html create mode 100644 docs/search/typedefs_7.js create mode 100644 docs/search/typedefs_8.html create mode 100644 docs/search/typedefs_8.js create mode 100644 docs/search/typedefs_9.html create mode 100644 docs/search/typedefs_9.js create mode 100644 docs/search/typedefs_a.html create mode 100644 docs/search/typedefs_a.js create mode 100644 docs/search/typedefs_b.html create mode 100644 docs/search/typedefs_b.js create mode 100644 docs/search/variables_0.html create mode 100644 docs/search/variables_0.js create mode 100644 docs/search/variables_1.html create mode 100644 docs/search/variables_1.js create mode 100644 docs/search/variables_10.html create mode 100644 docs/search/variables_10.js create mode 100644 docs/search/variables_11.html create mode 100644 docs/search/variables_11.js create mode 100644 docs/search/variables_12.html create mode 100644 docs/search/variables_12.js create mode 100644 docs/search/variables_13.html create mode 100644 docs/search/variables_13.js create mode 100644 docs/search/variables_14.html create mode 100644 docs/search/variables_14.js create mode 100644 docs/search/variables_15.html create mode 100644 docs/search/variables_15.js create mode 100644 docs/search/variables_16.html create mode 100644 docs/search/variables_16.js create mode 100644 docs/search/variables_2.html create mode 100644 docs/search/variables_2.js create mode 100644 docs/search/variables_3.html create mode 100644 docs/search/variables_3.js create mode 100644 docs/search/variables_4.html create mode 100644 docs/search/variables_4.js create mode 100644 docs/search/variables_5.html create mode 100644 docs/search/variables_5.js create mode 100644 docs/search/variables_6.html create mode 100644 docs/search/variables_6.js create mode 100644 docs/search/variables_7.html create mode 100644 docs/search/variables_7.js create mode 100644 docs/search/variables_8.html create mode 100644 docs/search/variables_8.js create mode 100644 docs/search/variables_9.html create mode 100644 docs/search/variables_9.js create mode 100644 docs/search/variables_a.html create mode 100644 docs/search/variables_a.js create mode 100644 docs/search/variables_b.html create mode 100644 docs/search/variables_b.js create mode 100644 docs/search/variables_c.html create mode 100644 docs/search/variables_c.js create mode 100644 docs/search/variables_d.html create mode 100644 docs/search/variables_d.js create mode 100644 docs/search/variables_e.html create mode 100644 docs/search/variables_e.js create mode 100644 docs/search/variables_f.html create mode 100644 docs/search/variables_f.js create mode 100644 docs/splitbar.png create mode 100644 docs/structIMB__JOB.html create mode 100644 docs/structIMB__JOB.js create mode 100644 docs/structIMB__MGR.html create mode 100644 docs/structIMB__MGR.js create mode 100644 docs/structIMB__SGL__IOV.html create mode 100644 docs/structIMB__SGL__IOV.js create mode 100644 docs/structchacha20__poly1305__context__data.html create mode 100644 docs/structchacha20__poly1305__context__data.js create mode 100644 docs/structgcm__context__data.html create mode 100644 docs/structgcm__context__data.js create mode 100644 docs/structgcm__key__data.html create mode 100644 docs/structgcm__key__data.js create mode 100644 docs/structimb__uint128__t.html create mode 100644 docs/structimb__uint128__t.js create mode 100644 docs/structkasumi__key__sched__s.html create mode 100644 docs/structkasumi__key__sched__s.js create mode 100644 docs/structsnow3g__key__schedule__s.html create mode 100644 docs/structsnow3g__key__schedule__s.js create mode 100644 docs/sync_off.png create mode 100644 docs/sync_on.png create mode 100644 docs/tab_a.png create mode 100644 docs/tab_b.png create mode 100644 docs/tab_h.png create mode 100644 docs/tab_s.png create mode 100644 docs/tabs.css diff --git a/README b/README index fe655f6c..af997511 100644 --- a/README +++ b/README @@ -12,8 +12,8 @@ The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. -The library is is hosted on GitHub and is used as software crypto -provider in DPDK, Intel(R) QAT Engine and FD.io. +The library is hosted on GitHub and is used as software crypto provider +in DPDK, Intel(R) QAT Engine and FD.io. Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The @@ -37,12 +37,13 @@ CONTENTS 2. Processor Extensions 3. Recommendations 4. Package Content -5. Compilation -6. Security Considerations & Options for Increased Security -7. Installation -8. Backwards compatibility -9. Disclaimer (ZUC, KASUMI, SNOW3G) -10. Legal Disclaimer +5. Documentation +6. Compilation +7. Security Considerations & Options for Increased Security +8. Installation +9. Backwards compatibility +10. Disclaimer (ZUC, KASUMI, SNOW3G) +11. Legal Disclaimer diff --git a/README.md b/README.md index c9861f99..32a1ec0d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. -The library is is hosted on [GitHub](https://github.com/intel/intel-ipsec-mb) and is used as software crypto +The library is hosted on [GitHub](https://github.com/intel/intel-ipsec-mb) and is used as software crypto provider in [DPDK](https://www.dpdk.org/), [Intel(R) QAT Engine](https://github.com/intel/QAT_Engine) and [FD.io](https://fd.io/). @@ -35,12 +35,13 @@ Contents 2. Processor Extensions 3. Recommendations 4. Package Content -5. Compilation -6. Security Considerations & Options for Increased Security -7. Installation -8. Backwards compatibility -9. Disclaimer (ZUC, KASUMI, SNOW3G) -10. Legal Disclaimer +5. Documentation +6. Compilation +7. Security Considerations & Options for Increased Security +8. Installation +9. Backwards compatibility +10. Disclaimer (ZUC, KASUMI, SNOW3G) +11. Legal Disclaimer 1\. Overview ============ diff --git a/docs/README_8md.html b/docs/README_8md.html new file mode 100644 index 00000000..a3121304 --- /dev/null +++ b/docs/README_8md.html @@ -0,0 +1,108 @@ + + + + + + + +intel-ipsec-mb: README.md File Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
README.md File Reference
+
+
+
+
+ + + + diff --git a/docs/annotated.html b/docs/annotated.html new file mode 100644 index 00000000..f4934ac2 --- /dev/null +++ b/docs/annotated.html @@ -0,0 +1,120 @@ + + + + + + + +intel-ipsec-mb: Data Structures + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Data Structures
+
+
+
Here are the data structures with brief descriptions:
+ + + + + + + + + + +
 Cchacha20_poly1305_context_dataHolds Chacha20-Poly1305 operation context
 Cgcm_context_dataHolds GCM operation context
 Cgcm_key_dataHolds intermediate key data needed to improve performance
 CIMB_JOB
 CIMB_MGR
 CIMB_SGL_IOV
 Cimb_uint128_t
 Ckasumi_key_sched_s
 Csnow3g_key_schedule_s
+
+
+
+ + + + diff --git a/docs/annotated_dup.js b/docs/annotated_dup.js new file mode 100644 index 00000000..e9aafff2 --- /dev/null +++ b/docs/annotated_dup.js @@ -0,0 +1,12 @@ +var annotated_dup = +[ + [ "chacha20_poly1305_context_data", "structchacha20__poly1305__context__data.html", "structchacha20__poly1305__context__data" ], + [ "gcm_context_data", "structgcm__context__data.html", "structgcm__context__data" ], + [ "gcm_key_data", "structgcm__key__data.html", "structgcm__key__data" ], + [ "IMB_JOB", "structIMB__JOB.html", "structIMB__JOB" ], + [ "IMB_MGR", "structIMB__MGR.html", "structIMB__MGR" ], + [ "IMB_SGL_IOV", "structIMB__SGL__IOV.html", "structIMB__SGL__IOV" ], + [ "imb_uint128_t", "structimb__uint128__t.html", "structimb__uint128__t" ], + [ "kasumi_key_sched_s", "structkasumi__key__sched__s.html", "structkasumi__key__sched__s" ], + [ "snow3g_key_schedule_s", "structsnow3g__key__schedule__s.html", "structsnow3g__key__schedule__s" ] +]; \ No newline at end of file diff --git a/docs/bc_s.png b/docs/bc_s.png new file mode 100644 index 0000000000000000000000000000000000000000..224b29aa9847d5a4b3902efd602b7ddf7d33e6c2 GIT binary patch literal 676 zcmV;V0$crwP)y__>=_9%My z{n931IS})GlGUF8K#6VIbs%684A^L3@%PlP2>_sk`UWPq@f;rU*V%rPy_ekbhXT&s z(GN{DxFv}*vZp`F>S!r||M`I*nOwwKX+BC~3P5N3-)Y{65c;ywYiAh-1*hZcToLHK ztpl1xomJ+Yb}K(cfbJr2=GNOnT!UFA7Vy~fBz8?J>XHsbZoDad^8PxfSa0GDgENZS zuLCEqzb*xWX2CG*b&5IiO#NzrW*;`VC9455M`o1NBh+(k8~`XCEEoC1Ybwf;vr4K3 zg|EB<07?SOqHp9DhLpS&bzgo70I+ghB_#)K7H%AMU3v}xuyQq9&Bm~++VYhF09a+U zl7>n7Jjm$K#b*FONz~fj;I->Bf;ule1prFN9FovcDGBkpg>)O*-}eLnC{6oZHZ$o% zXKW$;0_{8hxHQ>l;_*HATI(`7t#^{$(zLe}h*mqwOc*nRY9=?Sx4OOeVIfI|0V(V2 zBrW#G7Ss9wvzr@>H*`r>zE z+e8bOBgqIgldUJlG(YUDviMB`9+DH8n-s9SXRLyJHO1!=wY^79WYZMTa(wiZ!zP66 zA~!21vmF3H2{ngD;+`6j#~6j;$*f*G_2ZD1E;9(yaw7d-QnSCpK(cR1zU3qU0000< KMNUMnLSTYoA~SLT literal 0 HcmV?d00001 diff --git a/docs/bdwn.png b/docs/bdwn.png new file mode 100644 index 0000000000000000000000000000000000000000..940a0b950443a0bb1b216ac03c45b8a16c955452 GIT binary patch literal 147 zcmeAS@N?(olHy`uVBq!ia0vp^>_E)H!3HEvS)PKZC{Gv1kP61Pb5HX&C2wk~_T + + + + + + +intel-ipsec-mb: Data Structure Index + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Data Structure Index
+
+
+
C | G | I | K | S
+
+
+
C
+
chacha20_poly1305_context_data
+
+
G
+
gcm_context_data
gcm_key_data
+
+
I
+
IMB_JOB
IMB_MGR
IMB_SGL_IOV
imb_uint128_t
+
+
K
+
kasumi_key_sched_s
+
+
S
+
snow3g_key_schedule_s
+
+
+
+ + + + diff --git a/docs/closed.png b/docs/closed.png new file mode 100644 index 0000000000000000000000000000000000000000..98cc2c909da37a6df914fbf67780eebd99c597f5 GIT binary patch literal 132 zcmeAS@N?(olHy`uVBq!ia0vp^oFL4>1|%O$WD@{V-kvUwAr*{o@8{^CZMh(5KoB^r_<4^zF@3)Cp&&t3hdujKf f*?bjBoY!V+E))@{xMcbjXe@)LtDnm{r-UW|*e5JT literal 0 HcmV?d00001 diff --git a/docs/doc.png b/docs/doc.png new file mode 100644 index 0000000000000000000000000000000000000000..17edabff95f7b8da13c9516a04efe05493c29501 GIT binary patch literal 746 zcmV7=@pnbNXRFEm&G8P!&WHG=d)>K?YZ1bzou)2{$)) zumDct!>4SyxL;zgaG>wy`^Hv*+}0kUfCrz~BCOViSb$_*&;{TGGn2^x9K*!Sf0=lV zpP=7O;GA0*Jm*tTYj$IoXvimpnV4S1Z5f$p*f$Db2iq2zrVGQUz~yq`ahn7ck(|CE z7Gz;%OP~J6)tEZWDzjhL9h2hdfoU2)Nd%T<5Kt;Y0XLt&<@6pQx!nw*5`@bq#?l*?3z{Hlzoc=Pr>oB5(9i6~_&-}A(4{Q$>c>%rV&E|a(r&;?i5cQB=} zYSDU5nXG)NS4HEs0it2AHe2>shCyr7`6@4*6{r@8fXRbTA?=IFVWAQJL&H5H{)DpM#{W(GL+Idzf^)uRV@oB8u$ z8v{MfJbTiiRg4bza<41NAzrl{=3fl_D+$t+^!xlQ8S}{UtY`e z;;&9UhyZqQRN%2pot{*Ei0*4~hSF_3AH2@fKU!$NSflS>{@tZpDT4`M2WRTTVH+D? z)GFlEGGHe?koB}i|1w45!BF}N_q&^HJ&-tyR{(afC6H7|aml|tBBbv}55C5DNP8p3 z)~jLEO4Z&2hZmP^i-e%(@d!(E|KRafiU8Q5u(wU((j8un3OR*Hvj+t literal 0 HcmV?d00001 diff --git a/docs/doxygen.css b/docs/doxygen.css new file mode 100644 index 00000000..d32e583c --- /dev/null +++ b/docs/doxygen.css @@ -0,0 +1,1841 @@ +/* The standard CSS for doxygen 1.9.5 */ + +body, table, div, p, dl { + font: 400 14px/22px Roboto,sans-serif; +} + +p.reference, p.definition { + font: 400 14px/22px Roboto,sans-serif; +} + +/* @group Heading Levels */ + +h1.groupheader { + font-size: 150%; +} + +.title { + font: 400 14px/28px Roboto,sans-serif; + font-size: 150%; + font-weight: bold; + margin: 10px 2px; +} + +h2.groupheader { + border-bottom: 1px solid #879ECB; + color: #354C7B; + font-size: 150%; + font-weight: normal; + margin-top: 1.75em; + padding-top: 8px; + padding-bottom: 4px; + width: 100%; +} + +h3.groupheader { + font-size: 100%; +} + +h1, h2, h3, h4, h5, h6 { + -webkit-transition: text-shadow 0.5s linear; + -moz-transition: text-shadow 0.5s linear; + -ms-transition: text-shadow 0.5s linear; + -o-transition: text-shadow 0.5s linear; + transition: text-shadow 0.5s linear; + margin-right: 15px; +} + +h1.glow, h2.glow, h3.glow, h4.glow, h5.glow, h6.glow { + text-shadow: 0 0 15px cyan; +} + +dt { + font-weight: bold; +} + +ul.multicol { + -moz-column-gap: 1em; + -webkit-column-gap: 1em; + column-gap: 1em; + -moz-column-count: 3; + -webkit-column-count: 3; + column-count: 3; +} + +p.startli, p.startdd { + margin-top: 2px; +} + +th p.starttd, th p.intertd, th p.endtd { + font-size: 100%; + font-weight: 700; +} + +p.starttd { + margin-top: 0px; +} + +p.endli { + margin-bottom: 0px; +} + +p.enddd { + margin-bottom: 4px; +} + +p.endtd { + margin-bottom: 2px; +} + +p.interli { +} + +p.interdd { +} + +p.intertd { +} + +/* @end */ + +caption { + font-weight: bold; +} + +span.legend { + font-size: 70%; + text-align: center; +} + +h3.version { + font-size: 90%; + text-align: center; +} + +div.navtab { + border-right: 1px solid #A3B4D7; + padding-right: 15px; + text-align: right; + line-height: 110%; +} + +div.navtab table { + border-spacing: 0; +} + +td.navtab { + padding-right: 6px; + padding-left: 6px; +} +td.navtabHL { + background-image: url('tab_a.png'); + background-repeat:repeat-x; + padding-right: 6px; + padding-left: 6px; +} + +td.navtabHL a, td.navtabHL a:visited { + color: #fff; + text-shadow: 0px 1px 1px rgba(0, 0, 0, 1.0); +} + +a.navtab { + font-weight: bold; +} + +div.qindex{ + text-align: center; + width: 100%; + line-height: 140%; + font-size: 130%; + color: #A0A0A0; +} + +dt.alphachar{ + font-size: 180%; + font-weight: bold; +} + +.alphachar a{ + color: black; +} + +.alphachar a:hover, .alphachar a:visited{ + text-decoration: none; +} + +.classindex dl { + padding: 25px; + column-count:1 +} + +.classindex dd { + display:inline-block; + margin-left: 50px; + width: 90%; + line-height: 1.15em; +} + +.classindex dl.odd { + background-color: #F8F9FC; +} + +@media(min-width: 1120px) { + .classindex dl { + column-count:2 + } +} + +@media(min-width: 1320px) { + .classindex dl { + column-count:3 + } +} + + +/* @group Link Styling */ + +a { + color: #3D578C; + font-weight: normal; + text-decoration: none; +} + +.contents a:visited { + color: #4665A2; +} + +a:hover { + text-decoration: underline; +} + +.contents a.qindexHL:visited { + color: #FFFFFF; +} + +a.el { + font-weight: bold; +} + +a.elRef { +} + +a.code, a.code:visited, a.line, a.line:visited { + color: #4665A2; +} + +a.codeRef, a.codeRef:visited, a.lineRef, a.lineRef:visited { + color: #4665A2; +} + +a.code.hl_class { /* style for links to class names in code snippets */ } +a.code.hl_struct { /* style for links to struct names in code snippets */ } +a.code.hl_union { /* style for links to union names in code snippets */ } +a.code.hl_interface { /* style for links to interface names in code snippets */ } +a.code.hl_protocol { /* style for links to protocol names in code snippets */ } +a.code.hl_category { /* style for links to category names in code snippets */ } +a.code.hl_exception { /* style for links to exception names in code snippets */ } +a.code.hl_service { /* style for links to service names in code snippets */ } +a.code.hl_singleton { /* style for links to singleton names in code snippets */ } +a.code.hl_concept { /* style for links to concept names in code snippets */ } +a.code.hl_namespace { /* style for links to namespace names in code snippets */ } +a.code.hl_package { /* style for links to package names in code snippets */ } +a.code.hl_define { /* style for links to macro names in code snippets */ } +a.code.hl_function { /* style for links to function names in code snippets */ } +a.code.hl_variable { /* style for links to variable names in code snippets */ } +a.code.hl_typedef { /* style for links to typedef names in code snippets */ } +a.code.hl_enumvalue { /* style for links to enum value names in code snippets */ } +a.code.hl_enumeration { /* style for links to enumeration names in code snippets */ } +a.code.hl_signal { /* style for links to Qt signal names in code snippets */ } +a.code.hl_slot { /* style for links to Qt slot names in code snippets */ } +a.code.hl_friend { /* style for links to friend names in code snippets */ } +a.code.hl_dcop { /* style for links to KDE3 DCOP names in code snippets */ } +a.code.hl_property { /* style for links to property names in code snippets */ } +a.code.hl_event { /* style for links to event names in code snippets */ } +a.code.hl_sequence { /* style for links to sequence names in code snippets */ } +a.code.hl_dictionary { /* style for links to dictionary names in code snippets */ } + +/* @end */ + +dl.el { + margin-left: -1cm; +} + +ul { + overflow: visible; +} + +#side-nav ul { + overflow: visible; /* reset ul rule for scroll bar in GENERATE_TREEVIEW window */ +} + +#main-nav ul { + overflow: visible; /* reset ul rule for the navigation bar drop down lists */ +} + +.fragment { + text-align: left; + direction: ltr; + overflow-x: auto; /*Fixed: fragment lines overlap floating elements*/ + overflow-y: hidden; +} + +pre.fragment { + border: 1px solid #C4CFE5; + background-color: #FBFCFD; + padding: 4px 6px; + margin: 4px 8px 4px 2px; + overflow: auto; + word-wrap: break-word; + font-size: 9pt; + line-height: 125%; + font-family: monospace, fixed; + font-size: 105%; +} + +div.fragment { + padding: 0 0 1px 0; /*Fixed: last line underline overlap border*/ + margin: 4px 8px 4px 2px; + background-color: #FBFCFD; + border: 1px solid #C4CFE5; +} + +div.line { + font-family: monospace, fixed; + font-size: 13px; + min-height: 13px; + line-height: 1.0; + text-wrap: unrestricted; + white-space: -moz-pre-wrap; /* Moz */ + white-space: -pre-wrap; /* Opera 4-6 */ + white-space: -o-pre-wrap; /* Opera 7 */ + white-space: pre-wrap; /* CSS3 */ + word-wrap: break-word; /* IE 5.5+ */ + text-indent: -53px; + padding-left: 53px; + padding-bottom: 0px; + margin: 0px; + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +div.line:after { + content:"\000A"; + white-space: pre; +} + +div.line.glow { + background-color: cyan; + box-shadow: 0 0 10px cyan; +} + + +span.lineno { + padding-right: 4px; + margin-right: 9px; + text-align: right; + border-right: 2px solid #0F0; + background-color: #E8E8E8; + white-space: pre; +} +span.lineno a { + background-color: #D8D8D8; +} + +span.lineno a:hover { + background-color: #C8C8C8; +} + +.lineno { + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +div.ah, span.ah { + background-color: black; + font-weight: bold; + color: #FFFFFF; + margin-bottom: 3px; + margin-top: 3px; + padding: 0.2em; + border: solid thin #333; + border-radius: 0.5em; + -webkit-border-radius: .5em; + -moz-border-radius: .5em; + box-shadow: 2px 2px 3px #999; + -webkit-box-shadow: 2px 2px 3px #999; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + background-image: -webkit-gradient(linear, left top, left bottom, from(#eee), to(#000),color-stop(0.3, #444)); + background-image: -moz-linear-gradient(center top, #eee 0%, #444 40%, #000 110%); +} + +div.classindex ul { + list-style: none; + padding-left: 0; +} + +div.classindex span.ai { + display: inline-block; +} + +div.groupHeader { + margin-left: 16px; + margin-top: 12px; + font-weight: bold; +} + +div.groupText { + margin-left: 16px; + font-style: italic; +} + +body { + background-color: white; + color: black; + margin: 0; +} + +div.contents { + margin-top: 10px; + margin-left: 12px; + margin-right: 8px; +} + +td.indexkey { + background-color: #EBEFF6; + font-weight: bold; + border: 1px solid #C4CFE5; + margin: 2px 0px 2px 0; + padding: 2px 10px; + white-space: nowrap; + vertical-align: top; +} + +td.indexvalue { + background-color: #EBEFF6; + border: 1px solid #C4CFE5; + padding: 2px 10px; + margin: 2px 0px; +} + +tr.memlist { + background-color: #EEF1F7; +} + +p.formulaDsp { + text-align: center; +} + +img.formulaDsp { + +} + +img.formulaInl, img.inline { + vertical-align: middle; +} + +div.center { + text-align: center; + margin-top: 0px; + margin-bottom: 0px; + padding: 0px; +} + +div.center img { + border: 0px; +} + +address.footer { + text-align: right; + padding-right: 12px; +} + +img.footer { + border: 0px; + vertical-align: middle; +} + +.compoundTemplParams { + color: #4665A2; + font-size: 80%; + line-height: 120%; +} + +/* @group Code Colorization */ + +span.keyword { + color: #008000 +} + +span.keywordtype { + color: #604020 +} + +span.keywordflow { + color: #e08000 +} + +span.comment { + color: #800000 +} + +span.preprocessor { + color: #806020 +} + +span.stringliteral { + color: #002080 +} + +span.charliteral { + color: #008080 +} + +span.vhdldigit { + color: #ff00ff +} + +span.vhdlchar { + color: #000000 +} + +span.vhdlkeyword { + color: #700070 +} + +span.vhdllogic { + color: #ff0000 +} + +blockquote { + background-color: #F7F8FB; + border-left: 2px solid #9CAFD4; + margin: 0 24px 0 4px; + padding: 0 12px 0 16px; +} + +blockquote.DocNodeRTL { + border-left: 0; + border-right: 2px solid #9CAFD4; + margin: 0 4px 0 24px; + padding: 0 16px 0 12px; +} + +/* @end */ + +/* +.search { + color: #003399; + font-weight: bold; +} + +form.search { + margin-bottom: 0px; + margin-top: 0px; +} + +input.search { + font-size: 75%; + color: #000080; + font-weight: normal; + background-color: #e8eef2; +} +*/ + +td.tiny { + font-size: 75%; +} + +.dirtab { + padding: 4px; + border-collapse: collapse; + border: 1px solid #A3B4D7; +} + +th.dirtab { + background: #EBEFF6; + font-weight: bold; +} + +hr { + height: 0px; + border: none; + border-top: 1px solid #4A6AAA; +} + +hr.footer { + height: 1px; +} + +/* @group Member Descriptions */ + +table.memberdecls { + border-spacing: 0px; + padding: 0px; +} + +.memberdecls td, .fieldtable tr { + -webkit-transition-property: background-color, box-shadow; + -webkit-transition-duration: 0.5s; + -moz-transition-property: background-color, box-shadow; + -moz-transition-duration: 0.5s; + -ms-transition-property: background-color, box-shadow; + -ms-transition-duration: 0.5s; + -o-transition-property: background-color, box-shadow; + -o-transition-duration: 0.5s; + transition-property: background-color, box-shadow; + transition-duration: 0.5s; +} + +.memberdecls td.glow, .fieldtable tr.glow { + background-color: cyan; + box-shadow: 0 0 15px cyan; +} + +.mdescLeft, .mdescRight, +.memItemLeft, .memItemRight, +.memTemplItemLeft, .memTemplItemRight, .memTemplParams { + background-color: #F9FAFC; + border: none; + margin: 4px; + padding: 1px 0 0 8px; +} + +.mdescLeft, .mdescRight { + padding: 0px 8px 4px 8px; + color: #555; +} + +.memSeparator { + border-bottom: 1px solid #DEE4F0; + line-height: 1px; + margin: 0px; + padding: 0px; +} + +.memItemLeft, .memTemplItemLeft { + white-space: nowrap; +} + +.memItemRight, .memTemplItemRight { + width: 100%; +} + +.memTemplParams { + color: #4665A2; + white-space: nowrap; + font-size: 80%; +} + +/* @end */ + +/* @group Member Details */ + +/* Styles for detailed member documentation */ + +.memtitle { + padding: 8px; + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + border-top-right-radius: 4px; + border-top-left-radius: 4px; + margin-bottom: -1px; + background-image: url('nav_f.png'); + background-repeat: repeat-x; + background-color: #E2E8F2; + line-height: 1.25; + font-weight: 300; + float:left; +} + +.permalink +{ + font-size: 65%; + display: inline-block; + vertical-align: middle; +} + +.memtemplate { + font-size: 80%; + color: #4665A2; + font-weight: normal; + margin-left: 9px; +} + +.memnav { + background-color: #EBEFF6; + border: 1px solid #A3B4D7; + text-align: center; + margin: 2px; + margin-right: 15px; + padding: 2px; +} + +.mempage { + width: 100%; +} + +.memitem { + padding: 0; + margin-bottom: 10px; + margin-right: 5px; + -webkit-transition: box-shadow 0.5s linear; + -moz-transition: box-shadow 0.5s linear; + -ms-transition: box-shadow 0.5s linear; + -o-transition: box-shadow 0.5s linear; + transition: box-shadow 0.5s linear; + display: table !important; + width: 100%; +} + +.memitem.glow { + box-shadow: 0 0 15px cyan; +} + +.memname { + font-weight: 400; + margin-left: 6px; +} + +.memname td { + vertical-align: bottom; +} + +.memproto, dl.reflist dt { + border-top: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 0px 6px 0px; + color: #253555; + font-weight: bold; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + background-color: #DFE5F1; + /* opera specific markup */ + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + border-top-right-radius: 4px; + /* firefox specific markup */ + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + -moz-border-radius-topright: 4px; + /* webkit specific markup */ + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + -webkit-border-top-right-radius: 4px; + +} + +.overload { + font-family: "courier new",courier,monospace; + font-size: 65%; +} + +.memdoc, dl.reflist dd { + border-bottom: 1px solid #A8B8D9; + border-left: 1px solid #A8B8D9; + border-right: 1px solid #A8B8D9; + padding: 6px 10px 2px 10px; + background-color: #FBFCFD; + border-top-width: 0; + background-image:url('nav_g.png'); + background-repeat:repeat-x; + background-color: #FFFFFF; + /* opera specific markup */ + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); + /* firefox specific markup */ + -moz-border-radius-bottomleft: 4px; + -moz-border-radius-bottomright: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 5px 5px 5px; + /* webkit specific markup */ + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +dl.reflist dt { + padding: 5px; +} + +dl.reflist dd { + margin: 0px 0px 10px 0px; + padding: 5px; +} + +.paramkey { + text-align: right; +} + +.paramtype { + white-space: nowrap; +} + +.paramname { + color: #602020; + white-space: nowrap; +} +.paramname em { + font-style: normal; +} +.paramname code { + line-height: 14px; +} + +.params, .retval, .exception, .tparams { + margin-left: 0px; + padding-left: 0px; +} + +.params .paramname, .retval .paramname, .tparams .paramname, .exception .paramname { + font-weight: bold; + vertical-align: top; +} + +.params .paramtype, .tparams .paramtype { + font-style: italic; + vertical-align: top; +} + +.params .paramdir, .tparams .paramdir { + font-family: "courier new",courier,monospace; + vertical-align: top; +} + +table.mlabels { + border-spacing: 0px; +} + +td.mlabels-left { + width: 100%; + padding: 0px; +} + +td.mlabels-right { + vertical-align: bottom; + padding: 0px; + white-space: nowrap; +} + +span.mlabels { + margin-left: 8px; +} + +span.mlabel { + background-color: #728DC1; + border-top:1px solid #5373B4; + border-left:1px solid #5373B4; + border-right:1px solid #C4CFE5; + border-bottom:1px solid #C4CFE5; + text-shadow: none; + color: white; + margin-right: 4px; + padding: 2px 3px; + border-radius: 3px; + font-size: 7pt; + white-space: nowrap; + vertical-align: middle; +} + + + +/* @end */ + +/* these are for tree view inside a (index) page */ + +div.directory { + margin: 10px 0px; + border-top: 1px solid #9CAFD4; + border-bottom: 1px solid #9CAFD4; + width: 100%; +} + +.directory table { + border-collapse:collapse; +} + +.directory td { + margin: 0px; + padding: 0px; + vertical-align: top; +} + +.directory td.entry { + white-space: nowrap; + padding-right: 6px; + padding-top: 3px; +} + +.directory td.entry a { + outline:none; +} + +.directory td.entry a img { + border: none; +} + +.directory td.desc { + width: 100%; + padding-left: 6px; + padding-right: 6px; + padding-top: 3px; + border-left: 1px solid rgba(0,0,0,0.05); +} + +.directory tr.even { + padding-left: 6px; + background-color: #F7F8FB; +} + +.directory img { + vertical-align: -30%; +} + +.directory .levels { + white-space: nowrap; + width: 100%; + text-align: right; + font-size: 9pt; +} + +.directory .levels span { + cursor: pointer; + padding-left: 2px; + padding-right: 2px; + color: #3D578C; +} + +.arrow { + color: #9CAFD4; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + cursor: pointer; + font-size: 80%; + display: inline-block; + width: 16px; + height: 22px; +} + +.icon { + font-family: Arial, Helvetica; + font-weight: bold; + font-size: 12px; + height: 14px; + width: 16px; + display: inline-block; + background-color: #728DC1; + color: white; + text-align: center; + border-radius: 4px; + margin-left: 2px; + margin-right: 2px; +} + +.icona { + width: 24px; + height: 22px; + display: inline-block; +} + +.iconfopen { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderopen.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.iconfclosed { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('folderclosed.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +.icondoc { + width: 24px; + height: 18px; + margin-bottom: 4px; + background-image:url('doc.png'); + background-position: 0px -4px; + background-repeat: repeat-y; + vertical-align:top; + display: inline-block; +} + +table.directory { + font: 400 14px Roboto,sans-serif; +} + +/* @end */ + +div.dynheader { + margin-top: 8px; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +address { + font-style: normal; + color: #2A3D61; +} + +table.doxtable caption { + caption-side: top; +} + +table.doxtable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.doxtable td, table.doxtable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.doxtable th { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +table.fieldtable { + /*width: 100%;*/ + margin-bottom: 10px; + border: 1px solid #A8B8D9; + border-spacing: 0px; + -moz-border-radius: 4px; + -webkit-border-radius: 4px; + border-radius: 4px; + -moz-box-shadow: rgba(0, 0, 0, 0.15) 2px 2px 2px; + -webkit-box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); + box-shadow: 2px 2px 2px rgba(0, 0, 0, 0.15); +} + +.fieldtable td, .fieldtable th { + padding: 3px 7px 2px; +} + +.fieldtable td.fieldtype, .fieldtable td.fieldname { + white-space: nowrap; + border-right: 1px solid #A8B8D9; + border-bottom: 1px solid #A8B8D9; + vertical-align: top; +} + +.fieldtable td.fieldname { + padding-top: 3px; +} + +.fieldtable td.fielddoc { + border-bottom: 1px solid #A8B8D9; + /*width: 100%;*/ +} + +.fieldtable td.fielddoc p:first-child { + margin-top: 0px; +} + +.fieldtable td.fielddoc p:last-child { + margin-bottom: 2px; +} + +.fieldtable tr:last-child td { + border-bottom: none; +} + +.fieldtable th { + background-image:url('nav_f.png'); + background-repeat:repeat-x; + background-color: #E2E8F2; + font-size: 90%; + color: #253555; + padding-bottom: 4px; + padding-top: 5px; + text-align:left; + font-weight: 400; + -moz-border-radius-topleft: 4px; + -moz-border-radius-topright: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + border-top-left-radius: 4px; + border-top-right-radius: 4px; + border-bottom: 1px solid #A8B8D9; +} + + +.tabsearch { + top: 0px; + left: 10px; + height: 36px; + background-image: url('tab_b.png'); + z-index: 101; + overflow: hidden; + font-size: 13px; +} + +.navpath ul +{ + font-size: 11px; + background-image:url('tab_b.png'); + background-repeat:repeat-x; + background-position: 0 -5px; + height:30px; + line-height:30px; + color:#8AA0CC; + border:solid 1px #C2CDE4; + overflow:hidden; + margin:0px; + padding:0px; +} + +.navpath li +{ + list-style-type:none; + float:left; + padding-left:10px; + padding-right:15px; + background-image:url('bc_s.png'); + background-repeat:no-repeat; + background-position:right; + color:#364D7C; +} + +.navpath li.navelem a +{ + height:32px; + display:block; + text-decoration: none; + outline: none; + color: #283A5D; + font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif; + text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9); + text-decoration: none; +} + +.navpath li.navelem a:hover +{ + color:#6884BD; +} + +.navpath li.footer +{ + list-style-type:none; + float:right; + padding-left:10px; + padding-right:15px; + background-image:none; + background-repeat:no-repeat; + background-position:right; + color:#364D7C; + font-size: 8pt; +} + + +div.summary +{ + float: right; + font-size: 8pt; + padding-right: 5px; + width: 50%; + text-align: right; +} + +div.summary a +{ + white-space: nowrap; +} + +table.classindex +{ + margin: 10px; + white-space: nowrap; + margin-left: 3%; + margin-right: 3%; + width: 94%; + border: 0; + border-spacing: 0; + padding: 0; +} + +div.ingroups +{ + font-size: 8pt; + width: 50%; + text-align: left; +} + +div.ingroups a +{ + white-space: nowrap; +} + +div.header +{ + background-image:url('nav_h.png'); + background-repeat:repeat-x; + background-color: #F9FAFC; + margin: 0px; + border-bottom: 1px solid #C4CFE5; +} + +div.headertitle +{ + padding: 5px 5px 5px 10px; +} + +.PageDocRTL-title div.headertitle { + text-align: right; + direction: rtl; +} + +dl { + padding: 0 0 0 0; +} + +/* dl.note, dl.warning, dl.attention, dl.pre, dl.post, dl.invariant, dl.deprecated, dl.todo, dl.test, dl.bug, dl.examples */ +dl.section { + margin-left: 0px; + padding-left: 0px; +} + +dl.section.DocNodeRTL { + margin-right: 0px; + padding-right: 0px; +} + +dl.note { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #D0C000; +} + +dl.note.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #D0C000; +} + +dl.warning, dl.attention { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #FF0000; +} + +dl.warning.DocNodeRTL, dl.attention.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #FF0000; +} + +dl.pre, dl.post, dl.invariant { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #00D000; +} + +dl.pre.DocNodeRTL, dl.post.DocNodeRTL, dl.invariant.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #00D000; +} + +dl.deprecated { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #505050; +} + +dl.deprecated.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #505050; +} + +dl.todo { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #00C0E0; +} + +dl.todo.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #00C0E0; +} + +dl.test { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #3030E0; +} + +dl.test.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #3030E0; +} + +dl.bug { + margin-left: -7px; + padding-left: 3px; + border-left: 4px solid; + border-color: #C08050; +} + +dl.bug.DocNodeRTL { + margin-left: 0; + padding-left: 0; + border-left: 0; + margin-right: -7px; + padding-right: 3px; + border-right: 4px solid; + border-color: #C08050; +} + +dl.section dd { + margin-bottom: 6px; +} + + +#projectrow +{ + height: 56px; +} + +#projectlogo +{ + text-align: center; + vertical-align: bottom; + border-collapse: separate; +} + +#projectlogo img +{ + border: 0px none; +} + +#projectalign +{ + vertical-align: middle; + padding-left: 0.5em; +} + +#projectname +{ + font: 200% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 2px 0px; +} + +#projectbrief +{ + font: 90% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#projectnumber +{ + font: 50% Tahoma, Arial,sans-serif; + margin: 0px; + padding: 0px; +} + +#titlearea +{ + padding: 0px; + margin: 0px; + width: 100%; + border-bottom: 1px solid #5373B4; +} + +.image +{ + text-align: center; +} + +.dotgraph +{ + text-align: center; +} + +.mscgraph +{ + text-align: center; +} + +.plantumlgraph +{ + text-align: center; +} + +.diagraph +{ + text-align: center; +} + +.caption +{ + font-weight: bold; +} + +div.zoom +{ + border: 1px solid #90A5CE; +} + +dl.citelist { + margin-bottom:50px; +} + +dl.citelist dt { + color:#334975; + float:left; + font-weight:bold; + margin-right:10px; + padding:5px; + text-align:right; + width:52px; +} + +dl.citelist dd { + margin:2px 0 2px 72px; + padding:5px 0; +} + +div.toc { + padding: 14px 25px; + background-color: #F4F6FA; + border: 1px solid #D8DFEE; + border-radius: 7px 7px 7px 7px; + float: right; + height: auto; + margin: 0 8px 10px 10px; + width: 200px; +} + +.PageDocRTL-title div.toc { + float: left !important; + text-align: right; +} + +div.toc li { + background: url("bdwn.png") no-repeat scroll 0 5px transparent; + font: 10px/1.2 Verdana,DejaVu Sans,Geneva,sans-serif; + margin-top: 5px; + padding-left: 10px; + padding-top: 2px; +} + +.PageDocRTL-title div.toc li { + background-position-x: right !important; + padding-left: 0 !important; + padding-right: 10px; +} + +div.toc h3 { + font: bold 12px/1.2 Arial,FreeSans,sans-serif; + color: #4665A2; + border-bottom: 0 none; + margin: 0; +} + +div.toc ul { + list-style: none outside none; + border: medium none; + padding: 0px; +} + +div.toc li.level1 { + margin-left: 0px; +} + +div.toc li.level2 { + margin-left: 15px; +} + +div.toc li.level3 { + margin-left: 30px; +} + +div.toc li.level4 { + margin-left: 45px; +} + +span.emoji { + /* font family used at the site: https://unicode.org/emoji/charts/full-emoji-list.html + * font-family: "Noto Color Emoji", "Apple Color Emoji", "Segoe UI Emoji", Times, Symbola, Aegyptus, Code2000, Code2001, Code2002, Musica, serif, LastResort; + */ +} + +span.obfuscator { + display: none; +} + +.PageDocRTL-title div.toc li.level1 { + margin-left: 0 !important; + margin-right: 0; +} + +.PageDocRTL-title div.toc li.level2 { + margin-left: 0 !important; + margin-right: 15px; +} + +.PageDocRTL-title div.toc li.level3 { + margin-left: 0 !important; + margin-right: 30px; +} + +.PageDocRTL-title div.toc li.level4 { + margin-left: 0 !important; + margin-right: 45px; +} + +.inherit_header { + font-weight: bold; + color: gray; + cursor: pointer; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; +} + +.inherit_header td { + padding: 6px 0px 2px 5px; +} + +.inherit { + display: none; +} + +tr.heading h2 { + margin-top: 12px; + margin-bottom: 4px; +} + +/* tooltip related style info */ + +.ttc { + position: absolute; + display: none; +} + +#powerTip { + cursor: default; + /*white-space: nowrap;*/ + background-color: white; + border: 1px solid gray; + border-radius: 4px 4px 4px 4px; + box-shadow: 1px 1px 7px gray; + display: none; + font-size: smaller; + max-width: 80%; + opacity: 0.9; + padding: 1ex 1em 1em; + position: absolute; + z-index: 2147483647; +} + +#powerTip div.ttdoc { + color: grey; + font-style: italic; +} + +#powerTip div.ttname a { + font-weight: bold; +} + +#powerTip div.ttname { + font-weight: bold; +} + +#powerTip div.ttdeci { + color: #006318; +} + +#powerTip div { + margin: 0px; + padding: 0px; + font: 12px/16px Roboto,sans-serif; +} + +#powerTip:before, #powerTip:after { + content: ""; + position: absolute; + margin: 0px; +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.s:after, #powerTip.s:before, +#powerTip.w:after, #powerTip.w:before, +#powerTip.e:after, #powerTip.e:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.nw:after, #powerTip.nw:before, +#powerTip.sw:after, #powerTip.sw:before { + border: solid transparent; + content: " "; + height: 0; + width: 0; + position: absolute; +} + +#powerTip.n:after, #powerTip.s:after, +#powerTip.w:after, #powerTip.e:after, +#powerTip.nw:after, #powerTip.ne:after, +#powerTip.sw:after, #powerTip.se:after { + border-color: rgba(255, 255, 255, 0); +} + +#powerTip.n:before, #powerTip.s:before, +#powerTip.w:before, #powerTip.e:before, +#powerTip.nw:before, #powerTip.ne:before, +#powerTip.sw:before, #powerTip.se:before { + border-color: rgba(128, 128, 128, 0); +} + +#powerTip.n:after, #powerTip.n:before, +#powerTip.ne:after, #powerTip.ne:before, +#powerTip.nw:after, #powerTip.nw:before { + top: 100%; +} + +#powerTip.n:after, #powerTip.ne:after, #powerTip.nw:after { + border-top-color: #FFFFFF; + border-width: 10px; + margin: 0px -10px; +} +#powerTip.n:before { + border-top-color: #808080; + border-width: 11px; + margin: 0px -11px; +} +#powerTip.n:after, #powerTip.n:before { + left: 50%; +} + +#powerTip.nw:after, #powerTip.nw:before { + right: 14px; +} + +#powerTip.ne:after, #powerTip.ne:before { + left: 14px; +} + +#powerTip.s:after, #powerTip.s:before, +#powerTip.se:after, #powerTip.se:before, +#powerTip.sw:after, #powerTip.sw:before { + bottom: 100%; +} + +#powerTip.s:after, #powerTip.se:after, #powerTip.sw:after { + border-bottom-color: #FFFFFF; + border-width: 10px; + margin: 0px -10px; +} + +#powerTip.s:before, #powerTip.se:before, #powerTip.sw:before { + border-bottom-color: #808080; + border-width: 11px; + margin: 0px -11px; +} + +#powerTip.s:after, #powerTip.s:before { + left: 50%; +} + +#powerTip.sw:after, #powerTip.sw:before { + right: 14px; +} + +#powerTip.se:after, #powerTip.se:before { + left: 14px; +} + +#powerTip.e:after, #powerTip.e:before { + left: 100%; +} +#powerTip.e:after { + border-left-color: #FFFFFF; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.e:before { + border-left-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +#powerTip.w:after, #powerTip.w:before { + right: 100%; +} +#powerTip.w:after { + border-right-color: #FFFFFF; + border-width: 10px; + top: 50%; + margin-top: -10px; +} +#powerTip.w:before { + border-right-color: #808080; + border-width: 11px; + top: 50%; + margin-top: -11px; +} + +@media print +{ + #top { display: none; } + #side-nav { display: none; } + #nav-path { display: none; } + body { overflow:visible; } + h1, h2, h3, h4, h5, h6 { page-break-after: avoid; } + .summary { display: none; } + .memitem { page-break-inside: avoid; } + #doc-content + { + margin-left:0 !important; + height:auto !important; + width:auto !important; + overflow:inherit; + display:inline; + } +} + +/* @group Markdown */ + +table.markdownTable { + border-collapse:collapse; + margin-top: 4px; + margin-bottom: 4px; +} + +table.markdownTable td, table.markdownTable th { + border: 1px solid #2D4068; + padding: 3px 7px 2px; +} + +table.markdownTable tr { +} + +th.markdownTableHeadLeft, th.markdownTableHeadRight, th.markdownTableHeadCenter, th.markdownTableHeadNone { + background-color: #374F7F; + color: #FFFFFF; + font-size: 110%; + padding-bottom: 4px; + padding-top: 5px; +} + +th.markdownTableHeadLeft, td.markdownTableBodyLeft { + text-align: left +} + +th.markdownTableHeadRight, td.markdownTableBodyRight { + text-align: right +} + +th.markdownTableHeadCenter, td.markdownTableBodyCenter { + text-align: center +} + +.DocNodeRTL { + text-align: right; + direction: rtl; +} + +.DocNodeLTR { + text-align: left; + direction: ltr; +} + +table.DocNodeRTL { + width: auto; + margin-right: 0; + margin-left: auto; +} + +table.DocNodeLTR { + width: auto; + margin-right: auto; + margin-left: 0; +} + +code.JavaDocCode { + direction:ltr; +} + +tt, code, kbd, samp +{ + display: inline-block; + direction:ltr; +} +/* @end */ + +u { + text-decoration: underline; +} + diff --git a/docs/doxygen.png b/docs/doxygen.png new file mode 100644 index 0000000000000000000000000000000000000000..3ff17d807fd8aa003bed8bb2a69e8f0909592fd1 GIT binary patch literal 3779 zcmV;!4m|ORP)tMIv#Q0*~7*`IBSO7_x;@a8#Zk6_PeKR_s92J&)(m+);m9Iz3blw)z#Gi zP!9lj4$%+*>Hz@HCmM9L9|8c+0u=!H$O3?R0Kgx|#WP<6fKfC8fM-CQZT|_r@`>VO zX^Hgb|9cJqpdJA5$MCEK`F_2@2Y@s>^+;pF`~jdI0Pvr|vl4`=C)EH@1IFe7pdJ8F zH(qGi004~QnF)Ggga~8v08kGAs2hKTATxr7pwfNk|4#_AaT>w8P6TV+R2kbS$v==} zAjf`s0g#V8lB+b3)5oEI*q+{Yt$MZDruD2^;$+(_%Qn+%v0X-bJO=;@kiJ^ygLBnC z?1OVv_%aex1M@jKU|Z~$eI?PoF4Vj>fDzyo zAiLfpXY*a^Sj-S5D0S3@#V$sRW)g)_1e#$%8xdM>Jm7?!h zu0P2X=xoN>^!4DoPRgph2(2va07yfpXF+WH7EOg1GY%Zn z7~1A<(z7Q$ktEXhW_?GMpHp9l_UL18F3KOsxu81pqoBiNbFSGsof-W z6~eloMoz=4?OOnl2J268x5rOY`dCk0us(uS#Ud4yqOr@?=Q57a}tit|BhY>}~frH1sP`ScHS_d)oqH^lYy zZ%VP`#10MlE~P?cE(%(#(AUSv_T{+;t@$U}El}(1ig`vZo`Rm;+5&(AYzJ^Ae=h2X z@Re%vHwZU>|f0NI&%$*4eJweC5OROQrpPMA@*w|o z()A==l}(@bv^&>H1Ob3C=<^|hob?0+xJ?QQ3-ueQC}zy&JQNib!OqSO@-=>XzxlSF zAZ^U*1l6EEmg3r};_HY>&Jo_{dOPEFTWPmt=U&F#+0(O59^UIlHbNX+eF8UzyDR*T z(=5X$VF3!gm@RooS-&iiUYGG^`hMR(07zr_xP`d!^BH?uD>Phl8Rdifx3Af^Zr`Ku ztL+~HkVeL#bJ)7;`=>;{KNRvjmc}1}c58Sr#Treq=4{xo!ATy|c>iRSp4`dzMMVd@ zL8?uwXDY}Wqgh4mH`|$BTXpUIu6A1-cSq%hJw;@^Zr8TP=GMh*p(m(tN7@!^D~sl$ zz^tf4II4|};+irE$Fnm4NTc5%p{PRA`%}Zk`CE5?#h3|xcyQsS#iONZ z6H(@^i9td!$z~bZiJLTax$o>r(p}3o@< zyD7%(>ZYvy=6$U3e!F{Z`uSaYy`xQyl?b{}eg|G3&fz*`QH@mDUn)1%#5u`0m$%D} z?;tZ0u(mWeMV0QtzjgN!lT*pNRj;6510Wwx?Yi_=tYw|J#7@(Xe7ifDzXuK;JB;QO z#bg~K$cgm$@{QiL_3yr}y&~wuv=P=#O&Tj=Sr)aCUlYmZMcw?)T?c%0rUe1cS+o!qs_ zQ6Gp)-{)V!;=q}llyK3|^WeLKyjf%y;xHku;9(vM!j|~<7w1c*Mk-;P{T&yG) z@C-8E?QPynNQ<8f01D`2qexcVEIOU?y}MG)TAE6&VT5`rK8s(4PE;uQ92LTXUQ<>^ ztyQ@=@kRdh@ebUG^Z6NWWIL;_IGJ2ST>$t!$m$qvtj0Qmw8moN6GUV^!QKNK zHBXCtUH8)RY9++gH_TUV4^=-j$t}dD3qsN7GclJ^Zc&(j6&a_!$jCf}%c5ey`pm~1)@{yI3 zTdWyB+*X{JFw#z;PwRr5evb2!ueWF;v`B0HoUu4-(~aL=z;OXUUEtG`_$)Oxw6FKg zEzY`CyKaSBK3xt#8gA|r_|Kehn_HYVBMpEwbn9-fI*!u*eTA1ef8Mkl1=!jV4oYwWYM}i`A>_F4nhmlCIC6WLa zY%;4&@AlnaG11ejl61Jev21|r*m+?Kru3;1tFDl}#!OzUp6c>go4{C|^erwpG*&h6bspUPJag}oOkN2912Y3I?(eRc@U9>z#HPBHC?nps7H5!zP``90!Q1n80jo+B3TWXp!8Pe zwuKuLLI6l3Gv@+QH*Y}2wPLPQ1^EZhT#+Ed8q8Wo z1pTmIBxv14-{l&QVKxAyQF#8Q@NeJwWdKk>?cpiJLkJr+aZ!Me+Cfp!?FWSRf^j2k z73BRR{WSKaMkJ>1Nbx5dan5hg^_}O{Tj6u%iV%#QGz0Q@j{R^Ik)Z*+(YvY2ziBG)?AmJa|JV%4UT$k`hcOg5r9R?5>?o~JzK zJCrj&{i#hG>N7!B4kNX(%igb%kDj0fOQThC-8mtfap82PNRXr1D>lbgg)dYTQ(kbx z`Ee5kXG~Bh+BHQBf|kJEy6(ga%WfhvdQNDuOfQoe377l#ht&DrMGeIsI5C<&ai zWG$|hop2@@q5YDa)_-A?B02W;#fH!%k`daQLEItaJJ8Yf1L%8x;kg?)k)00P-lH+w z)5$QNV6r2$YtnV(4o=0^3{kmaXn*Dm0F*fU(@o)yVVjk|ln8ea6BMy%vZAhW9|wvA z8RoDkVoMEz1d>|5(k0Nw>22ZT){V<3$^C-cN+|~hKt2)){+l-?3m@-$c?-dlzQ)q- zZ)j%n^gerV{|+t}9m1_&&Ly!9$rtG4XX|WQ8`xYzGC~U@nYh~g(z9)bdAl#xH)xd5a=@|qql z|FzEil{P5(@gy!4ek05i$>`E^G~{;pnf6ftpLh$h#W?^#4UkPfa;;?bsIe&kz!+40 zI|6`F2n020)-r`pFaZ38F!S-lJM-o&inOw|66=GMeP@xQU5ghQH{~5Uh~TMTd;I9` z>YhVB`e^EVj*S7JF39ZgNf}A-0DwOcTT63ydN$I3b?yBQtUI*_fae~kPvzoD$zjX3 zoqBe#>12im4WzZ=f^4+u=!lA|#r%1`WB0-6*3BL#at`47#ebPpR|D1b)3BjT34nYY z%Ds%d?5$|{LgOIaRO{{oC&RK`O91$fqwM0(C_TALcozu*fWHb%%q&p-q{_8*2Zsi^ zh1ZCnr^UYa;4vQEtHk{~zi>wwMC5o{S=$P0X681y`SXwFH?Ewn{x-MOZynmc)JT5v zuHLwh;tLfxRrr%|k370}GofLl7thg>ACWWY&msqaVu&ry+`7+Ss>NL^%T1|z{IGMA zW-SKl=V-^{(f!Kf^#3(|T2W47d(%JVCI4JgRrT1pNz>+ietmFToNv^`gzC@&O-)+i zPQ~RwK8%C_vf%;%e>NyTp~dM5;!C|N0Q^6|CEb7Bw=Vz~$1#FA;Z*?mKSC)Hl-20s t8QyHj(g6VK0RYbl8UjE)0O0w=e*@m04r>stuEhWV002ovPDHLkV1hl;dM*F} literal 0 HcmV?d00001 diff --git a/docs/doxygen.svg b/docs/doxygen.svg new file mode 100644 index 00000000..d42dad52 --- /dev/null +++ b/docs/doxygen.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/dynsections.js b/docs/dynsections.js new file mode 100644 index 00000000..3174bd7b --- /dev/null +++ b/docs/dynsections.js @@ -0,0 +1,121 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +function toggleVisibility(linkObj) +{ + var base = $(linkObj).attr('id'); + var summary = $('#'+base+'-summary'); + var content = $('#'+base+'-content'); + var trigger = $('#'+base+'-trigger'); + var src=$(trigger).attr('src'); + if (content.is(':visible')===true) { + content.hide(); + summary.show(); + $(linkObj).addClass('closed').removeClass('opened'); + $(trigger).attr('src',src.substring(0,src.length-8)+'closed.png'); + } else { + content.show(); + summary.hide(); + $(linkObj).removeClass('closed').addClass('opened'); + $(trigger).attr('src',src.substring(0,src.length-10)+'open.png'); + } + return false; +} + +function updateStripes() +{ + $('table.directory tr'). + removeClass('even').filter(':visible:even').addClass('even'); +} + +function toggleLevel(level) +{ + $('table.directory tr').each(function() { + var l = this.id.split('_').length-1; + var i = $('#img'+this.id.substring(3)); + var a = $('#arr'+this.id.substring(3)); + if (l + + + + + + +intel-ipsec-mb: File List + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
File List
+
+
+
Here is a list of all files with brief descriptions:
+ + +
 intel-ipsec-mb.h
+
+
+
+ + + + diff --git a/docs/files_dup.js b/docs/files_dup.js new file mode 100644 index 00000000..80fcb488 --- /dev/null +++ b/docs/files_dup.js @@ -0,0 +1,4 @@ +var files_dup = +[ + [ "intel-ipsec-mb.h", "intel-ipsec-mb_8h.html", "intel-ipsec-mb_8h" ] +]; \ No newline at end of file diff --git a/docs/folderclosed.png b/docs/folderclosed.png new file mode 100644 index 0000000000000000000000000000000000000000..bb8ab35edce8e97554e360005ee9fc5bffb36e66 GIT binary patch literal 616 zcmV-u0+;=XP)a9#ETzayK)T~Jw&MMH>OIr#&;dC}is*2Mqdf&akCc=O@`qC+4i z5Iu3w#1M@KqXCz8TIZd1wli&kkl2HVcAiZ8PUn5z_kG@-y;?yK06=cA0U%H0PH+kU zl6dp}OR(|r8-RG+YLu`zbI}5TlOU6ToR41{9=uz^?dGTNL;wIMf|V3`d1Wj3y!#6` zBLZ?xpKR~^2x}?~zA(_NUu3IaDB$tKma*XUdOZN~c=dLt_h_k!dbxm_*ibDM zlFX`g{k$X}yIe%$N)cn1LNu=q9_CS)*>A zsX_mM4L@`(cSNQKMFc$RtYbx{79#j-J7hk*>*+ZZhM4Hw?I?rsXCi#mRWJ=-0LGV5a-WR0Qgt<|Nqf)C-@80`5gIz45^_20000IqP)X=#(TiCT&PiIIVc55T}TU}EUh*{q$|`3@{d>{Tc9Bo>e= zfmF3!f>fbI9#GoEHh0f`i5)wkLpva0ztf%HpZneK?w-7AK@b4Itw{y|Zd3k!fH?q2 zlhckHd_V2M_X7+)U&_Xcfvtw60l;--DgZmLSw-Y?S>)zIqMyJ1#FwLU*%bl38ok+! zh78H87n`ZTS;uhzAR$M`zZ`bVhq=+%u9^$5jDplgxd44}9;IRqUH1YHH|@6oFe%z( zo4)_>E$F&^P-f(#)>(TrnbE>Pefs9~@iN=|)Rz|V`sGfHNrJ)0gJb8xx+SBmRf@1l zvuzt=vGfI)<-F9!o&3l?>9~0QbUDT(wFdnQPv%xdD)m*g%!20>Bc9iYmGAp<9YAa( z0QgYgTWqf1qN++Gqp z8@AYPTB3E|6s=WLG?xw0tm|U!o=&zd+H0oRYE;Dbx+Na9s^STqX|Gnq%H8s(nGDGJ j8vwW|`Ts`)fSK|Kx=IK@RG@g200000NkvXXu0mjfauFEA literal 0 HcmV?d00001 diff --git a/docs/functions.html b/docs/functions.html new file mode 100644 index 00000000..76d04225 --- /dev/null +++ b/docs/functions.html @@ -0,0 +1,120 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/functions_a.html b/docs/functions_a.html new file mode 100644 index 00000000..a5b941a6 --- /dev/null +++ b/docs/functions_a.html @@ -0,0 +1,126 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- a -

+
+
+ + + + diff --git a/docs/functions_c.html b/docs/functions_c.html new file mode 100644 index 00000000..cfe15a61 --- /dev/null +++ b/docs/functions_c.html @@ -0,0 +1,140 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- c -

+
+
+ + + + diff --git a/docs/functions_d.html b/docs/functions_d.html new file mode 100644 index 00000000..4b51bce6 --- /dev/null +++ b/docs/functions_d.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/functions_dup.js b/docs/functions_dup.js new file mode 100644 index 00000000..8e1cb168 --- /dev/null +++ b/docs/functions_dup.js @@ -0,0 +1,26 @@ +var functions_dup = +[ + [ "_", "functions.html", null ], + [ "a", "functions_a.html", null ], + [ "c", "functions_c.html", null ], + [ "d", "functions_d.html", null ], + [ "e", "functions_e.html", null ], + [ "f", "functions_f.html", null ], + [ "g", "functions_g.html", null ], + [ "h", "functions_h.html", null ], + [ "i", "functions_i.html", null ], + [ "j", "functions_j.html", null ], + [ "k", "functions_k.html", null ], + [ "l", "functions_l.html", null ], + [ "m", "functions_m.html", null ], + [ "n", "functions_n.html", null ], + [ "o", "functions_o.html", null ], + [ "p", "functions_p.html", null ], + [ "q", "functions_q.html", null ], + [ "r", "functions_r.html", null ], + [ "s", "functions_s.html", null ], + [ "u", "functions_u.html", null ], + [ "v", "functions_v.html", null ], + [ "x", "functions_x.html", null ], + [ "z", "functions_z.html", null ] +]; \ No newline at end of file diff --git a/docs/functions_e.html b/docs/functions_e.html new file mode 100644 index 00000000..6fc4851a --- /dev/null +++ b/docs/functions_e.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- e -

+
+
+ + + + diff --git a/docs/functions_f.html b/docs/functions_f.html new file mode 100644 index 00000000..99e9e3b4 --- /dev/null +++ b/docs/functions_f.html @@ -0,0 +1,119 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- f -

+
+
+ + + + diff --git a/docs/functions_g.html b/docs/functions_g.html new file mode 100644 index 00000000..66991d9d --- /dev/null +++ b/docs/functions_g.html @@ -0,0 +1,155 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- g -

+
+
+ + + + diff --git a/docs/functions_h.html b/docs/functions_h.html new file mode 100644 index 00000000..709f4e64 --- /dev/null +++ b/docs/functions_h.html @@ -0,0 +1,123 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- h -

+
+
+ + + + diff --git a/docs/functions_i.html b/docs/functions_i.html new file mode 100644 index 00000000..f46d584e --- /dev/null +++ b/docs/functions_i.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/functions_j.html b/docs/functions_j.html new file mode 100644 index 00000000..70aeb320 --- /dev/null +++ b/docs/functions_j.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- j -

+
+
+ + + + diff --git a/docs/functions_k.html b/docs/functions_k.html new file mode 100644 index 00000000..ad2a4c13 --- /dev/null +++ b/docs/functions_k.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/functions_l.html b/docs/functions_l.html new file mode 100644 index 00000000..b5f8f3e4 --- /dev/null +++ b/docs/functions_l.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- l -

+
+
+ + + + diff --git a/docs/functions_m.html b/docs/functions_m.html new file mode 100644 index 00000000..f577c22c --- /dev/null +++ b/docs/functions_m.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- m -

+
+
+ + + + diff --git a/docs/functions_n.html b/docs/functions_n.html new file mode 100644 index 00000000..b9fff98f --- /dev/null +++ b/docs/functions_n.html @@ -0,0 +1,111 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- n -

+
+
+ + + + diff --git a/docs/functions_o.html b/docs/functions_o.html new file mode 100644 index 00000000..a6ee0fc1 --- /dev/null +++ b/docs/functions_o.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- o -

+
+
+ + + + diff --git a/docs/functions_p.html b/docs/functions_p.html new file mode 100644 index 00000000..31e75b6f --- /dev/null +++ b/docs/functions_p.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- p -

+
+
+ + + + diff --git a/docs/functions_q.html b/docs/functions_q.html new file mode 100644 index 00000000..e5e9744f --- /dev/null +++ b/docs/functions_q.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- q -

+
+
+ + + + diff --git a/docs/functions_r.html b/docs/functions_r.html new file mode 100644 index 00000000..d705a007 --- /dev/null +++ b/docs/functions_r.html @@ -0,0 +1,111 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- r -

+
+
+ + + + diff --git a/docs/functions_s.html b/docs/functions_s.html new file mode 100644 index 00000000..52cb4341 --- /dev/null +++ b/docs/functions_s.html @@ -0,0 +1,154 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- s -

+
+
+ + + + diff --git a/docs/functions_u.html b/docs/functions_u.html new file mode 100644 index 00000000..0481f0ec --- /dev/null +++ b/docs/functions_u.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- u -

+
+
+ + + + diff --git a/docs/functions_v.html b/docs/functions_v.html new file mode 100644 index 00000000..9ff764a6 --- /dev/null +++ b/docs/functions_v.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- v -

+
+
+ + + + diff --git a/docs/functions_vars.html b/docs/functions_vars.html new file mode 100644 index 00000000..b4bfd01b --- /dev/null +++ b/docs/functions_vars.html @@ -0,0 +1,120 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- _ -

+
+
+ + + + diff --git a/docs/functions_vars.js b/docs/functions_vars.js new file mode 100644 index 00000000..c151f2c8 --- /dev/null +++ b/docs/functions_vars.js @@ -0,0 +1,26 @@ +var functions_vars = +[ + [ "_", "functions_vars.html", null ], + [ "a", "functions_vars_a.html", null ], + [ "c", "functions_vars_c.html", null ], + [ "d", "functions_vars_d.html", null ], + [ "e", "functions_vars_e.html", null ], + [ "f", "functions_vars_f.html", null ], + [ "g", "functions_vars_g.html", null ], + [ "h", "functions_vars_h.html", null ], + [ "i", "functions_vars_i.html", null ], + [ "j", "functions_vars_j.html", null ], + [ "k", "functions_vars_k.html", null ], + [ "l", "functions_vars_l.html", null ], + [ "m", "functions_vars_m.html", null ], + [ "n", "functions_vars_n.html", null ], + [ "o", "functions_vars_o.html", null ], + [ "p", "functions_vars_p.html", null ], + [ "q", "functions_vars_q.html", null ], + [ "r", "functions_vars_r.html", null ], + [ "s", "functions_vars_s.html", null ], + [ "u", "functions_vars_u.html", null ], + [ "v", "functions_vars_v.html", null ], + [ "x", "functions_vars_x.html", null ], + [ "z", "functions_vars_z.html", null ] +]; \ No newline at end of file diff --git a/docs/functions_vars_a.html b/docs/functions_vars_a.html new file mode 100644 index 00000000..452dc0a3 --- /dev/null +++ b/docs/functions_vars_a.html @@ -0,0 +1,126 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- a -

+
+
+ + + + diff --git a/docs/functions_vars_c.html b/docs/functions_vars_c.html new file mode 100644 index 00000000..8eda7d8a --- /dev/null +++ b/docs/functions_vars_c.html @@ -0,0 +1,140 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- c -

+
+
+ + + + diff --git a/docs/functions_vars_d.html b/docs/functions_vars_d.html new file mode 100644 index 00000000..36ff68dc --- /dev/null +++ b/docs/functions_vars_d.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- d -

+
+
+ + + + diff --git a/docs/functions_vars_e.html b/docs/functions_vars_e.html new file mode 100644 index 00000000..30180a59 --- /dev/null +++ b/docs/functions_vars_e.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- e -

+
+
+ + + + diff --git a/docs/functions_vars_f.html b/docs/functions_vars_f.html new file mode 100644 index 00000000..29aa9240 --- /dev/null +++ b/docs/functions_vars_f.html @@ -0,0 +1,119 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- f -

+
+
+ + + + diff --git a/docs/functions_vars_g.html b/docs/functions_vars_g.html new file mode 100644 index 00000000..91a979e8 --- /dev/null +++ b/docs/functions_vars_g.html @@ -0,0 +1,155 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- g -

+
+
+ + + + diff --git a/docs/functions_vars_h.html b/docs/functions_vars_h.html new file mode 100644 index 00000000..83448f77 --- /dev/null +++ b/docs/functions_vars_h.html @@ -0,0 +1,123 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- h -

+
+
+ + + + diff --git a/docs/functions_vars_i.html b/docs/functions_vars_i.html new file mode 100644 index 00000000..13b0d3e1 --- /dev/null +++ b/docs/functions_vars_i.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- i -

+
+
+ + + + diff --git a/docs/functions_vars_j.html b/docs/functions_vars_j.html new file mode 100644 index 00000000..be4dfb86 --- /dev/null +++ b/docs/functions_vars_j.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- j -

+
+
+ + + + diff --git a/docs/functions_vars_k.html b/docs/functions_vars_k.html new file mode 100644 index 00000000..368f9502 --- /dev/null +++ b/docs/functions_vars_k.html @@ -0,0 +1,117 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- k -

+
+
+ + + + diff --git a/docs/functions_vars_l.html b/docs/functions_vars_l.html new file mode 100644 index 00000000..f9b57ca2 --- /dev/null +++ b/docs/functions_vars_l.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- l -

+
+
+ + + + diff --git a/docs/functions_vars_m.html b/docs/functions_vars_m.html new file mode 100644 index 00000000..cd52e6c6 --- /dev/null +++ b/docs/functions_vars_m.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- m -

+
+
+ + + + diff --git a/docs/functions_vars_n.html b/docs/functions_vars_n.html new file mode 100644 index 00000000..704a4d2d --- /dev/null +++ b/docs/functions_vars_n.html @@ -0,0 +1,111 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- n -

+
+
+ + + + diff --git a/docs/functions_vars_o.html b/docs/functions_vars_o.html new file mode 100644 index 00000000..f79fd564 --- /dev/null +++ b/docs/functions_vars_o.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- o -

+
+
+ + + + diff --git a/docs/functions_vars_p.html b/docs/functions_vars_p.html new file mode 100644 index 00000000..634d2ebe --- /dev/null +++ b/docs/functions_vars_p.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- p -

+
+
+ + + + diff --git a/docs/functions_vars_q.html b/docs/functions_vars_q.html new file mode 100644 index 00000000..d6543f1c --- /dev/null +++ b/docs/functions_vars_q.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- q -

+
+
+ + + + diff --git a/docs/functions_vars_r.html b/docs/functions_vars_r.html new file mode 100644 index 00000000..5aaea0a2 --- /dev/null +++ b/docs/functions_vars_r.html @@ -0,0 +1,111 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- r -

+
+
+ + + + diff --git a/docs/functions_vars_s.html b/docs/functions_vars_s.html new file mode 100644 index 00000000..72b5a7f3 --- /dev/null +++ b/docs/functions_vars_s.html @@ -0,0 +1,154 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- s -

+
+
+ + + + diff --git a/docs/functions_vars_u.html b/docs/functions_vars_u.html new file mode 100644 index 00000000..b40ce7b8 --- /dev/null +++ b/docs/functions_vars_u.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- u -

+
+
+ + + + diff --git a/docs/functions_vars_v.html b/docs/functions_vars_v.html new file mode 100644 index 00000000..a0ffe2bc --- /dev/null +++ b/docs/functions_vars_v.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- v -

+
+
+ + + + diff --git a/docs/functions_vars_x.html b/docs/functions_vars_x.html new file mode 100644 index 00000000..bdf7d62a --- /dev/null +++ b/docs/functions_vars_x.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- x -

+
+
+ + + + diff --git a/docs/functions_vars_z.html b/docs/functions_vars_z.html new file mode 100644 index 00000000..9006a13d --- /dev/null +++ b/docs/functions_vars_z.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields - Variables + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- z -

+
+
+ + + + diff --git a/docs/functions_x.html b/docs/functions_x.html new file mode 100644 index 00000000..dea9d49f --- /dev/null +++ b/docs/functions_x.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- x -

+
+
+ + + + diff --git a/docs/functions_z.html b/docs/functions_z.html new file mode 100644 index 00000000..14f7b685 --- /dev/null +++ b/docs/functions_z.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Data Fields + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all struct and union fields with links to the structures/unions they belong to:
+ +

- z -

+
+
+ + + + diff --git a/docs/globals.html b/docs/globals.html new file mode 100644 index 00000000..20b13a2d --- /dev/null +++ b/docs/globals.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- _ -

+
+
+ + + + diff --git a/docs/globals_a.html b/docs/globals_a.html new file mode 100644 index 00000000..4997a9e4 --- /dev/null +++ b/docs/globals_a.html @@ -0,0 +1,238 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- a -

+
+
+ + + + diff --git a/docs/globals_c.html b/docs/globals_c.html new file mode 100644 index 00000000..0a622664 --- /dev/null +++ b/docs/globals_c.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- c -

+
+
+ + + + diff --git a/docs/globals_d.html b/docs/globals_d.html new file mode 100644 index 00000000..5e91def7 --- /dev/null +++ b/docs/globals_d.html @@ -0,0 +1,111 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- d -

+
+
+ + + + diff --git a/docs/globals_defs.html b/docs/globals_defs.html new file mode 100644 index 00000000..4b47da4e --- /dev/null +++ b/docs/globals_defs.html @@ -0,0 +1,303 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- _ -

+ + +

- d -

+ + +

- i -

+ + +

- k -

+
+
+ + + + diff --git a/docs/globals_dup.js b/docs/globals_dup.js new file mode 100644 index 00000000..696bad0f --- /dev/null +++ b/docs/globals_dup.js @@ -0,0 +1,17 @@ +var globals_dup = +[ + [ "_", "globals.html", null ], + [ "a", "globals_a.html", null ], + [ "c", "globals_c.html", null ], + [ "d", "globals_d.html", null ], + [ "f", "globals_f.html", null ], + [ "g", "globals_g.html", null ], + [ "h", "globals_h.html", null ], + [ "i", "globals_i.html", null ], + [ "k", "globals_k.html", null ], + [ "m", "globals_m.html", null ], + [ "q", "globals_q.html", null ], + [ "s", "globals_s.html", null ], + [ "x", "globals_x.html", null ], + [ "z", "globals_z.html", null ] +]; \ No newline at end of file diff --git a/docs/globals_enum.html b/docs/globals_enum.html new file mode 100644 index 00000000..c29d22cf --- /dev/null +++ b/docs/globals_enum.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  +
+
+ + + + diff --git a/docs/globals_eval.html b/docs/globals_eval.html new file mode 100644 index 00000000..cf7fa0fe --- /dev/null +++ b/docs/globals_eval.html @@ -0,0 +1,253 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- i -

+
+
+ + + + diff --git a/docs/globals_f.html b/docs/globals_f.html new file mode 100644 index 00000000..76c87f7e --- /dev/null +++ b/docs/globals_f.html @@ -0,0 +1,114 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- f -

+
+
+ + + + diff --git a/docs/globals_func.html b/docs/globals_func.html new file mode 100644 index 00000000..e574cc92 --- /dev/null +++ b/docs/globals_func.html @@ -0,0 +1,226 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- a -

+
+
+ + + + diff --git a/docs/globals_func.js b/docs/globals_func.js new file mode 100644 index 00000000..25c1034d --- /dev/null +++ b/docs/globals_func.js @@ -0,0 +1,13 @@ +var globals_func = +[ + [ "a", "globals_func.html", null ], + [ "d", "globals_func_d.html", null ], + [ "f", "globals_func_f.html", null ], + [ "g", "globals_func_g.html", null ], + [ "i", "globals_func_i.html", null ], + [ "k", "globals_func_k.html", null ], + [ "m", "globals_func_m.html", null ], + [ "q", "globals_func_q.html", null ], + [ "s", "globals_func_s.html", null ], + [ "z", "globals_func_z.html", null ] +]; \ No newline at end of file diff --git a/docs/globals_func_d.html b/docs/globals_func_d.html new file mode 100644 index 00000000..89936bac --- /dev/null +++ b/docs/globals_func_d.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- d -

+
+
+ + + + diff --git a/docs/globals_func_f.html b/docs/globals_func_f.html new file mode 100644 index 00000000..d1fa5b40 --- /dev/null +++ b/docs/globals_func_f.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- f -

+
+
+ + + + diff --git a/docs/globals_func_g.html b/docs/globals_func_g.html new file mode 100644 index 00000000..9fc4df91 --- /dev/null +++ b/docs/globals_func_g.html @@ -0,0 +1,116 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- g -

+
+
+ + + + diff --git a/docs/globals_func_i.html b/docs/globals_func_i.html new file mode 100644 index 00000000..e21a557a --- /dev/null +++ b/docs/globals_func_i.html @@ -0,0 +1,121 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- i -

+
+
+ + + + diff --git a/docs/globals_func_k.html b/docs/globals_func_k.html new file mode 100644 index 00000000..02c142f1 --- /dev/null +++ b/docs/globals_func_k.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- k -

+
+
+ + + + diff --git a/docs/globals_func_m.html b/docs/globals_func_m.html new file mode 100644 index 00000000..66cd36ed --- /dev/null +++ b/docs/globals_func_m.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- m -

+
+
+ + + + diff --git a/docs/globals_func_q.html b/docs/globals_func_q.html new file mode 100644 index 00000000..81979bf3 --- /dev/null +++ b/docs/globals_func_q.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- q -

+
+
+ + + + diff --git a/docs/globals_func_s.html b/docs/globals_func_s.html new file mode 100644 index 00000000..dbf3094a --- /dev/null +++ b/docs/globals_func_s.html @@ -0,0 +1,158 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- s -

+
+
+ + + + diff --git a/docs/globals_func_z.html b/docs/globals_func_z.html new file mode 100644 index 00000000..e46e30ba --- /dev/null +++ b/docs/globals_func_z.html @@ -0,0 +1,110 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- z -

+
+
+ + + + diff --git a/docs/globals_g.html b/docs/globals_g.html new file mode 100644 index 00000000..90a98353 --- /dev/null +++ b/docs/globals_g.html @@ -0,0 +1,119 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- g -

+
+
+ + + + diff --git a/docs/globals_h.html b/docs/globals_h.html new file mode 100644 index 00000000..5f6b8079 --- /dev/null +++ b/docs/globals_h.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- h -

+
+
+ + + + diff --git a/docs/globals_i.html b/docs/globals_i.html new file mode 100644 index 00000000..e5d7e978 --- /dev/null +++ b/docs/globals_i.html @@ -0,0 +1,458 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- i -

+
+
+ + + + diff --git a/docs/globals_k.html b/docs/globals_k.html new file mode 100644 index 00000000..bfad4425 --- /dev/null +++ b/docs/globals_k.html @@ -0,0 +1,124 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- k -

+
+
+ + + + diff --git a/docs/globals_m.html b/docs/globals_m.html new file mode 100644 index 00000000..c1c41cfa --- /dev/null +++ b/docs/globals_m.html @@ -0,0 +1,112 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- m -

+
+
+ + + + diff --git a/docs/globals_q.html b/docs/globals_q.html new file mode 100644 index 00000000..efea2f5f --- /dev/null +++ b/docs/globals_q.html @@ -0,0 +1,113 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- q -

+
+
+ + + + diff --git a/docs/globals_s.html b/docs/globals_s.html new file mode 100644 index 00000000..1dfd5073 --- /dev/null +++ b/docs/globals_s.html @@ -0,0 +1,174 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- s -

+
+
+ + + + diff --git a/docs/globals_type.html b/docs/globals_type.html new file mode 100644 index 00000000..3cbdc694 --- /dev/null +++ b/docs/globals_type.html @@ -0,0 +1,217 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+  + +

- a -

+ + +

- c -

+ + +

- d -

+ + +

- f -

+ + +

- g -

+ + +

- h -

+ + +

- i -

+ + +

- k -

+ + +

- q -

+ + +

- s -

+ + +

- x -

+ + +

- z -

+
+
+ + + + diff --git a/docs/globals_x.html b/docs/globals_x.html new file mode 100644 index 00000000..3f779e49 --- /dev/null +++ b/docs/globals_x.html @@ -0,0 +1,109 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- x -

+
+
+ + + + diff --git a/docs/globals_z.html b/docs/globals_z.html new file mode 100644 index 00000000..8edf2451 --- /dev/null +++ b/docs/globals_z.html @@ -0,0 +1,115 @@ + + + + + + + +intel-ipsec-mb: Globals + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
Here is a list of all functions, variables, defines, enums, and typedefs with links to the files they belong to:
+ +

- z -

+
+
+ + + + diff --git a/docs/index.html b/docs/index.html index f938789f..f1590b03 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1,19 +1,603 @@ - - - - - - Intel® IPsec Multi-Buffer - - -
- Intel® Multi-Buffer Crypto for IPsec Library -
-
- Documentation coming soon... -
-
- For more information, please visit the intel-ipsec-mb GitHub project. + + + + + + + +intel-ipsec-mb: intel-ipsec-mb + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
intel-ipsec-mb
+
+
+

Coverity Status Linux Build Shared gcc Linux Build Static gcc Linux Build Shared clang Linux Build Static clang Linux Build Shared clang AESNI emulation FreeBSD Build Shared clang FreeBSD Build Shared gcc

+

+Intel(R) Multi-Buffer Crypto for IPsec Library

+

The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM.

+

The library is hosted on GitHub and is used as software crypto provider in DPDK, Intel(R) QAT Engine and FD.io.

+

Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The library can also be used directly through its native API.

+

Key differentiating features:

+
    +
  • operation chaining (encryption and authentication)
    • +
  • advanced cryptographic pipelining
      +
    • job manager with scheduling and dispatching functions
      • +
    • API hides underlying implementation details from an application
      • +
    +
    • +
  • multi-buffer and function stitching innovations
    • +
  • low level implementations using latest instruction extensions
    • +
+

+Contents

+
    +
  1. Overview
    2. +
  2. Processor Extensions
    3. +
  3. Recommendations
    4. +
  4. Package Content
    5. +
  5. Documentation
    6. +
  6. Compilation
    7. +
  7. Security Considerations & Options for Increased Security
    8. +
  8. Installation
    9. +
  9. Backwards compatibility
    10. +
  10. Disclaimer (ZUC, KASUMI, SNOW3G)
    11. +
  11. Legal Disclaimer
    12. +
+

+1. Overview

+

Intel Multi-Buffer Crypto for IPsec Library is highly-optimized software implementations of the core cryptographic processing for IPsec, which provides industry-leading performance on a range of Intel(R) Processors.

+

For information on how the library works, see the Intel White Paper: "Fast Multi-buffer IPsec Implementations on Intel Architecture Processors". Jim Guilford, Sean Gulley, et. al.

+

The easiest way to find it is to search the Internet for the title and Intel White Paper.

+

Table 1. List of supported cipher algorithms and their implementations.

+----------------------------------------------------------------------+
+
| | Implementation |
+
| Encryption +-----------------------------------------------------|
+
| | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(5)|
+
|----------------+--------+--------+--------+--------+--------+--------|
+
| AES128-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES192-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES256-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES128-CCM | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES256-CCM | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES128-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES192-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES256-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES128-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| AES192-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| AES256-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| NULL | Y | N | N | N | N | N |
+
| AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) |
+
| AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) |
+
| DES-DOCSIS | Y | N | N | N | Y x16 | N |
+
| 3DES | Y | N | N | N | Y x16 | N |
+
| DES | Y | N | N | N | Y x16 | N |
+
| KASUMI-F8 | Y | N | N | N | N | N |
+
| ZUC-EEA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| ZUC-EEA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| SNOW3G-UEA2 | N | Y x4 | Y | Y | Y x16 | Y x16 |
+
| AES128-CBCS(9) | N | Y(1) | Y(3) | N | N | Y(6) |
+
| Chacha20 | N | Y | Y | Y | Y | N |
+
| Chacha20 AEAD | N | Y | Y | Y | Y | N |
+
| SNOW-V | N | Y | Y | N | N | N |
+
| SNOW-V AEAD | N | Y | Y | N | N | N |
+
| PON-CRC-BIP | N | Y by8 | Y by8 | N | N | Y |
+
+----------------------------------------------------------------------+
+

Notes:
+ (1,2) - By default, decryption is by4 and encryption is x4.
+ On CPU's supporting GFNI, decryption is by8 and encryption is x8.
+ (3,4) - decryption is by8 and encryption is x8
+ (5) - AVX512 plus VAES, VPCLMULQDQ and GFNI extensions
+ (6) - decryption is by16 and encryption is x16
+ (7) - same as AES128-CBC for AVX, combines cipher and CRC32
+ (8) - decryption is by16 and encryption is x16
+ (9) - currently 1:9 crypt:skip pattern supported
+ (10) - by default, decryption and encryption are AVX by8.
+ On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16.
+

+

Legend:
+ byY - single buffer Y blocks at a time
+ xY - Y buffers at a time

+

As an example of how to read table 1 and 2, if one uses AVX512 interface to perform AES128-CBC encryption then there is no native AVX512 implementation for this cipher. In such case, the library uses best available implementation which is AVX for AES128-CBC.

+

Table 2. List of supported integrity algorithms and their implementations.

+-------------------------------------------------------------------------+
+
| | Implementation |
+
| Integrity +-----------------------------------------------------|
+
| | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(3)|
+
|-------------------+--------+--------+--------+--------+--------+--------|
+
| AES-XCBC-96 | N | Y x4 | Y x8 | N | N | Y x16 |
+
| HMAC-MD5-96 | Y(1) | Y x4x2 | Y x4x2 | Y x8x2 | N | N |
+
| HMAC-SHA1-96 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-224_112 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| NULL | Y | N | N | N | N | N |
+
| AES128-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES256-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES128-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES256-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| KASUMI-F9 | Y | N | N | N | N | N |
+
| ZUC-EIA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| ZUC-EIA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| SNOW3G-UIA2(8) | N | Y by4 | Y by4 | N | Y by32 | Y by32 |
+
| DOCSIS-CRC32(4) | N | Y | Y | N | Y | Y |
+
| HEC | N | Y | Y | N | N | N |
+
| POLY1305 | Y | N | N | N | Y | Y |
+
| POLY1305 AEAD | Y | N | N | N | Y | Y |
+
| SNOW-V AEAD | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| GHASH | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| CRC(6) | N | Y by8 | Y by8 | N | N | Y by16 |
+
| PON-CRC-BIP(7) | N | Y | Y | N | N | Y |
+
+-------------------------------------------------------------------------+
+

Notes:
+ (1) - MD5 over one block implemented in C
+ (2) - Implementation using SHANI extensions is x2
+ (3) - AVX512 plus VAES, VPCLMULQDQ, GFNI and IFMA extensions
+ (4) - used only with AES256-DOCSIS and AES128-DOCSIS ciphers
+ (5) - x8 on selected CPU's supporting GFNI
+ (6) - Supported CRC types:

    +
  • CRC32: Ethernet FCS, SCTP, WIMAX OFDMA
    +
    • +
  • CRC24: LTE A, LTE B
    +
    • +
  • CRC16: X25, FP data
    +
    • +
  • CRC11: FP header
    +
    • +
  • CRC10: IUUP data
    +
    • +
  • CRC8: WIMAX OFDMA HCS
    +
    • +
  • CRC7: FP header
    +
    • +
  • CRC6: IUUP header
    + (7) - used only with PON-AES128-CTR cipher
    + (8) - x4/x16 for init keystream generation, then by4/by32
    +
    • +
+

Legend:
+ byY- single buffer Y blocks at a time
+ xY- Y buffers at a time
+

+

Table 3. Encryption and integrity algorithm combinations

+---------------------------------------------------------------------+
+
| Encryption | Allowed Integrity Algorithms |
+
|---------------+-----------------------------------------------------|
+
| AES128-GCM | AES128-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES192-GCM | AES192-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES256-GCM | AES256-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES128-CCM | AES128-CCM |
+
|---------------+-----------------------------------------------------|
+
| AES256-CCM | AES256-CCM |
+
|---------------+-----------------------------------------------------|
+
| AES128-CBC, | AES-XCBC-96, |
+
| AES192-CBC, | HMAC-SHA1-96, HMAC-SHA2-224_112, HMAC-SHA2-256_128, |
+
| AES256-CBC, | HMAC-SHA2-384_192, HMAC-SHA2-512_256, |
+
| AES128-CTR, | AES128-CMAC-96, |
+
| AES192-CTR, | NULL, |
+
| AES256-CTR, | KASUMI-F9, |
+
| AES128-ECB, | ZUC-EIA3, ZUC-EIA3-256, |
+
| AES192-ECB, | SNOW3G-UIA3, |
+
| AES256-ECB, | POLY1305, |
+
| NULL, | AES128-GMAC, AES192-GMAC, AES256-GMAC, GHASH |
+
| AES128-DOCSIS,| |
+
| AES256-DOCSIS,| |
+
| DES-DOCSIS, | |
+
| 3DES, | |
+
| DES, | |
+
| Chacha20, | |
+
| KASUMI-F8, | |
+
| ZUC-EEA3, | |
+
| ZUC-EEA3-256, | |
+
| SNOW3G-UEA3 | |
+
| SNOW-V | |
+
|---------------+-----------------------------------------------------|
+
| AES128-DOCSIS,| DOCSIS-CRC32 |
+
| AES256-DOCSIS | |
+
|---------------+-----------------------------------------------------|
+
| PON-AES128-CTR| PON-CRC-BIP |
+
|---------------+-----------------------------------------------------|
+
| CHACHA20 AEAD | POLY1305 AEAD |
+
+---------------+-----------------------------------------------------+
+
| SNOW-V AEAD | SNOW-V AEAD (GHASH) |
+
+---------------+-----------------------------------------------------+
+

+2. Processor Extensions

+

Table 4. Processor extensions used in the library

+-------------------------------------------------------------------------+
+
| Algorithm | Interface | Extensions |
+
|-------------------+-----------+-----------------------------------------|
+
| HMAC-SHA1-96, | AVX512 | AVX512F, AVX512BW, AVX512VL |
+
| HMAC-SHA2-224_112,| | |
+
| HMAC-SHA2-256_128,| | |
+
| HMAC-SHA2-384_192,| | |
+
| HMAC-SHA2-512_256 | | |
+
|-------------------+-----------+-----------------------------------------|
+
| DES, 3DES, | AVX512 | AVX512F, AVX512BW |
+
| DOCSIS-DES | | |
+
|-------------------+-----------+-----------------------------------------|
+
| HMAC-SHA1-96, | SSE | SHANI |
+
| HMAC-SHA2-224_112,| | - presence is autodetected and library |
+
| HMAC-SHA2-256_128,| | falls back to SSE implementation |
+
| HMAC-SHA2-384_192,| | if not present |
+
| HMAC-SHA2-512_256 | | |
+
+-------------------+-----------+-----------------------------------------+
+

+3. Recommendations

+

Legacy or to be avoided algorithms listed in the table below are implemented in the library in order to support legacy applications. Please use corresponding alternative algorithms instead.

+--------------------------------------------------------------+
+
| # | Algorithm | Recommendation | Alternative |
+
|---+---------------------+----------------+-------------------|
+
| 1 | DES encryption | Avoid | AES encryption |
+
|---+---------------------+----------------+-------------------|
+
| 2 | 3DES encryption | Avoid | AES encryption |
+
|---+---------------------+----------------+-------------------|
+
| 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA256 |
+
|---+---------------------+----------------+-------------------|
+
| 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR |
+
|---+---------------------+----------------+-------------------|
+
| 3 | HMAC-SHA1 integrity | Avoid | HMAC-SHA256 |
+
+--------------------------------------------------------------+
+

Intel(R) Multi-Buffer Crypto for IPsec Library depends on C library and it is recommended to use its latest version.

+

Applications using the Intel(R) Multi-Buffer Crypto for IPsec Library rely on Operating System to provide process isolation. As the result, it is recommended to use latest Operating System patches and security updates.

+

+4. Package Content

+
    +
  • test - Library test applications
    • +
  • perf - Library performance application
    • +
  • lib - Library source files
    • +
  • lib/sse - Intel(R) SSE optimized routines
    • +
  • lib/avx - Intel(R) AVX optimized routines
    • +
  • lib/avx2 - Intel(R) AVX2 optimized routines
    • +
  • lib/avx512 - Intel(R) AVX512 optimized routines
    • +
  • lib/no-aesni - Non-AESNI accelerated routines
    • +
+

Note:
+ There is just one branch used in the project. All development is done on the master branch.
+ Code taken from the tip of the master branch should not be considered fit for production.
+

+

Refer to the releases tab for stable code versions:
+ https://github.com/intel/intel-ipsec-mb/releases

+

+5. Documentation

+

Full documentation can be found at: https://intel.github.io/intel-ipsec-mb

+

To generate documentation locally, run:
+ > make doxy

+

+6. Compilation

+

+Linux (64-bit only)

+

Required tools:
+

    +
  • GNU make
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 4.8.3 (or newer)
    +
    • +
+

Shared library:
+ > make

+

Static library:
+ > make SHARED=n

+

Clean the build:
+ > make clean
+ or
+ > make clean SHARED=n

+

Build with debugging information:
+ > make DEBUG=y

+

Build with AESNI emulation support (disabled by default):
+ > make AESNI_EMU=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > make help

+

+Windows MSVS (x64 only)

+

Required tools:
+

    +
  • Microsoft (R) Visual Studio 2015:
    +
      +
    • NMAKE: Microsoft (R) Program Maintenance Utility Version 14.00.24210.0
      +
      • +
    • CL: Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24215.1 for x64
      +
      • +
    • LIB: Microsoft (R) Library Manager Version 14.00.24215.1
      +
      • +
    • LINK: Microsoft (R) Incremental Linker Version 14.00.24215.1
      +
      • +
    • Note: Building on later versions should work but is not verified
      +
      • +
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
+

Shared library (DLL):
+ > nmake /f win_x64.mak

+

Static library:
+ > nmake /f win_x64.mak SHARED=n

+

Clean the build:
+ > nmake /f win_x64.mak clean
+ or
+ > nmake /f win_x64.mak clean SHARED=n

+

Build without safety features:
+

    +
  • SAFE_DATA clears sensitive information stored temporarily on stack, registers or internal data structures
    +
    • +
  • SAFE_PARAM adds extra checks on input parameters
    +
    • +
  • SAFE_LOOKUP uses constant-time lookups (enabled by default)
    +
    • +
  • SAFE_OPTIONS additional option to disable all safe options. Enabled by default.
    + Disable to turn off: SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP.
    +
    • +
+

> nmake /f win_x64.mak SAFE_DATA=n SAFE_PARAM=n > nmake /f win_x64.mak SAFE_OPTIONS=n

+

Build with debugging information:
+ > nmake /f win_x64.mak DEBUG=y

+

Build with AESNI emulation support (disabled by default):
+ > nmake /f win_x64.mak AESNI_EMU=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > nmake /f win_x64.mak help

+

+Windows Mingw-w64 (64-bit only)

+

Required tools:
+

    +
  • GNU mingw32-make.exe
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 10.3.0 (or newer)
    • +
+

Shared library:
+ > mingw32-make.exe

+

Static library:
+ > mingw32-make.exe SHARED=n

+

Clean the build:
+ > mingw32-make.exe clean
+ or
+ > mingw32-make.exe clean SHARED=n

+

Build with debugging information:
+ > mingw32-make.exe DEBUG=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > mingw32-make.exe help

+

+FreeBSD (64-bit only)

+

Required tools:
+

    +
  • GNU make
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 4.8.3 (or newer) / clang 5.0 (or newer)
    +
    • +
+

Shared library:
+ > gmake

+

Static library:
+ > gmake SHARED=n

+

Clean the build:
+ > gmake clean
+ or
+ > gmake clean SHARED=n

+

Build with debugging information:
+ > gmake DEBUG=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > gmake help

+

+7. Security Considerations & Options for Increased Security

+

+Security Considerations

+

The security of a system that uses cryptography depends on the strength of the cryptographic algorithms as well as the strength of the keys. Cryptographic key strength is dependent on several factors, with some of the most important factors including the length of the key, the entropy of the key bits, and maintaining the secrecy of the key.

+

The selection of an appropriate algorithm and mode of operation critically affects the security of a system. Appropriate selection criteria is beyond the scope of this document and should be determined based upon usage, appropriate standards and consultation with a cryptographic expert. This library includes some algorithms, which are considered cryptographically weak and are included only for legacy and interoperability reasons. See the "Recommendations" section for more details.

+

Secure creation of key material is not a part of this library. This library assumes that cryptographic keys have been created using approved methods with an appropriate and secure entropy source. Users of this library are referred to NIST SP800-133 Revision 1, Recommendation for Cryptographic Key Generation, found at https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-133r1.pdf

+

Even with the use of strong cryptographic algorithms and robustly generated keys, software implementations of cryptographic algorithms may be attacked at the implementation through cache-timing attacks, buffer-over-reads, and other software vulnerabilities. Counter-measures against these types of attacks are possible but require additional processing cycles. Whether a particular system should provide such counter-measures depends on the threats to that system, and cannot be determined by a general library such as this one. In order to provide the most flexible implementation, this library allows certain counter-measures to be enabled or disabled at compile time. These options are listed below as the "Options for Increased Security" and are enabled through various build flags.

+

+Options for Increased Security

+

There are three build options that are used to increase safety in the code and help protect external functions from incorrect input data. The SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP options are enabled by default. Due to the potential performance impact associated to the extra code, these can be disabled by setting the parameter equal to "n" (e.g. make SAFE_LOOKUP=n).

+

No specific code has been added, and no specific validation or security tests have been performed to help protect against or check for side-channel attacks.

+

+SAFE_DATA

+

Stack and registers containing sensitive information, such as keys or IVs, are cleared upon completion of a function call.

+

+SAFE_PARAM

+

Input parameters are checked, looking generally for NULL pointers or an incorrect input length.

+

+SAFE_LOOKUP

+

Lookups which depend on sensitive information are implemented with constant time functions.

+

Algorithms where these constant time functions are used are the following:
+

    +
  • AESNI emulation
    +
    • +
  • DES: SSE, AVX and AVX2 implementations
    +
    • +
  • KASUMI: all architectures
    +
    • +
  • SNOW3G: all architectures
    +
    • +
+

If SAFE_LOOKUP is not enabled in the build (e.g. make SAFE_LOOKUP=n) then the algorithms listed above may be susceptible to timing attacks which could expose the cryptographic key.

+

+SAFE_OPTIONS

+

SAFE_OPTIONS is a parameter that can be used to disable/enable all supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). It is set to y by default and all safe options are enabled. SAFE_OPTIONS=n disables all safe options.

+

+Security API

+

Force clearing/zeroing of memory

IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size);
+
IMB_DLL_EXPORT
#define IMB_DLL_EXPORT
Definition: intel-ipsec-mb.h:86
+
imb_clear_mem
IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size)
Force clearing/zeroing of memory.
+

To assist in clearing sensitive application data such as keys, plaintext etc. the library provides the imb_clear_mem() API. This API zeros _'size'_ bytes of memory pointed to by _'mem'_ followed by the sfence instruction to ensure memory is cleared before the function returns.

+

+Galois Counter Mode (GCM) TAG Size

+

The library GCM and GMAC implementation provides flexibility as to tag size selection. As explained in NIST Special Publication 800-38D section 5.2.1.2 and Appendix C, using tag sizes shorter than 96 bits can be insecure. Please refer to the aforementioned sections to understand the details, trade offs and mitigations of using shorter tag sizes.

+

+8. Installation

+

+Linux (64-bit only)

+

First compile the library and then install:
+ > make
+ > sudo make install

+

To uninstall the library run:
+ > sudo make uninstall

+

If you want to change install location then define PREFIX:
+ > sudo make install PREFIX=<path>

+

If there is no need to run ldconfig at install stage please use NOLDCONFIG=y option:
+ > sudo make install NOLDCONFIG=y

+

If library was compiled as an archive (not a default option) then install it using SHARED=n option:
+ > sudo make install SHARED=n

+

+Windows (x64 only)

+

First compile the library and then install from a command prompt in administrator mode:
+ > nmake /f win_x64.mak
+ > nmake /f win_x64.mak install

+

To uninstall the library run:
+ > nmake /f win_x64.mak uninstall

+

If you want to change install location then define PREFIX (default C:\Program Files):
+ > nmake /f win_x64.mak install PREFIX=<path>

+

If library was compiled as a static library (not a default option) then install it using SHARED=n option:
+ > nmake /f win_x64.mak install SHARED=n

+

+FreeBSD (64-bit only)

+

First compile the library and then install:
+ > gmake
+ > sudo gmake install

+

To uninstall the library run:
+ > sudo gmake uninstall

+

If you want to change install location then define PREFIX:
+ > sudo gmake install PREFIX=<path>

+

If there is no need to run ldconfig at install stage please use NOLDCONFIG=y option:
+ > sudo gmake install NOLDCONFIG=y

+

If library was compiled as an archive (not a default option) then install it using SHARED=n option:
+ > sudo gmake install SHARED=n

+

+9. Backwards compatibility

+

In version 0.54, some symbols have been renamed to avoid too generic names (such as cipher modes or hash algorithms).

+

When building an application and linking it against the IPSec Multi Buffer library, by default the old symbols (up to v0.53) are exported, to maintain backwards compatibility.

+

Applications should move to the new API as soon as possible, as the old symbols are marked as deprecated and will be removed in a future release.

+

For applications which face symbol conflicts due to these old generic names, they should be compiled with the flag -DNO_COMPAT_IMB_API_053, which will not export the old symbols.

+

+10. Disclaimer (ZUC, KASUMI, SNOW3G)

+

Please note that cryptographic material, such as ciphering algorithms, may be subject to national regulations. What is more, use of some algorithms in real networks and production equipment can be subject to agreement or licensing by the GSMA and/or the ETSI.

+

For more details please see:
+

+

+11. Legal Disclaimer

+

THIS SOFTWARE IS PROVIDED BY INTEL"AS IS". NO LICENSE, EXPRESS OR
+ IMPLIED, BY ESTOPPEL OR OTHERWISE, TO ANY INTELLECTUAL PROPERTY RIGHTS
+ ARE GRANTED THROUGH USE. EXCEPT AS PROVIDED IN INTEL'S TERMS AND
+ CONDITIONS OF SALE, INTEL ASSUMES NO LIABILITY WHATSOEVER AND INTEL
+ DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR
+ USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO
+ FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT
+ OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT.
+

+
+
+
+ + + diff --git a/docs/intel-ipsec-mb_8h.html b/docs/intel-ipsec-mb_8h.html new file mode 100644 index 00000000..daa60735 --- /dev/null +++ b/docs/intel-ipsec-mb_8h.html @@ -0,0 +1,19646 @@ + + + + + + + +intel-ipsec-mb: intel-ipsec-mb.h File Reference + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
+Data Structures | +Macros | +Typedefs | +Enumerations | +Functions
+
intel-ipsec-mb.h File Reference
+
+
+
#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+
+

Go to the source code of this file.

+ + + + + + + + + + + + + + + + + + + + + + + +

+Data Structures

struct  imb_uint128_t
 
struct  IMB_SGL_IOV
 
struct  IMB_JOB
 
struct  kasumi_key_sched_s
 
struct  gcm_context_data
 holds GCM operation context More...
 
struct  chacha20_poly1305_context_data
 holds Chacha20-Poly1305 operation context More...
 
struct  gcm_key_data
 holds intermediate key data needed to improve performance More...
 
struct  snow3g_key_schedule_s
 
struct  IMB_MGR
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Macros

#define DECLARE_ALIGNED(decl, alignval)    __declspec(align(alignval)) decl
 
#define __forceinline    static __forceinline
 
#define IMB_DLL_EXPORT
 
#define IMB_DLL_LOCAL
 
#define IMB_VERSION_STR   "1.2.0"
 
#define IMB_VERSION_NUM   0x10200
 
#define IMB_VERSION(a, b, c)   (((a) << 16) + ((b) << 8) + (c))
 
#define IMB_ASSERT(x)
 
#define IMB_DIM(x)   (sizeof(x) / sizeof(x[0]))
 
#define IMB_DES_KEY_SCHED_SIZE   (16 * 8)
 
#define IMB_DES_BLOCK_SIZE   8
 
#define IMB_AES_BLOCK_SIZE   16
 
#define IMB_SHA1_DIGEST_SIZE_IN_BYTES   20
 
#define IMB_SHA224_DIGEST_SIZE_IN_BYTES   28
 
#define IMB_SHA256_DIGEST_SIZE_IN_BYTES   32
 
#define IMB_SHA384_DIGEST_SIZE_IN_BYTES   48
 
#define IMB_SHA512_DIGEST_SIZE_IN_BYTES   64
 
#define IMB_SHA1_BLOCK_SIZE   64
 
#define IMB_SHA_256_BLOCK_SIZE   64
 
#define IMB_SHA_384_BLOCK_SIZE   128
 
#define IMB_SHA_512_BLOCK_SIZE   128
 
#define IMB_KASUMI_KEY_SIZE   16
 
#define IMB_KASUMI_IV_SIZE   8
 
#define IMB_KASUMI_BLOCK_SIZE   8
 
#define IMB_KASUMI_DIGEST_SIZE   4
 
#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE   14
 
#define IMB_DOCSIS_CRC32_TAG_SIZE   4
 
#define KASUMI_KEY_SCHEDULE_SIZE   64
 
#define IMB_GCM_BLOCK_LEN   16
 
#define IMB_MAX_TAG_LEN   (16)
 
#define IMB_GCM_IV_DATA_LEN   (12)
 
#define IMB_GCM_128_KEY_LEN   (16)
 
#define IMB_GCM_192_KEY_LEN   (24)
 
#define IMB_GCM_256_KEY_LEN   (32)
 
#define IMB_GCM_ENC_KEY_LEN   16
 
#define IMB_GCM_KEY_SETS   (15)
 
#define IMB_FLAG_SHANI_OFF   (1ULL << 0)
 
#define IMB_FLAG_AESNI_OFF   (1ULL << 1)
 
#define IMB_FEATURE_SHANI   (1ULL << 0)
 
#define IMB_FEATURE_AESNI   (1ULL << 1)
 
#define IMB_FEATURE_PCLMULQDQ   (1ULL << 2)
 
#define IMB_FEATURE_CMOV   (1ULL << 3)
 
#define IMB_FEATURE_SSE4_2   (1ULL << 4)
 
#define IMB_FEATURE_AVX   (1ULL << 5)
 
#define IMB_FEATURE_AVX2   (1ULL << 6)
 
#define IMB_FEATURE_AVX512F   (1ULL << 7)
 
#define IMB_FEATURE_AVX512DQ   (1ULL << 8)
 
#define IMB_FEATURE_AVX512CD   (1ULL << 9)
 
#define IMB_FEATURE_AVX512BW   (1ULL << 10)
 
#define IMB_FEATURE_AVX512VL   (1ULL << 11)
 
#define IMB_FEATURE_AVX512_SKX
 
#define IMB_FEATURE_VAES   (1ULL << 12)
 
#define IMB_FEATURE_VPCLMULQDQ   (1ULL << 13)
 
#define IMB_FEATURE_SAFE_DATA   (1ULL << 14)
 
#define IMB_FEATURE_SAFE_PARAM   (1ULL << 15)
 
#define IMB_FEATURE_GFNI   (1ULL << 16)
 
#define IMB_FEATURE_AVX512_IFMA   (1ULL << 17)
 
#define IMB_FEATURE_BMI2   (1ULL << 18)
 
#define IMB_FEATURE_AESNI_EMU   (1ULL << 19)
 
#define IMB_CPUFLAGS_NO_AESNI   (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
 
#define IMB_CPUFLAGS_SSE
 
#define IMB_CPUFLAGS_AVX   (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
 
#define IMB_CPUFLAGS_AVX2
 
#define IMB_CPUFLAGS_AVX512   (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
 
#define IMB_MAX_JOBS   128
 
#define IMB_GET_NEXT_JOB(_mgr)   ((_mgr)->get_next_job((_mgr)))
 Get next available job. More...
 
#define IMB_SUBMIT_JOB(_mgr)   ((_mgr)->submit_job((_mgr)))
 Submit job for processing after validating. More...
 
#define IMB_SUBMIT_JOB_NOCHECK(_mgr)   ((_mgr)->submit_job_nocheck((_mgr)))
 Submit job for processing without validating. More...
 
#define IMB_GET_COMPLETED_JOB(_mgr)   ((_mgr)->get_completed_job((_mgr)))
 Get next completed job. More...
 
#define IMB_FLUSH_JOB(_mgr)   ((_mgr)->flush_job((_mgr)))
 Force processing until next job in queue is completed. More...
 
#define IMB_QUEUE_SIZE(_mgr)   ((_mgr)->queue_size((_mgr)))
 Get number of jobs queued to be processed. More...
 
#define IMB_SUBMIT_BURST(_mgr, _jobs, _n_jobs)    ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs)))
 
#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs)    ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs)))
 
#define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, _dir, _key_size)
 
#define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, _dir, _key_size)
 
#define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash)    ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
 
#define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash)    ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
 
#define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_KEYEXP_192(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_KEYEXP_256(_mgr, _key, _enc_exp_key, _dec_exp_key)    ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
 
#define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _exp_key, _key1, _key2)    ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
 
#define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _exp_key, _key1, _key2)    ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
 
#define IMB_AES_XCBC_KEYEXP(_mgr, _key, _exp_key, _exp_key2, _exp_key3)    ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
 
#define IMB_DES_KEYSCHED(_mgr, _exp_key, _key)    ((_mgr)->des_key_sched((_exp_key), (_key)))
 
#define IMB_SHA1_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha1_one_block((_src), (_tag)))
 
#define IMB_SHA1(_mgr, _src, _length, _tag)    ((_mgr)->sha1((_src), (_length), (_tag)))
 
#define IMB_SHA224_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha224_one_block((_src), (_tag)))
 
#define IMB_SHA224(_mgr, _src, _length, _tag)    ((_mgr)->sha224((_src), (_length), (_tag)))
 
#define IMB_SHA256_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha256_one_block((_src), (_tag)))
 
#define IMB_SHA256(_mgr, _src, _length, _tag)    ((_mgr)->sha256((_src), (_length), (_tag)))
 
#define IMB_SHA384_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha384_one_block((_src), (_tag)))
 
#define IMB_SHA384(_mgr, _src, _length, _tag)    ((_mgr)->sha384((_src), (_length), (_tag)))
 
#define IMB_SHA512_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->sha512_one_block((_src), (_tag)))
 
#define IMB_SHA512(_mgr, _src, _length, _tag)    ((_mgr)->sha512((_src), (_length), (_tag)))
 
#define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag)    ((_mgr)->md5_one_block((_src), (_tag)))
 
#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len)    ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
 AES-CFB-128 Encrypt/Decrypt up to one block. More...
 
#define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES192_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES256_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES128_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES192_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES256_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, _aadl, _tag, _tagl)
 
#define IMB_AES128_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES192_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES256_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl)    ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
 
#define IMB_AES128_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES192_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES256_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, _aadl)
 
#define IMB_AES128_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES192_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES256_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES128_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES192_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES256_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len)    ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
 
#define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES192_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES256_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl)    ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
 
#define IMB_AES128_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES192_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES256_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len)    ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
 
#define IMB_AES128_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES192_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES256_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl)    ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
 
#define IMB_AES128_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm128_precomp((_key)))
 
#define IMB_AES192_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm192_precomp((_key)))
 
#define IMB_AES256_GCM_PRECOMP(_mgr, _key)    ((_mgr)->gcm256_precomp((_key)))
 
#define IMB_AES128_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm128_pre((_key), (_exp_key)))
 
#define IMB_AES192_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm192_pre((_key), (_exp_key)))
 
#define IMB_AES256_GCM_PRE(_mgr, _key, _exp_key)    ((_mgr)->gcm256_pre((_key), (_exp_key)))
 
#define IMB_GHASH_PRE(_mgr, _key, _exp_key)    ((_mgr)->ghash_pre((_key), (_exp_key)))
 
#define IMB_GHASH(_mgr, _exp_key, _src, _len, _tag, _tagl)    ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
 
#define IMB_CHACHA20_POLY1305_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl)
 
#define IMB_CHACHA20_POLY1305_ENC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len)
 
#define IMB_CHACHA20_POLY1305_DEC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len)
 
#define IMB_CHACHA20_POLY1305_ENC_FINALIZE(_mgr, _ctx, _tag, _tagl)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
 
#define IMB_CHACHA20_POLY1305_DEC_FINALIZE(_mgr, _ctx, _tag, _tagl)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
 
#define IMB_ZUC_EEA3_1_BUFFER(_mgr, _key, _iv, _src, _dst, _len)    ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
 ZUC EEA3 Confidentiality functions. More...
 
#define IMB_ZUC_EEA3_4_BUFFER(_mgr, _key, _iv, _src, _dst, _len)    ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
 
#define IMB_ZUC_EEA3_N_BUFFER(_mgr, _key, _iv, _src, _dst, _len, _count)    ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
 
#define IMB_ZUC_EIA3_1_BUFFER(_mgr, _key, _iv, _src, _len, _tag)    ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
 ZUC EIA3 Integrity function. More...
 
#define IMB_ZUC_EIA3_N_BUFFER(_mgr, _key, _iv, _src, _len, _tag, _count)    ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
 
#define IMB_KASUMI_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len)    ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
 Kasumi byte-level f8 operation on a single buffer. More...
 
#define IMB_KASUMI_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, _offset)
 Kasumi bit-level f8 operation on a single buffer. More...
 
#define IMB_KASUMI_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, _len1, _src2, _dst2, _len2)
 Kasumi byte-level f8 operation in parallel on two buffers. More...
 
#define IMB_KASUMI_F8_3_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _src1, _dst1, _src2, _dst2, _src3, _dst3, _len)
 kasumi byte-level f8 operation in parallel on three buffers More...
 
#define IMB_KASUMI_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _src1, _dst1, _src2, _dst2, _src3, _dst3, _src4, _dst4, _len)
 kasumi byte-level f8 operation in parallel on four buffers More...
 
#define IMB_KASUMI_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 Kasumi f8 operation on N buffers. More...
 
#define IMB_KASUMI_F9_1_BUFFER(_mgr, _exp_key, _src, _len, _tag)    ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
 Kasumi bit-level f9 operation on a single buffer. More...
 
#define IMB_KASUMI_F9_1_BUFFER_USER(_mgr, _exp_key, _iv, _src, _len, _tag, _dir)
 Kasumi bit-level f9 operation on a single buffer. More...
 
#define IMB_KASUMI_INIT_F8_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
 
#define IMB_KASUMI_INIT_F9_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
 
#define IMB_KASUMI_KEY_SCHED_SIZE(_mgr)   ((_mgr)->kasumi_key_sched_size())
 
#define IMB_SNOW3G_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, _offset)
 
#define IMB_SNOW3G_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len)    ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
 
#define IMB_SNOW3G_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, _len1, _src2, _dst2, _len2)
 
#define IMB_SNOW3G_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _src1, _dst1, _len1, _src2, _dst2, _len2, _src3, _dst3, _len3, _src4, _dst4, _len4)
 
#define IMB_SNOW3G_F8_8_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, _iv5, _iv6, _iv7, _iv8, _src1, _dst1, _len1, _src2, _dst2, _len2, _src3, _dst3, _len3, _src4, _dst4, _len4, _src5, _dst5, _len5, _src6, _dst6, _len6, _src7, _dst7, _len7, _src8, _dst8, _len8)
 
#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len)
 
#define IMB_SNOW3G_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 
#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len, _count)
 
#define IMB_SNOW3G_F9_1_BUFFER(_mgr, _exp_key, _iv, _src, _len, _tag)    ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
 
#define IMB_SNOW3G_INIT_KEY_SCHED(_mgr, _key, _exp_key)    ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
 
#define IMB_SNOW3G_KEY_SCHED_SIZE(_mgr)   ((_mgr)->snow3g_key_sched_size())
 
#define IMB_HEC_32(_mgr, _src)   ((_mgr)->hec_32(_src))
 
#define IMB_HEC_64(_mgr, _src)   ((_mgr)->hec_64(_src))
 
#define IMB_CRC32_ETHERNET_FCS(_mgr, _src, _len)    (_mgr)->crc32_ethernet_fcs(_src, _len)
 
#define IMB_CRC16_X25(_mgr, _src, _len)    (_mgr)->crc16_x25(_src, _len)
 
#define IMB_CRC32_SCTP(_mgr, _src, _len)    (_mgr)->crc32_sctp(_src, _len)
 
#define IMB_CRC24_LTE_A(_mgr, _src, _len)    (_mgr)->crc24_lte_a(_src, _len)
 
#define IMB_CRC24_LTE_B(_mgr, _src, _len)    (_mgr)->crc24_lte_b(_src, _len)
 
#define IMB_CRC16_FP_DATA(_mgr, _src, _len)    (_mgr)->crc16_fp_data(_src, _len)
 
#define IMB_CRC11_FP_HEADER(_mgr, _src, _len)    (_mgr)->crc11_fp_header(_src, _len)
 
#define IMB_CRC7_FP_HEADER(_mgr, _src, _len)    (_mgr)->crc7_fp_header(_src, _len)
 
#define IMB_CRC10_IUUP_DATA(_mgr, _src, _len)    (_mgr)->crc10_iuup_data(_src, _len)
 
#define IMB_CRC6_IUUP_HEADER(_mgr, _src, _len)    (_mgr)->crc6_iuup_header(_src, _len)
 
#define IMB_CRC32_WIMAX_OFDMA_DATA(_mgr, _src, _len)    (_mgr)->crc32_wimax_ofdma_data(_src, _len)
 
#define IMB_CRC8_WIMAX_OFDMA_HCS(_mgr, _src, _len)    (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Typedefs

typedef struct IMB_JOB IMB_JOB
 
typedef struct kasumi_key_sched_s kasumi_key_sched_t
 
typedef void(* init_mb_mgr_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* get_next_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* submit_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* get_completed_job_t) (struct IMB_MGR *)
 
typedef IMB_JOB *(* flush_job_t) (struct IMB_MGR *)
 
typedef uint32_t(* queue_size_t) (struct IMB_MGR *)
 
typedef uint32_t(* submit_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t)
 
typedef uint32_t(* submit_cipher_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
 
typedef uint32_t(* submit_hash_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
 
typedef void(* keyexp_t) (const void *, void *, void *)
 
typedef void(* cmac_subkey_gen_t) (const void *, void *, void *)
 
typedef void(* hash_one_block_t) (const void *, void *)
 
typedef void(* hash_fn_t) (const void *, const uint64_t, void *)
 
typedef void(* xcbc_keyexp_t) (const void *, void *, void *, void *)
 
typedef int(* des_keysched_t) (uint64_t *, const void *)
 
typedef void(* aes_cfb_t) (void *, const void *, const void *, const void *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
 
typedef void(* aes_gcm_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
 
typedef void(* aes_gcm_init_var_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
 
typedef void(* aes_gcm_enc_dec_update_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
 
typedef void(* aes_gcm_enc_dec_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
 
typedef void(* aes_gcm_precomp_t) (struct gcm_key_data *)
 
typedef void(* aes_gcm_pre_t) (const void *, struct gcm_key_data *)
 
typedef void(* aes_gmac_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
 
typedef void(* aes_gmac_update_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
 
typedef void(* aes_gmac_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
 
typedef void(* chacha_poly_init_t) (const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
 
typedef void(* chacha_poly_enc_dec_update_t) (const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
 
typedef void(* chacha_poly_finalize_t) (struct chacha20_poly1305_context_data *, void *, const uint64_t)
 
typedef void(* ghash_t) (const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
 
typedef void(* zuc_eea3_1_buffer_t) (const void *, const void *, const void *, void *, const uint32_t)
 
typedef void(* zuc_eea3_4_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
 
typedef void(* zuc_eea3_n_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
 
typedef void(* zuc_eia3_1_buffer_t) (const void *, const void *, const void *, const uint32_t, uint32_t *)
 
typedef void(* zuc_eia3_n_buffer_t) (const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
 
typedef void(* kasumi_f8_1_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_1_buffer_bit_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
 
typedef void(* kasumi_f8_2_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_3_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_4_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
 
typedef void(* kasumi_f8_n_buffer_t) (const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
 
typedef void(* kasumi_f9_1_buffer_user_t) (const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
 
typedef void(* kasumi_f9_1_buffer_t) (const kasumi_key_sched_t *, const void *, const uint32_t, void *)
 
typedef int(* kasumi_init_f8_key_sched_t) (const void *, kasumi_key_sched_t *)
 
typedef int(* kasumi_init_f9_key_sched_t) (const void *, kasumi_key_sched_t *)
 
typedef size_t(* kasumi_key_sched_size_t) (void)
 
typedef struct snow3g_key_schedule_s snow3g_key_schedule_t
 
typedef void(* snow3g_f8_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_1_buffer_bit_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
 
typedef void(* snow3g_f8_2_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_4_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_8_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
 
typedef void(* snow3g_f8_8_buffer_multikey_t) (const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[])
 
typedef void(* snow3g_f8_n_buffer_t) (const snow3g_key_schedule_t *, const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
 
typedef void(* snow3g_f8_n_buffer_multikey_t) (const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
 
typedef void(* snow3g_f9_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
 
typedef int(* snow3g_init_key_sched_t) (const void *, snow3g_key_schedule_t *)
 
typedef size_t(* snow3g_key_sched_size_t) (void)
 
typedef uint32_t(* hec_32_t) (const uint8_t *)
 
typedef uint64_t(* hec_64_t) (const uint8_t *)
 
typedef uint32_t(* crc32_fn_t) (const void *, const uint64_t)
 
typedef struct IMB_MGR IMB_MGR
 
+ + + + + + + + + + + + + + + + + + + +

+Enumerations

enum  IMB_ARCH {
+  IMB_ARCH_NONE = 0 +, IMB_ARCH_NOAESNI +, IMB_ARCH_SSE +, IMB_ARCH_AVX +,
+  IMB_ARCH_AVX2 +, IMB_ARCH_AVX512 +, IMB_ARCH_NUM +
+ }
 
enum  IMB_STATUS {
+  IMB_STATUS_BEING_PROCESSED = 0 +, IMB_STATUS_COMPLETED_CIPHER = 1 +, IMB_STATUS_COMPLETED_AUTH = 2 +, IMB_STATUS_COMPLETED = 3 +,
+  IMB_STATUS_INVALID_ARGS = 4 +, IMB_STATUS_INTERNAL_ERROR +, IMB_STATUS_ERROR +
+ }
 
enum  IMB_ERR {
+  IMB_ERR_MIN = 2000 +, IMB_ERR_NULL_MBMGR +, IMB_ERR_JOB_NULL_SRC +, IMB_ERR_JOB_NULL_DST +,
+  IMB_ERR_JOB_NULL_KEY +, IMB_ERR_JOB_NULL_IV +, IMB_ERR_JOB_NULL_AUTH +, IMB_ERR_JOB_NULL_AAD +,
+  IMB_ERR_JOB_CIPH_LEN +, IMB_ERR_JOB_AUTH_LEN +, IMB_ERR_JOB_IV_LEN +, IMB_ERR_JOB_KEY_LEN +,
+  IMB_ERR_JOB_AUTH_TAG_LEN +, IMB_ERR_JOB_AAD_LEN +, IMB_ERR_JOB_SRC_OFFSET +, IMB_ERR_JOB_CHAIN_ORDER +,
+  IMB_ERR_CIPH_MODE +, IMB_ERR_HASH_ALGO +, IMB_ERR_JOB_NULL_AUTH_KEY +, IMB_ERR_JOB_NULL_SGL_CTX +,
+  IMB_ERR_JOB_NULL_NEXT_IV +, IMB_ERR_JOB_PON_PLI +, IMB_ERR_NULL_SRC +, IMB_ERR_NULL_DST +,
+  IMB_ERR_NULL_KEY +, IMB_ERR_NULL_EXP_KEY +, IMB_ERR_NULL_IV +, IMB_ERR_NULL_AUTH +,
+  IMB_ERR_NULL_AAD +, IMB_ERR_CIPH_LEN +, IMB_ERR_AUTH_LEN +, IMB_ERR_IV_LEN +,
+  IMB_ERR_KEY_LEN +, IMB_ERR_AUTH_TAG_LEN +, IMB_ERR_AAD_LEN +, IMB_ERR_SRC_OFFSET +,
+  IMB_ERR_NULL_AUTH_KEY +, IMB_ERR_NULL_CTX +, IMB_ERR_NO_AESNI_EMU +, IMB_ERR_JOB_NULL_HMAC_OPAD +,
+  IMB_ERR_JOB_NULL_HMAC_IPAD +, IMB_ERR_JOB_NULL_XCBC_K1_EXP +, IMB_ERR_JOB_NULL_XCBC_K2 +, IMB_ERR_JOB_NULL_XCBC_K3 +,
+  IMB_ERR_JOB_CIPH_DIR +, IMB_ERR_JOB_NULL_GHASH_INIT_TAG +, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR +, IMB_ERR_NULL_JOB +,
+  IMB_ERR_MAX +
+ }
 
enum  IMB_CIPHER_MODE {
+  IMB_CIPHER_CBC = 1 +, IMB_CIPHER_CNTR +, IMB_CIPHER_NULL +, IMB_CIPHER_DOCSIS_SEC_BPI +,
+  IMB_CIPHER_GCM +, IMB_CIPHER_CUSTOM +, IMB_CIPHER_DES +, IMB_CIPHER_DOCSIS_DES +,
+  IMB_CIPHER_CCM +, IMB_CIPHER_DES3 +, IMB_CIPHER_PON_AES_CNTR +, IMB_CIPHER_ECB +,
+  IMB_CIPHER_CNTR_BITLEN +, IMB_CIPHER_ZUC_EEA3 +, IMB_CIPHER_SNOW3G_UEA2_BITLEN +, IMB_CIPHER_KASUMI_UEA1_BITLEN +,
+  IMB_CIPHER_CBCS_1_9 +, IMB_CIPHER_CHACHA20 +, IMB_CIPHER_CHACHA20_POLY1305 +, IMB_CIPHER_CHACHA20_POLY1305_SGL +,
+  IMB_CIPHER_SNOW_V +, IMB_CIPHER_SNOW_V_AEAD +, IMB_CIPHER_GCM_SGL +, IMB_CIPHER_NUM +
+ }
 
enum  IMB_CIPHER_DIRECTION { IMB_DIR_ENCRYPT = 1 +, IMB_DIR_DECRYPT + }
 
enum  IMB_HASH_ALG {
+  IMB_AUTH_HMAC_SHA_1 = 1 +, IMB_AUTH_HMAC_SHA_224 +, IMB_AUTH_HMAC_SHA_256 +, IMB_AUTH_HMAC_SHA_384 +,
+  IMB_AUTH_HMAC_SHA_512 +, IMB_AUTH_AES_XCBC +, IMB_AUTH_MD5 +, IMB_AUTH_NULL +,
+  IMB_AUTH_AES_GMAC +, IMB_AUTH_CUSTOM +, IMB_AUTH_AES_CCM +, IMB_AUTH_AES_CMAC +,
+  IMB_AUTH_SHA_1 +, IMB_AUTH_SHA_224 +, IMB_AUTH_SHA_256 +, IMB_AUTH_SHA_384 +,
+  IMB_AUTH_SHA_512 +, IMB_AUTH_AES_CMAC_BITLEN +, IMB_AUTH_PON_CRC_BIP +, IMB_AUTH_ZUC_EIA3_BITLEN +,
+  IMB_AUTH_DOCSIS_CRC32 +, IMB_AUTH_SNOW3G_UIA2_BITLEN +, IMB_AUTH_KASUMI_UIA1 +, IMB_AUTH_AES_GMAC_128 +,
+  IMB_AUTH_AES_GMAC_192 +, IMB_AUTH_AES_GMAC_256 +, IMB_AUTH_AES_CMAC_256 +, IMB_AUTH_POLY1305 +,
+  IMB_AUTH_CHACHA20_POLY1305 +, IMB_AUTH_CHACHA20_POLY1305_SGL +, IMB_AUTH_ZUC256_EIA3_BITLEN +, IMB_AUTH_SNOW_V_AEAD +,
+  IMB_AUTH_GCM_SGL +, IMB_AUTH_CRC32_ETHERNET_FCS +, IMB_AUTH_CRC32_SCTP +, IMB_AUTH_CRC32_WIMAX_OFDMA_DATA +,
+  IMB_AUTH_CRC24_LTE_A +, IMB_AUTH_CRC24_LTE_B +, IMB_AUTH_CRC16_X25 +, IMB_AUTH_CRC16_FP_DATA +,
+  IMB_AUTH_CRC11_FP_HEADER +, IMB_AUTH_CRC10_IUUP_DATA +, IMB_AUTH_CRC8_WIMAX_OFDMA_HCS +, IMB_AUTH_CRC7_FP_HEADER +,
+  IMB_AUTH_CRC6_IUUP_HEADER +, IMB_AUTH_GHASH +, IMB_AUTH_NUM +
+ }
 
enum  IMB_CHAIN_ORDER { IMB_ORDER_CIPHER_HASH = 1 +, IMB_ORDER_HASH_CIPHER + }
 
enum  IMB_KEY_SIZE_BYTES { IMB_KEY_128_BYTES = 16 +, IMB_KEY_192_BYTES = 24 +, IMB_KEY_256_BYTES = 32 + }
 
enum  IMB_SGL_STATE { IMB_SGL_INIT = 0 +, IMB_SGL_UPDATE +, IMB_SGL_COMPLETE +, IMB_SGL_ALL + }
 
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Functions

IMB_DLL_EXPORT const char * imb_get_version_str (void)
 Get library version in string format. More...
 
IMB_DLL_EXPORT unsigned imb_get_version (void)
 Get library version in numerical format. More...
 
IMB_DLL_EXPORT int imb_get_errno (IMB_MGR *mb_mgr)
 API to get error status. More...
 
IMB_DLL_EXPORT const char * imb_get_strerror (int errnum)
 API to get description for errnum. More...
 
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr (uint64_t flags)
 Allocates memory for multi-buffer manager instance. More...
 
IMB_DLL_EXPORT void free_mb_mgr (IMB_MGR *ptr)
 Frees memory allocated previously by alloc_mb_mgr() More...
 
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size (void)
 Calculates necessary memory size for IMB_MGR. More...
 
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr (void *ptr, const uint64_t flags, const unsigned reset_mgr)
 Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory. More...
 
IMB_DLL_EXPORT uint64_t imb_get_feature_flags (void)
 Retrieves the bitmask with the features supported by the library, without having to allocate/initialize IMB_MGR;. More...
 
IMB_DLL_EXPORT void init_mb_mgr_avx (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure. More...
 
IMB_DLL_EXPORT void init_mb_mgr_avx2 (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure. More...
 
IMB_DLL_EXPORT void init_mb_mgr_avx512 (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure. More...
 
IMB_DLL_EXPORT void init_mb_mgr_sse (IMB_MGR *state)
 Initialize Multi-Buffer Manager structure. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx (IMB_MGR *state)
 Submit job for processing after validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2 (IMB_MGR *state)
 Submit job for processing after validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512 (IMB_MGR *state)
 Submit job for processing after validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_sse (IMB_MGR *state)
 Submit job for processing after validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx (IMB_MGR *state)
 Submit job for processing without validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2 (IMB_MGR *state)
 Submit job for processing without validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512 (IMB_MGR *state)
 Submit job for processing without validating. More...
 
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse (IMB_MGR *state)
 Submit job for processing without validating. More...
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx (IMB_MGR *state)
 Force processing until next job in queue is completed. More...
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2 (IMB_MGR *state)
 Force processing until next job in queue is completed. More...
 
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512 (IMB_MGR *state)
 Force processing until next job in queue is completed. More...
 
IMB_DLL_EXPORT IMB_JOB * flush_job_sse (IMB_MGR *state)
 Force processing until next job in queue is completed. More...
 
IMB_DLL_EXPORT uint32_t queue_size_avx (IMB_MGR *state)
 Get number of jobs queued to be processed. More...
 
IMB_DLL_EXPORT uint32_t queue_size_avx2 (IMB_MGR *state)
 Get number of jobs queued to be processed. More...
 
IMB_DLL_EXPORT uint32_t queue_size_avx512 (IMB_MGR *state)
 Get number of jobs queued to be processed. More...
 
IMB_DLL_EXPORT uint32_t queue_size_sse (IMB_MGR *state)
 Get number of jobs queued to be processed. More...
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx (IMB_MGR *state)
 Get next completed job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2 (IMB_MGR *state)
 Get next completed job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512 (IMB_MGR *state)
 Get next completed job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse (IMB_MGR *state)
 Get next completed job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx (IMB_MGR *state)
 Get next available job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2 (IMB_MGR *state)
 Get next available job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512 (IMB_MGR *state)
 Get next available job. More...
 
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse (IMB_MGR *state)
 Get next available job. More...
 
IMB_DLL_EXPORT void init_mb_mgr_auto (IMB_MGR *state, IMB_ARCH *arch)
 Automatically initialize most performant Multi-buffer manager based on CPU features. More...
 
IMB_DLL_EXPORT int des_key_schedule (uint64_t *ks, const void *key)
 DES key schedule set up. More...
 
IMB_DLL_EXPORT void sha1_sse (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha1_avx (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha1_avx2 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha1_avx512 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha1_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha1_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha1_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha1_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha224_sse (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha224_avx (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha224_avx2 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha224_avx512 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha224_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha224_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha224_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha224_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha256_sse (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha256_avx (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha256_avx2 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha256_avx512 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha256_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha256_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha256_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha256_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha384_sse (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha384_avx (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha384_avx2 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha384_avx512 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha384_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha384_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha384_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha384_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha512_sse (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha512_avx (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha512_avx2 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha512_avx512 (const void *data, const uint64_t length, void *digest)
 
IMB_DLL_EXPORT void sha512_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha512_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha512_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void sha512_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_sse (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx2 (const void *data, void *digest)
 
IMB_DLL_EXPORT void md5_one_block_avx512 (const void *data, void *digest)
 
IMB_DLL_EXPORT void aes_keyexp_128_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_sse (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx2 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_avx512 (const void *key, void *enc_exp_keys, void *dec_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512 (const void *key, void *enc_exp_keys)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2 (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512 (const void *key, void *k1_exp, void *k2, void *k3)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2 (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512 (const void *key_exp, void *key1, void *key2)
 
IMB_DLL_EXPORT void aes_cfb_128_one_sse (void *out, const void *in, const void *iv, const void *keys, uint64_t len)
 AES-CFB-128 Encrypt/Decrypt up to one block. More...
 
IMB_DLL_EXPORT void aes_cfb_128_one_avx (void *out, const void *in, const void *iv, const void *keys, uint64_t len)
 AES-CFB-128 Encrypt/Decrypt up to one block. More...
 
IMB_DLL_EXPORT void aes_cfb_128_one_avx2 (void *out, const void *in, const void *iv, const void *keys, uint64_t len)
 AES-CFB-128 Encrypt/Decrypt up to one block. More...
 
IMB_DLL_EXPORT void aes_cfb_128_one_avx512 (void *out, const void *in, const void *iv, const void *keys, uint64_t len)
 AES-CFB-128 Encrypt/Decrypt up to one block. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-128 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-192 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
 AES-GCM-256 Decryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_128_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_192_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_256_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
 Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Encrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
 Decrypt a block of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End encryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-128 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-192 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4 (const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
 End decryption of a AES-GCM-256 encryption message. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-128 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-192 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4 (struct gcm_key_data *key_data)
 Precomputation of AES-GCM-256 HashKey constants. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_128_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-128 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_192_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-192 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_256_sse (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data. More...
 
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4 (const void *key, struct gcm_key_data *key_data)
 Pre-processes AES-GCM-256 key data. More...
 
IMB_DLL_EXPORT int zuc_eea3_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of ZUC-EEA3 Initialization Vector. More...
 
IMB_DLL_EXPORT int zuc_eia3_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of ZUC-EIA3 Initialization Vector. More...
 
IMB_DLL_EXPORT int kasumi_f8_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of KASUMI F8 Initialization Vector. More...
 
IMB_DLL_EXPORT int kasumi_f9_iv_gen (const uint32_t count, const uint32_t fresh, void *iv_ptr)
 Generation of KASUMI F9 Initialization Vector. More...
 
IMB_DLL_EXPORT int snow3g_f8_iv_gen (const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
 Generation of SNOW3G F8 Initialization Vector. More...
 
IMB_DLL_EXPORT int snow3g_f9_iv_gen (const uint32_t count, const uint32_t fresh, const uint8_t dir, void *iv_ptr)
 Generation of SNOW3G F9 Initialization Vector. More...
 
IMB_DLL_EXPORT void imb_clear_mem (void *mem, const size_t size)
 Force clearing/zeroing of memory. More...
 
+

Macro Definition Documentation

+ +

◆ __forceinline

+ +
+
+ + + + +
#define __forceinline    static __forceinline
+
+ +
+
+ +

◆ DECLARE_ALIGNED

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define DECLARE_ALIGNED( decl,
 alignval 
)    __declspec(align(alignval)) decl
+
+

Macros for aligning data structures and function inlines

+ +
+
+ +

◆ IMB_AES128_CFB_ONE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_CFB_ONE( _mgr,
 _dst,
 _src,
 _iv,
 _exp_key,
 _len 
)    ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[out]_dstPlaintext/Ciphertext output
[in]_srcPlaintext/Ciphertext input
[in]_ivPointer to 16 byte IV
[in]_exp_keyPointer to expanded AES keys
[in]_lenLength of data in bytes
+
+
+ +
+
+ +

◆ IMB_AES128_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm128_dec((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES128_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm128_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES128_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm128_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES128_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm128_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES128_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm128_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES128_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES128_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm192_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
(_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES192_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm192_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES192_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm192_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES192_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm192_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES192_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm192_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES192_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES192_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_DEC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm256_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
(_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES256_GCM_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_DEC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_ENC

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len,
 _iv,
 _aad,
 _aadl,
 _tag,
 _tagl 
)
+
+Value:
((_mgr)->gcm256_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
(_aad), (_aadl), (_tag), (_tagl)))
+
+
+
+ +

◆ IMB_AES256_GCM_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_ENC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _dst,
 _src,
 _len 
)    ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)    ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_INIT_VAR_IV

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_INIT_VAR_IV( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->gcm256_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
(_aad), (_aadl)))
+
+
+
+ +

◆ IMB_AES256_GCM_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->gcm256_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_AES256_GCM_PRECOMP

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GCM_PRECOMP( _mgr,
 _key 
)    ((_mgr)->gcm256_precomp((_key)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_FINALIZE( _mgr,
 _exp_key,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_INIT( _mgr,
 _exp_key,
 _ctx,
 _iv,
 _ivl 
)    ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
+ +
+
+ +

◆ IMB_AES256_GMAC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES256_GMAC_UPDATE( _mgr,
 _exp_key,
 _ctx,
 _src,
 _len 
)    ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
+
+ +
+
+ +

◆ IMB_AES_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_AES_BLOCK_SIZE   16
+
+ +
+
+ +

◆ IMB_AES_CMAC_SUBKEY_GEN_128

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_CMAC_SUBKEY_GEN_128( _mgr,
 _exp_key,
 _key1,
 _key2 
)    ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
+
+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyInput expanded AES-128-CMAC key
[out]_key1Subkey 1
[out]_key2Subkey 2
+
+
+ +
+
+ +

◆ IMB_AES_CMAC_SUBKEY_GEN_256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_CMAC_SUBKEY_GEN_256( _mgr,
 _exp_key,
 _key1,
 _key2 
)    ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
+
+

Generate AES-256-CMAC subkeys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyInput expanded AES-256-CMAC key
[out]_key1Subkey 1
[out]_key2Subkey 2
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_128

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_128( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-128 key
[out]_enc_exp_keyAES-128 encryption expansion key
[out]_dec_exp_keyAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_192

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_192( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-192 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-192 key
[out]_enc_exp_keyAES-192 encryption expansion key
[out]_dec_exp_keyAES-192 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_KEYEXP_256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_KEYEXP_256( _mgr,
 _key,
 _enc_exp_key,
 _dec_exp_key 
)    ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
+
+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-256 key
[out]_enc_exp_keyAES-256 encryption expansion key
[out]_dec_exp_keyAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ IMB_AES_XCBC_KEYEXP

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_AES_XCBC_KEYEXP( _mgr,
 _key,
 _exp_key,
 _exp_key2,
 _exp_key3 
)    ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
+
+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyAES-128-XCBC key
[out]_exp_keyk1 expansion key
[out]_exp_key2k2 expansion key
[out]_exp_key3k3 expansion key
+
+
+ +
+
+ +

◆ IMB_ASSERT

+ +
+
+ + + + + + + + +
#define IMB_ASSERT( x)
+
+

Custom ASSERT and DIM macros

+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_DEC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_DEC_FINALIZE( _mgr,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_DEC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_DEC_UPDATE( _mgr,
 _key,
 _ctx,
 _dst,
 _src,
 _len 
)
+
+Value:
((_mgr)->chacha20_poly1305_dec_update((_key), (_ctx), (_dst), (_src), \
+
(_len)))
+
+
+
+ +

◆ IMB_CHACHA20_POLY1305_ENC_FINALIZE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_ENC_FINALIZE( _mgr,
 _ctx,
 _tag,
 _tagl 
)    ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_CHACHA20_POLY1305_ENC_UPDATE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_ENC_UPDATE( _mgr,
 _key,
 _ctx,
 _dst,
 _src,
 _len 
)
+
+Value:
((_mgr)->chacha20_poly1305_enc_update((_key), (_ctx), (_dst), (_src), \
+
(_len)))
+
+
+
+ +

◆ IMB_CHACHA20_POLY1305_INIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CHACHA20_POLY1305_INIT( _mgr,
 _key,
 _ctx,
 _iv,
 _aad,
 _aadl 
)
+
+Value:
((_mgr)->chacha20_poly1305_init((_key), (_ctx), (_iv), (_aad), \
+
(_aadl)))
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX   (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_AVX2

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX2
+
+Value:
(IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \
+
IMB_FEATURE_BMI2)
+
IMB_CPUFLAGS_AVX
#define IMB_CPUFLAGS_AVX
Definition: intel-ipsec-mb.h:1034
+
IMB_FEATURE_AVX2
#define IMB_FEATURE_AVX2
Definition: intel-ipsec-mb.h:1010
+
IMB_FEATURE_BMI2
#define IMB_FEATURE_BMI2
Definition: intel-ipsec-mb.h:1025
+
+
+
+ +

◆ IMB_CPUFLAGS_AVX512

+ +
+
+ + + + +
#define IMB_CPUFLAGS_AVX512   (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
+
+ +
+
+ +

◆ IMB_CPUFLAGS_NO_AESNI

+ +
+
+ + + + +
#define IMB_CPUFLAGS_NO_AESNI   (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
+
+

CPU flags needed for each implementation

+ +
+
+ +

◆ IMB_CPUFLAGS_SSE

+ +
+
+ + + + +
#define IMB_CPUFLAGS_SSE
+
+Value:
(IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \
+
IMB_FEATURE_PCLMULQDQ)
+
IMB_FEATURE_AESNI
#define IMB_FEATURE_AESNI
Definition: intel-ipsec-mb.h:1005
+
IMB_FEATURE_PCLMULQDQ
#define IMB_FEATURE_PCLMULQDQ
Definition: intel-ipsec-mb.h:1006
+
IMB_CPUFLAGS_NO_AESNI
#define IMB_CPUFLAGS_NO_AESNI
Definition: intel-ipsec-mb.h:1031
+
+
+
+ +

◆ IMB_CRC10_IUUP_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC10_IUUP_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc10_iuup_data(_src, _len)
+
+

IUUP CRC10 function (3GPP TS 25.415)

+ +
+
+ +

◆ IMB_CRC11_FP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC11_FP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc11_fp_header(_src, _len)
+
+

Framing Protocol CRC11 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC16_FP_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC16_FP_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc16_fp_data(_src, _len)
+
+

Framing Protocol CRC16 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC16_X25

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC16_X25( _mgr,
 _src,
 _len 
)    (_mgr)->crc16_x25(_src, _len)
+
+

CRC16 X25 function

+ +
+
+ +

◆ IMB_CRC24_LTE_A

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC24_LTE_A( _mgr,
 _src,
 _len 
)    (_mgr)->crc24_lte_a(_src, _len)
+
+

LTE CRC24A function

+ +
+
+ +

◆ IMB_CRC24_LTE_B

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC24_LTE_B( _mgr,
 _src,
 _len 
)    (_mgr)->crc24_lte_b(_src, _len)
+
+

LTE CRC24B function

+ +
+
+ +

◆ IMB_CRC32_ETHERNET_FCS

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_ETHERNET_FCS( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_ethernet_fcs(_src, _len)
+
+

CRC32 Ethernet FCS function

+ +
+
+ +

◆ IMB_CRC32_SCTP

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_SCTP( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_sctp(_src, _len)
+
+

CRC32 SCTP function

+ +
+
+ +

◆ IMB_CRC32_WIMAX_OFDMA_DATA

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC32_WIMAX_OFDMA_DATA( _mgr,
 _src,
 _len 
)    (_mgr)->crc32_wimax_ofdma_data(_src, _len)
+
+

WIMAX OFDMA DATA CRC32 function (IEEE 802.16)

+ +
+
+ +

◆ IMB_CRC6_IUUP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC6_IUUP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc6_iuup_header(_src, _len)
+
+

IUUP CRC6 function (3GPP TS 25.415)

+ +
+
+ +

◆ IMB_CRC7_FP_HEADER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC7_FP_HEADER( _mgr,
 _src,
 _len 
)    (_mgr)->crc7_fp_header(_src, _len)
+
+

Framing Protocol CRC7 function (3GPP TS 25.435, 3GPP TS 25.427)

+ +
+
+ +

◆ IMB_CRC8_WIMAX_OFDMA_HCS

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_CRC8_WIMAX_OFDMA_HCS( _mgr,
 _src,
 _len 
)    (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
+
+

WIMAX OFDMA HCS CRC8 function (IEEE 802.16)

+ +
+
+ +

◆ IMB_DES_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_DES_BLOCK_SIZE   8
+
+ +
+
+ +

◆ IMB_DES_KEY_SCHED_SIZE

+ +
+
+ + + + +
#define IMB_DES_KEY_SCHED_SIZE   (16 * 8)
+
+

Algorithm constants 16 rounds x 8 bytes

+ +
+
+ +

◆ IMB_DES_KEYSCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_DES_KEYSCHED( _mgr,
 _exp_key,
 _key 
)    ((_mgr)->des_key_sched((_exp_key), (_key)))
+
+ +
+
+ +

◆ IMB_DIM

+ +
+
+ + + + + + + + +
#define IMB_DIM( x)   (sizeof(x) / sizeof(x[0]))
+
+ +
+
+ +

◆ IMB_DLL_EXPORT

+ +
+
+ + + + +
#define IMB_DLL_EXPORT
+
+

Windows DLL export is done via DEF file

+ +
+
+ +

◆ IMB_DLL_LOCAL

+ +
+
+ + + + +
#define IMB_DLL_LOCAL
+
+ +
+
+ +

◆ IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE

+ +
+
+ + + + +
#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE   14
+
+

Minimum Ethernet frame size to calculate CRC32 Source Address (6 bytes) + Destination Address (6 bytes) + Type/Len (2 bytes)

+ +
+
+ +

◆ IMB_DOCSIS_CRC32_TAG_SIZE

+ +
+
+ + + + +
#define IMB_DOCSIS_CRC32_TAG_SIZE   4
+
+ +
+
+ +

◆ IMB_FEATURE_AESNI

+ +
+
+ + + + +
#define IMB_FEATURE_AESNI   (1ULL << 1)
+
+ +
+
+ +

◆ IMB_FEATURE_AESNI_EMU

+ +
+
+ + + + +
#define IMB_FEATURE_AESNI_EMU   (1ULL << 19)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX

+ +
+
+ + + + +
#define IMB_FEATURE_AVX   (1ULL << 5)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX2

+ +
+
+ + + + +
#define IMB_FEATURE_AVX2   (1ULL << 6)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512_IFMA

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512_IFMA   (1ULL << 17)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512_SKX

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512_SKX
+
+Value:
(IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \
+
IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \
+
IMB_FEATURE_AVX512VL)
+
IMB_FEATURE_AVX512BW
#define IMB_FEATURE_AVX512BW
Definition: intel-ipsec-mb.h:1014
+
IMB_FEATURE_AVX512VL
#define IMB_FEATURE_AVX512VL
Definition: intel-ipsec-mb.h:1015
+
IMB_FEATURE_AVX512DQ
#define IMB_FEATURE_AVX512DQ
Definition: intel-ipsec-mb.h:1012
+
IMB_FEATURE_AVX512F
#define IMB_FEATURE_AVX512F
Definition: intel-ipsec-mb.h:1011
+
IMB_FEATURE_AVX512CD
#define IMB_FEATURE_AVX512CD
Definition: intel-ipsec-mb.h:1013
+
+
+
+ +

◆ IMB_FEATURE_AVX512BW

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512BW   (1ULL << 10)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512CD

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512CD   (1ULL << 9)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512DQ

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512DQ   (1ULL << 8)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512F

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512F   (1ULL << 7)
+
+ +
+
+ +

◆ IMB_FEATURE_AVX512VL

+ +
+
+ + + + +
#define IMB_FEATURE_AVX512VL   (1ULL << 11)
+
+ +
+
+ +

◆ IMB_FEATURE_BMI2

+ +
+
+ + + + +
#define IMB_FEATURE_BMI2   (1ULL << 18)
+
+ +
+
+ +

◆ IMB_FEATURE_CMOV

+ +
+
+ + + + +
#define IMB_FEATURE_CMOV   (1ULL << 3)
+
+ +
+
+ +

◆ IMB_FEATURE_GFNI

+ +
+
+ + + + +
#define IMB_FEATURE_GFNI   (1ULL << 16)
+
+ +
+
+ +

◆ IMB_FEATURE_PCLMULQDQ

+ +
+
+ + + + +
#define IMB_FEATURE_PCLMULQDQ   (1ULL << 2)
+
+ +
+
+ +

◆ IMB_FEATURE_SAFE_DATA

+ +
+
+ + + + +
#define IMB_FEATURE_SAFE_DATA   (1ULL << 14)
+
+ +
+
+ +

◆ IMB_FEATURE_SAFE_PARAM

+ +
+
+ + + + +
#define IMB_FEATURE_SAFE_PARAM   (1ULL << 15)
+
+ +
+
+ +

◆ IMB_FEATURE_SHANI

+ +
+
+ + + + +
#define IMB_FEATURE_SHANI   (1ULL << 0)
+
+

Multi-buffer manager detected features

    +
  • if bit is set then hardware supports given extension
    • +
  • valid after call to init_mb_mgr() or alloc_mb_mgr()
    • +
  • some HW supported features can be disabled via IMB_FLAG_xxx (see above)
    • +
+ +
+
+ +

◆ IMB_FEATURE_SSE4_2

+ +
+
+ + + + +
#define IMB_FEATURE_SSE4_2   (1ULL << 4)
+
+ +
+
+ +

◆ IMB_FEATURE_VAES

+ +
+
+ + + + +
#define IMB_FEATURE_VAES   (1ULL << 12)
+
+ +
+
+ +

◆ IMB_FEATURE_VPCLMULQDQ

+ +
+
+ + + + +
#define IMB_FEATURE_VPCLMULQDQ   (1ULL << 13)
+
+ +
+
+ +

◆ IMB_FLAG_AESNI_OFF

+ +
+
+ + + + +
#define IMB_FLAG_AESNI_OFF   (1ULL << 1)
+
+

disable use of AESNI extension

+ +
+
+ +

◆ IMB_FLAG_SHANI_OFF

+ +
+
+ + + + +
#define IMB_FLAG_SHANI_OFF   (1ULL << 0)
+
+

disable use of SHANI extension

+ +
+
+ +

◆ IMB_FLUSH_JOB

+ +
+
+ + + + + + + + +
#define IMB_FLUSH_JOB( _mgr)   ((_mgr)->flush_job((_mgr)))
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ IMB_GCM_128_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_128_KEY_LEN   (16)
+
+ +
+
+ +

◆ IMB_GCM_192_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_192_KEY_LEN   (24)
+
+ +
+
+ +

◆ IMB_GCM_256_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_256_KEY_LEN   (32)
+
+ +
+
+ +

◆ IMB_GCM_BLOCK_LEN

+ +
+
+ + + + +
#define IMB_GCM_BLOCK_LEN   16
+
+ +
+
+ +

◆ IMB_GCM_ENC_KEY_LEN

+ +
+
+ + + + +
#define IMB_GCM_ENC_KEY_LEN   16
+
+ +
+
+ +

◆ IMB_GCM_IV_DATA_LEN

+ +
+
+ + + + +
#define IMB_GCM_IV_DATA_LEN   (12)
+
+

IV data is limited to 16 bytes as follows: 12 bytes is provided by an application - pre-counter block j0: 4 byte salt (from Security Association) concatenated with 8 byte Initialization Vector (from IPSec ESP Payload). 4 byte value 0x00000001 is padded automatically by the library - there is no need to add these 4 bytes on application side anymore.

+ +
+
+ +

◆ IMB_GCM_KEY_SETS

+ +
+
+ + + + +
#define IMB_GCM_KEY_SETS   (15)
+
+

exp key + 14 exp round keys

+ +
+
+ +

◆ IMB_GET_COMPLETED_JOB

+ +
+
+ + + + + + + + +
#define IMB_GET_COMPLETED_JOB( _mgr)   ((_mgr)->get_completed_job((_mgr)))
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ IMB_GET_NEXT_JOB

+ +
+
+ + + + + + + + +
#define IMB_GET_NEXT_JOB( _mgr)   ((_mgr)->get_next_job((_mgr)))
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ IMB_GHASH

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_GHASH( _mgr,
 _exp_key,
 _src,
 _len,
 _tag,
 _tagl 
)    ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
+
+ +
+
+ +

◆ IMB_GHASH_PRE

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_GHASH_PRE( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->ghash_pre((_key), (_exp_key)))
+
+ +
+
+ +

◆ IMB_HEC_32

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_HEC_32( _mgr,
 _src 
)   ((_mgr)->hec_32(_src))
+
+

HEC compute functions

+ +
+
+ +

◆ IMB_HEC_64

+ +
+
+ + + + + + + + + + + + + + + + + + +
#define IMB_HEC_64( _mgr,
 _src 
)   ((_mgr)->hec_64(_src))
+
+ +
+
+ +

◆ IMB_KASUMI_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_BLOCK_SIZE   8
+
+ +
+
+ +

◆ IMB_KASUMI_DIGEST_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_DIGEST_SIZE   4
+
+ +
+
+ +

◆ IMB_KASUMI_F8_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
+ +

Kasumi byte-level f8 operation on a single buffer.

+

This function performs kasumi f8 operation on a single buffer. The key has already been scheduled with kasumi_init_f8_key_sched(). No extra bits are modified.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in BYTES
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_1_BUFFER_BIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_1_BUFFER_BIT( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _offset 
)
+
+Value:
((_mgr)->f8_1_buffer_bit((_exp_key), (_iv), (_src), (_dst), (_len), \
+
(_offset)))
+
+

Kasumi bit-level f8 operation on a single buffer.

+

This function performs kasumi f8 operation on a single buffer. The key has already been scheduled with kasumi_init_f8_key_sched(). No extra bits are modified.

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in BITS
[in]_offsetOffset in BITS from begin of input buffer
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_2_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_2_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2 
)
+
+Value:
((_mgr)->f8_2_buffer((_exp_key), (_iv1), (_iv2), (_src1), (_dst1), \
+
(_len1), (_src2), (_dst2), (_len2)))
+
+

Kasumi byte-level f8 operation in parallel on two buffers.

+

This function performs kasumi f8 operation on a two buffers. They will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in BYTES of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in BYTES of input buffer 2
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_3_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_3_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _src1,
 _dst1,
 _src2,
 _dst2,
 _src3,
 _dst3,
 _len 
)
+
+Value:
((_mgr)->f8_3_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_src1), \
+
(_dst1), (_src2), (_dst2), (_src3), (_dst3), \
+
(_len)))
+
+

kasumi byte-level f8 operation in parallel on three buffers

+

This function performs kasumi f8 operation on a three buffers. They must all have the same length and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_iv3Initialization vector for buffer in3
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_lenCommon length in bytes for all buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_4_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _src1,
 _dst1,
 _src2,
 _dst2,
 _src3,
 _dst3,
 _src4,
 _dst4,
 _len 
)
+
+Value:
((_mgr)->f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_iv4), \
+
(_src1), (_dst1), (_src2), (_dst2), \
+
(_src3), (_dst3), (_src4), (_dst4), (_len)))
+
+

kasumi byte-level f8 operation in parallel on four buffers

+

This function performs kasumi f8 operation on four buffers. They must all have the same length and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1Initialization vector for buffer in1
[in]_iv2Initialization vector for buffer in2
[in]_iv3Initialization vector for buffer in3
[in]_iv4Initialization vector for buffer in4
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_lenCommon length in bytes for all buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F8_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F8_N_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->f8_n_buffer((_exp_key), (_iv), (_src), (_dst), (_len), \
+
(_count)))
+
+

Kasumi f8 operation on N buffers.

+

All input buffers can have different lengths and they will be processed with the same key, which has already been scheduled with kasumi_init_f8_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers
[in]_lenArray of corresponding input buffer lengths in BITS
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_KASUMI_F9_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F9_1_BUFFER( _mgr,
 _exp_key,
 _src,
 _len,
 _tag 
)    ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
+
+ +

Kasumi bit-level f9 operation on a single buffer.

+

The first QWORD of in represents the COUNT and FRESH, the last QWORD represents the DIRECTION and PADDING. (See 3GPP TS 35.201 v10.0 section 4)

+

The key has already been scheduled with kasumi_init_f9_key_sched().

+
Parameters
+ + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_srcInput buffer
[in]_lenLength in BYTES of the data to be hashed
[out]_tagComputed digest
+
+
+ +
+
+ +

◆ IMB_KASUMI_F9_1_BUFFER_USER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_F9_1_BUFFER_USER( _mgr,
 _exp_key,
 _iv,
 _src,
 _len,
 _tag,
 _dir 
)
+
+Value:
((_mgr)->f9_1_buffer_user((_exp_key), (_iv), (_src), (_len), \
+
(_tag), (_dir)))
+
+

Kasumi bit-level f9 operation on a single buffer.

+

The key has already been scheduled with kasumi_init_f9_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivInitialization vector
[in]_srcInput buffer
[in]_lenLength in BITS of the data to be hashed
[out]_tagComputed digest
[in]_dirDirection bit
+
+
+ +
+
+ +

◆ IMB_KASUMI_INIT_F8_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_INIT_F8_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
+
+

KASUMI F8 key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyConfidentiality key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success, -1 on failure
+ +
+
+ +

◆ IMB_KASUMI_INIT_F9_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_KASUMI_INIT_F9_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
+
+

KASUMI F9 key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyIntegrity key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success, -1 on failure
+ +
+
+ +

◆ IMB_KASUMI_IV_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_IV_SIZE   8
+
+ +
+
+ +

◆ IMB_KASUMI_KEY_SCHED_SIZE

+ +
+
+ + + + + + + + +
#define IMB_KASUMI_KEY_SCHED_SIZE( _mgr)   ((_mgr)->kasumi_key_sched_size())
+
+

This function returns the size of the kasumi_key_sched_t, used to store the key schedule.

+
Parameters
+ + +
[in]_mgrPointer to multi-buffer structure
+
+
+
Returns
size of kasumi_key_sched_t type success
+ +
+
+ +

◆ IMB_KASUMI_KEY_SIZE

+ +
+
+ + + + +
#define IMB_KASUMI_KEY_SIZE   16
+
+ +
+
+ +

◆ IMB_MAX_JOBS

+ +
+
+ + + + +
#define IMB_MAX_JOBS   128
+
+ +
+
+ +

◆ IMB_MAX_TAG_LEN

+ +
+
+ + + + +
#define IMB_MAX_TAG_LEN   (16)
+
+

Authenticated Tag Length in bytes. Valid values are 16 (most likely), 12 or 8.

+ +
+
+ +

◆ IMB_MD5_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_MD5_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->md5_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (16 bytes)
+
+
+ +
+
+ +

◆ IMB_QUEUE_SIZE

+ +
+
+ + + + + + + + +
#define IMB_QUEUE_SIZE( _mgr)   ((_mgr)->queue_size((_mgr)))
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ IMB_SHA1

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA1( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha1((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA1_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA1_BLOCK_SIZE   64
+
+

512 bits is 64 byte blocks

+ +
+
+ +

◆ IMB_SHA1_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA1_DIGEST_SIZE_IN_BYTES   20
+
+ +
+
+ +

◆ IMB_SHA1_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA1_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha1_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA224

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA224( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha224((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (28 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA224_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA224_DIGEST_SIZE_IN_BYTES   28
+
+ +
+
+ +

◆ IMB_SHA224_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA224_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha224_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (28 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA256

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA256( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha256((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (32 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA256_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA256_DIGEST_SIZE_IN_BYTES   32
+
+ +
+
+ +

◆ IMB_SHA256_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA256_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha256_one_block((_src), (_tag)))
+
+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src64-byte data buffer
[out]_tagDigest output (32 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA384

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA384( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha384((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (48 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA384_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA384_DIGEST_SIZE_IN_BYTES   48
+
+ +
+
+ +

◆ IMB_SHA384_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA384_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha384_one_block((_src), (_tag)))
+
+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src128-byte data buffer
[out]_tagDigest output (48 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA512

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA512( _mgr,
 _src,
 _length,
 _tag 
)    ((_mgr)->sha512((_src), (_length), (_tag)))
+
+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_srcData buffer
[in]_lengthLength of data in bytes for authentication.
[out]_tagDigest output (20 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA512_DIGEST_SIZE_IN_BYTES

+ +
+
+ + + + +
#define IMB_SHA512_DIGEST_SIZE_IN_BYTES   64
+
+ +
+
+ +

◆ IMB_SHA512_ONE_BLOCK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SHA512_ONE_BLOCK( _mgr,
 _src,
 _tag 
)    ((_mgr)->sha512_one_block((_src), (_tag)))
+
+

Authenticate 128-byte data buffer with SHA512.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_src128-byte data buffer
[out]_tagDigest output (64 bytes)
+
+
+ +
+
+ +

◆ IMB_SHA_256_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_256_BLOCK_SIZE   64
+
+

512 bits is 64 byte blocks

+ +
+
+ +

◆ IMB_SHA_384_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_384_BLOCK_SIZE   128
+
+ +
+
+ +

◆ IMB_SHA_512_BLOCK_SIZE

+ +
+
+ + + + +
#define IMB_SHA_512_BLOCK_SIZE   128
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
+

This function performs snow3g f8 operation on a single buffer. The key has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = count iv[2] = (bearer << 27) | ((dir & 0x1) << 26) iv[1] = pIV[3] iv[0] = pIV[2]
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in bits of input buffer
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_1_BUFFER_BIT

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_1_BUFFER_BIT( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _offset 
)
+
+Value:
((_mgr)->snow3g_f8_1_buffer_bit((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_offset)))
+

This function performs snow3g f8 operation on a single buffer. The key has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = count iv[2] = (bearer << 27) | ((dir & 0x1) << 26) iv[1] = pIV[3] iv[0] = pIV[2]
[in]_srcInput buffer
[out]_dstOutput buffer
[in]_lenLength in bits of input buffer
[in]_offsetOffset in input/output buffer (in bits)
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_2_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_2_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2 
)
+
+Value:
((_mgr)->snow3g_f8_2_buffer((_exp_key), (_iv1), (_iv2), \
+
(_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2)))
+

This function performs snow3g f8 operation on two buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_4_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2,
 _src3,
 _dst3,
 _len3,
 _src4,
 _dst4,
 _len4 
)
+
+Value:
((_mgr)->snow3g_f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
(_iv4), (_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2), \
+
(_src3), (_dst3), (_len3), \
+
(_src4), (_dst4), (_len4)))
+

This function performs snow3g f8 operation on four buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_iv3IV to use for buffer pBufferIn3
[in]_iv4IV to use for buffer pBufferIn4
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_len3Length in bytes of input buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_len4Length in bytes of input buffer 4
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_8_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_8_BUFFER( _mgr,
 _exp_key,
 _iv1,
 _iv2,
 _iv3,
 _iv4,
 _iv5,
 _iv6,
 _iv7,
 _iv8,
 _src1,
 _dst1,
 _len1,
 _src2,
 _dst2,
 _len2,
 _src3,
 _dst3,
 _len3,
 _src4,
 _dst4,
 _len4,
 _src5,
 _dst5,
 _len5,
 _src6,
 _dst6,
 _len6,
 _src7,
 _dst7,
 _len7,
 _src8,
 _dst8,
 _len8 
)
+
+Value:
((_mgr)->snow3g_f8_8_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
(_iv4), (_iv5), (_iv6), (_iv7), \
+
(_iv8), (_src1), (_dst1), (_len1), \
+
(_src2), (_dst2), (_len2), \
+
(_src3), (_dst3), (_len3), \
+
(_src4), (_dst4), (_len4), \
+
(_src5), (_dst5), (_len5), \
+
(_src6), (_dst6), (_len6), \
+
(_src7), (_dst7), (_len7), \
+
(_src8), (_dst8), (_len8)))
+

This function performs snow3g f8 operation on eight buffers. They will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iv1IV to use for buffer pBufferIn1
[in]_iv2IV to use for buffer pBufferIn2
[in]_iv3IV to use for buffer pBufferIn3
[in]_iv4IV to use for buffer pBufferIn4
[in]_iv5IV to use for buffer pBufferIn5
[in]_iv6IV to use for buffer pBufferIn6
[in]_iv7IV to use for buffer pBufferIn7
[in]_iv8IV to use for buffer pBufferIn8
[in]_src1Input buffer 1
[out]_dst1Output buffer 1
[in]_len1Length in bytes of input buffer 1
[in]_src2Input buffer 2
[out]_dst2Output buffer 2
[in]_len2Length in bytes of input buffer 2
[in]_src3Input buffer 3
[out]_dst3Output buffer 3
[in]_len3Length in bytes of input buffer 3
[in]_src4Input buffer 4
[out]_dst4Output buffer 4
[in]_len4Length in bytes of input buffer 4
[in]_src5Input buffer 5
[out]_dst5Output buffer 5
[in]_len5Length in bytes of input buffer 5
[in]_src6Input buffer 6
[out]_dst6Output buffer 6
[in]_len6Length in bytes of input buffer 6
[in]_src7Input buffer 7
[out]_dst7Output buffer 7
[in]_len7Length in bytes of input buffer 7
[in]_src8Input buffer 8
[out]_dst8Output buffer 8
[in]_len8Length in bytes of input buffer 8
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_8_BUFFER_MULTIKEY

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len 
)
+
+Value:
((_mgr)->snow3g_f8_8_buffer_multikey((_exp_key), (_iv), (_src), (_dst),\
+
(_len)))
+

This function performs snow3g f8 operation on eight buffers. They will be processed with individual keys, which have already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyArray of 8 Contexts, where the scheduled keys are stored
[in]_ivArray of 8 IV values
[in]_srcArray of 8 input buffers
[out]_dstArray of 8 output buffers
[in]_lenArray of 8 corresponding input buffer lengths
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_N_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->snow3g_f8_n_buffer((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_count)))
+

This function performs snow3g f8 operation in parallel on N buffers. All input buffers can have different lengths and they will be processed with the same key, which has already been scheduled with snow3g_init_key_sched().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers - out[0] set to NULL on failure
[in]_lenArray of corresponding input buffer lengths
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F8_N_BUFFER_MULTIKEY

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY( _mgr,
 _exp_key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)
+
+Value:
((_mgr)->snow3g_f8_n_buffer_multikey((_exp_key), (_iv), (_src), \
+
(_dst), (_len), (_count)))
+

This function performs snow3g f8 operation in parallel on N buffers. All input buffers can have different lengths. Confidentiallity keys can vary, schedules with snow3g_init_key_sched_multi().

+
Parameters
+ + + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyArray of Contexts, where the scheduled keys are stored
[in]_ivArray of IV values
[in]_srcArray of input buffers
[out]_dstArray of output buffers
    +
  • out[0] set to NULL on failure
    • +
+
[in]_lenArray of corresponding input buffer lengths
[in]_countNumber of input buffers
+
+
+ +
+
+ +

◆ IMB_SNOW3G_F9_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_F9_1_BUFFER( _mgr,
 _exp_key,
 _iv,
 _src,
 _len,
 _tag 
)    ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
+
+

This function performs a snow3g f9 operation on a single block of data. The key has already been scheduled with snow3g_init_f8_key_sched().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to multi-buffer structure
[in]_exp_keyContext where the scheduled keys are stored
[in]_iviv[3] = _BSWAP32(fresh^(dir<<15)) iv[2] = _BSWAP32(count^(dir<<31)) iv[1] = _BSWAP32(fresh) iv[0] = _BSWAP32(count)
[in]_srcInput buffer
[in]_lenLength in bits of the data to be hashed
[out]_tagComputed digest
+
+
+ +
+
+ +

◆ IMB_SNOW3G_INIT_KEY_SCHED

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SNOW3G_INIT_KEY_SCHED( _mgr,
 _key,
 _exp_key 
)    ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
+
+

Snow3g key schedule init function.

+
Parameters
+ + + + +
[in]_mgrPointer to multi-buffer structure
[in]_keyConfidentiality/Integrity key (expected in LE format)
[out]_exp_keyKey schedule context to be initialised
+
+
+
Returns
0 on success
+
+-1 on error
+ +
+
+ +

◆ IMB_SNOW3G_KEY_SCHED_SIZE

+ +
+
+ + + + + + + + +
#define IMB_SNOW3G_KEY_SCHED_SIZE( _mgr)   ((_mgr)->snow3g_key_sched_size())
+
+

This function returns the size of the snow3g_key_schedule_t, used to store the key schedule.

+
Parameters
+ + +
[in]_mgrPointer to multi-buffer structure
+
+
+
Returns
size of snow3g_key_schedule_t type
+ +
+
+ +

◆ IMB_SUBMIT_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_BURST( _mgr,
 _jobs,
 _n_jobs 
)    ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs)))
+
+

Submit multiple jobs to be processed synchronously after validating.

+
Parameters
+ + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_BURST_NOCHECK( _mgr,
 _jobs,
 _n_jobs 
)    ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs)))
+
+

Submit multiple jobs to be processed synchronously without validating.

+

This is more performant but less secure than IMB_SUBMIT_BURST().

+
Parameters
+ + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_CIPHER_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_CIPHER_BURST( _mgr,
 _jobs,
 _n_jobs,
 _cipher,
 _dir,
 _key_size 
)
+
+Value:
((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \
+
(_cipher), (_dir), (_key_size)))
+

Submit multiple cipher jobs to be processed synchronously after validating.

+
Parameters
+ + + + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_cipherCipher algorithm of type IMB_CIPHER_MODE
[in]_dirCipher direction of type IMB_CIPHER_DIRECTION
[in]_key_sizeKey size in bytes of type IMB_KEY_SIZE_BYTES
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_CIPHER_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_CIPHER_BURST_NOCHECK( _mgr,
 _jobs,
 _n_jobs,
 _cipher,
 _dir,
 _key_size 
)
+
+Value:
((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\
+
(_cipher), (_dir), (_key_size)))
+

Submit multiple cipher jobs to be processed synchronously without validating.

+

This is more performant but less secure than IMB_SUBMIT_CIPHER_BURST().

+
Parameters
+ + + + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_cipherCipher algorithm of type IMB_CIPHER_MODE
[in]_dirCipher direction of type IMB_CIPHER_DIRECTION
[in]_key_sizeKey size in bytes of type IMB_KEY_SIZE_BYTES
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_HASH_BURST

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_HASH_BURST( _mgr,
 _jobs,
 _n_jobs,
 _hash 
)    ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
+
+

Submit multiple hash jobs to be processed synchronously after validating.

+
Parameters
+ + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_hashHash algorithm of type IMB_HASH_ALG
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_HASH_BURST_NOCHECK

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_SUBMIT_HASH_BURST_NOCHECK( _mgr,
 _jobs,
 _n_jobs,
 _hash 
)    ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
+
+

Submit multiple hash jobs to be processed synchronously without validating.

+

This is more performant but less secure than IMB_SUBMIT_HASH_BURST().

+
Parameters
+ + + + + +
[in]_mgrPointer to initialized IMB_MGR structure
[in,out]_jobsPointer to array of IMB_JOB structures
[in]_n_jobsNumber of jobs to process
[in]_hashHash algorithm of type IMB_HASH_ALG
+
+
+
Returns
Number of completed jobs
+ +
+
+ +

◆ IMB_SUBMIT_JOB

+ +
+
+ + + + + + + + +
#define IMB_SUBMIT_JOB( _mgr)   ((_mgr)->submit_job((_mgr)))
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ IMB_SUBMIT_JOB_NOCHECK

+ +
+
+ + + + + + + + +
#define IMB_SUBMIT_JOB_NOCHECK( _mgr)   ((_mgr)->submit_job_nocheck((_mgr)))
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]_mgrPointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ IMB_VERSION

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_VERSION( a,
 b,
 c 
)   (((a) << 16) + ((b) << 8) + (c))
+
+

Macro to translate version number

+ +
+
+ +

◆ IMB_VERSION_NUM

+ +
+
+ + + + +
#define IMB_VERSION_NUM   0x10200
+
+ +
+
+ +

◆ IMB_VERSION_STR

+ +
+
+ + + + +
#define IMB_VERSION_STR   "1.2.0"
+
+

Library version

+ +
+
+ +

◆ IMB_ZUC_EEA3_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_1_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
+ +

ZUC EEA3 Confidentiality functions.

+
Parameters
+ + + + + + + +
_mgrPointer to multi-buffer structure
_keyPointer to key
_ivPointer to 16-byte IV
_srcPointer to Plaintext/Ciphertext input.
_dstPointer to Ciphertext/Plaintext output.
_lenLength of input data in bytes.
+
+
+ +
+
+ +

◆ IMB_ZUC_EEA3_4_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_4_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len 
)    ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
+ +
+
+ +

◆ IMB_ZUC_EEA3_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EEA3_N_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _dst,
 _len,
 _count 
)    ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
+
+ +
+
+ +

◆ IMB_ZUC_EIA3_1_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EIA3_1_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _len,
 _tag 
)    ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
+
+ +

ZUC EIA3 Integrity function.

+
Parameters
+ + + + + + + +
_mgrPointer to multi-buffer structure
_keyPointer to key
_ivPointer to 16-byte IV
_srcPointer to Plaintext/Ciphertext input.
_lenLength of input data in bits.
_tagPointer to Authenticated Tag output (4 bytes)
+
+
+ +
+
+ +

◆ IMB_ZUC_EIA3_N_BUFFER

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#define IMB_ZUC_EIA3_N_BUFFER( _mgr,
 _key,
 _iv,
 _src,
 _len,
 _tag,
 _count 
)    ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
+
+ +
+
+ +

◆ KASUMI_KEY_SCHEDULE_SIZE

+ +
+
+ + + + +
#define KASUMI_KEY_SCHEDULE_SIZE   64
+
+ +
+
+

Typedef Documentation

+ +

◆ aes_cfb_t

+ +
+
+ + + + +
typedef void(* aes_cfb_t) (void *, const void *, const void *, const void *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_finalize_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_iv_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_enc_dec_update_t

+ +
+
+ + + + +
typedef void(* aes_gcm_enc_dec_update_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_init_t

+ +
+
+ + + + +
typedef void(* aes_gcm_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
+
+ +
+
+ +

◆ aes_gcm_init_var_iv_t

+ +
+
+ + + + +
typedef void(* aes_gcm_init_var_iv_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gcm_pre_t

+ +
+
+ + + + +
typedef void(* aes_gcm_pre_t) (const void *, struct gcm_key_data *)
+
+ +
+
+ +

◆ aes_gcm_precomp_t

+ +
+
+ + + + +
typedef void(* aes_gcm_precomp_t) (struct gcm_key_data *)
+
+ +
+
+ +

◆ aes_gmac_finalize_t

+ +
+
+ + + + +
typedef void(* aes_gmac_finalize_t) (const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gmac_init_t

+ +
+
+ + + + +
typedef void(* aes_gmac_init_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ aes_gmac_update_t

+ +
+
+ + + + +
typedef void(* aes_gmac_update_t) (const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
+
+ +
+
+ +

◆ chacha_poly_enc_dec_update_t

+ +
+
+ + + + +
typedef void(* chacha_poly_enc_dec_update_t) (const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
+
+ +
+
+ +

◆ chacha_poly_finalize_t

+ +
+
+ + + + +
typedef void(* chacha_poly_finalize_t) (struct chacha20_poly1305_context_data *, void *, const uint64_t)
+
+ +
+
+ +

◆ chacha_poly_init_t

+ +
+
+ + + + +
typedef void(* chacha_poly_init_t) (const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
+
+ +
+
+ +

◆ cmac_subkey_gen_t

+ +
+
+ + + + +
typedef void(* cmac_subkey_gen_t) (const void *, void *, void *)
+
+ +
+
+ +

◆ crc32_fn_t

+ +
+
+ + + + +
typedef uint32_t(* crc32_fn_t) (const void *, const uint64_t)
+
+ +
+
+ +

◆ des_keysched_t

+ +
+
+ + + + +
typedef int(* des_keysched_t) (uint64_t *, const void *)
+
+ +
+
+ +

◆ flush_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* flush_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ get_completed_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* get_completed_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ get_next_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* get_next_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ ghash_t

+ +
+
+ + + + +
typedef void(* ghash_t) (const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
+
+ +
+
+ +

◆ hash_fn_t

+ +
+
+ + + + +
typedef void(* hash_fn_t) (const void *, const uint64_t, void *)
+
+ +
+
+ +

◆ hash_one_block_t

+ +
+
+ + + + +
typedef void(* hash_one_block_t) (const void *, void *)
+
+ +
+
+ +

◆ hec_32_t

+ +
+
+ + + + +
typedef uint32_t(* hec_32_t) (const uint8_t *)
+
+ +
+
+ +

◆ hec_64_t

+ +
+
+ + + + +
typedef uint64_t(* hec_64_t) (const uint8_t *)
+
+ +
+
+ +

◆ IMB_JOB

+ +
+
+ + + + +
typedef struct IMB_JOB IMB_JOB
+
+

Job structure.

+

For AES, enc_keys and dec_keys are expected to point to expanded keys structure.

    +
  • AES-CTR, AES-ECB and AES-CCM, only enc_keys is used
    • +
  • DOCSIS (AES-CBC + AES-CFB), both pointers are used enc_keys has to be set always for the partial block
    • +
+

For DES, enc_keys and dec_keys are expected to point to DES key schedule.

    +
  • same key schedule used for enc and dec operations
    • +
+

For 3DES, enc_keys and dec_keys are expected to point to an array of 3 pointers for the corresponding 3 key schedules.

    +
  • same key schedule used for enc and dec operations
    • +
+ +
+
+ +

◆ IMB_MGR

+ +
+
+ + + + +
typedef struct IMB_MGR IMB_MGR
+
+ +
+
+ +

◆ init_mb_mgr_t

+ +
+
+ + + + +
typedef void(* init_mb_mgr_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ kasumi_f8_1_buffer_bit_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_1_buffer_bit_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_1_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_1_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_2_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_2_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_3_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_3_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_4_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_4_buffer_t) (const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f8_n_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f8_n_buffer_t) (const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_f9_1_buffer_t

+ +
+
+ + + + +
typedef void(* kasumi_f9_1_buffer_t) (const kasumi_key_sched_t *, const void *, const uint32_t, void *)
+
+ +
+
+ +

◆ kasumi_f9_1_buffer_user_t

+ +
+
+ + + + +
typedef void(* kasumi_f9_1_buffer_user_t) (const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
+
+ +
+
+ +

◆ kasumi_init_f8_key_sched_t

+ +
+
+ + + + +
typedef int(* kasumi_init_f8_key_sched_t) (const void *, kasumi_key_sched_t *)
+
+ +
+
+ +

◆ kasumi_init_f9_key_sched_t

+ +
+
+ + + + +
typedef int(* kasumi_init_f9_key_sched_t) (const void *, kasumi_key_sched_t *)
+
+ +
+
+ +

◆ kasumi_key_sched_size_t

+ +
+
+ + + + +
typedef size_t(* kasumi_key_sched_size_t) (void)
+
+ +
+
+ +

◆ kasumi_key_sched_t

+ +
+
+ + + + +
typedef struct kasumi_key_sched_s kasumi_key_sched_t
+
+

Structure to maintain internal key scheduling

+ +
+
+ +

◆ keyexp_t

+ +
+
+ + + + +
typedef void(* keyexp_t) (const void *, void *, void *)
+
+ +
+
+ +

◆ queue_size_t

+ +
+
+ + + + +
typedef uint32_t(* queue_size_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ snow3g_f8_1_buffer_bit_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_1_buffer_bit_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_1_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_2_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_2_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_4_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_4_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_8_buffer_multikey_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_8_buffer_multikey_t) (const snow3g_key_schedule_t *const[], const void *const[], const void *const[], void *[], const uint32_t[])
+
+ +
+
+ +

◆ snow3g_f8_8_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_8_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_n_buffer_multikey_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_n_buffer_multikey_t) (const snow3g_key_schedule_t *const[], const void *const[], const void *const[], void *[], const uint32_t[], const uint32_t)
+
+ +
+
+ +

◆ snow3g_f8_n_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f8_n_buffer_t) (const snow3g_key_schedule_t *, const void *const[], const void *const[], void *[], const uint32_t[], const uint32_t)
+
+ +
+
+ +

◆ snow3g_f9_1_buffer_t

+ +
+
+ + + + +
typedef void(* snow3g_f9_1_buffer_t) (const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
+
+ +
+
+ +

◆ snow3g_init_key_sched_t

+ +
+
+ + + + +
typedef int(* snow3g_init_key_sched_t) (const void *, snow3g_key_schedule_t *)
+
+ +
+
+ +

◆ snow3g_key_sched_size_t

+ +
+
+ + + + +
typedef size_t(* snow3g_key_sched_size_t) (void)
+
+ +
+
+ +

◆ snow3g_key_schedule_t

+ +
+
+ + + + +
typedef struct snow3g_key_schedule_s snow3g_key_schedule_t
+
+

Snow3G key scheduling structure

+ +
+
+ +

◆ submit_burst_t

+ +
+
+ + + + +
typedef uint32_t(* submit_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t)
+
+ +
+
+ +

◆ submit_cipher_burst_t

+ +
+
+ + + + +
typedef uint32_t(* submit_cipher_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
+
+ +
+
+ +

◆ submit_hash_burst_t

+ +
+
+ + + + +
typedef uint32_t(* submit_hash_burst_t) (struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
+
+ +
+
+ +

◆ submit_job_t

+ +
+
+ + + + +
typedef IMB_JOB *(* submit_job_t) (struct IMB_MGR *)
+
+ +
+
+ +

◆ xcbc_keyexp_t

+ +
+
+ + + + +
typedef void(* xcbc_keyexp_t) (const void *, void *, void *, void *)
+
+ +
+
+ +

◆ zuc_eea3_1_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_1_buffer_t) (const void *, const void *, const void *, void *, const uint32_t)
+
+ +
+
+ +

◆ zuc_eea3_4_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_4_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
+
+ +
+
+ +

◆ zuc_eea3_n_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eea3_n_buffer_t) (const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
+
+ +
+
+ +

◆ zuc_eia3_1_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eia3_1_buffer_t) (const void *, const void *, const void *, const uint32_t, uint32_t *)
+
+ +
+
+ +

◆ zuc_eia3_n_buffer_t

+ +
+
+ + + + +
typedef void(* zuc_eia3_n_buffer_t) (const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
+
+ +
+
+

Enumeration Type Documentation

+ +

◆ IMB_ARCH

+ +
+
+ + + + +
enum IMB_ARCH
+
+

Architecture definitions

+ + + + + + + + +
Enumerator
IMB_ARCH_NONE 
IMB_ARCH_NOAESNI 
IMB_ARCH_SSE 
IMB_ARCH_AVX 
IMB_ARCH_AVX2 
IMB_ARCH_AVX512 
IMB_ARCH_NUM 
+ +
+
+ +

◆ IMB_CHAIN_ORDER

+ +
+
+ + + + +
enum IMB_CHAIN_ORDER
+
+ + + +
Enumerator
IMB_ORDER_CIPHER_HASH 
IMB_ORDER_HASH_CIPHER 
+ +
+
+ +

◆ IMB_CIPHER_DIRECTION

+ +
+
+ + + + +
enum IMB_CIPHER_DIRECTION
+
+ + + +
Enumerator
IMB_DIR_ENCRYPT 
IMB_DIR_DECRYPT 
+ +
+
+ +

◆ IMB_CIPHER_MODE

+ +
+
+ + + + +
enum IMB_CIPHER_MODE
+
+

IMB_ERR_MIN should be higher than __ELASTERROR to avoid overlap with standard error values Define enums from API v0.53, so applications that were using this version will still be compiled successfully. Note: this list has been extended with new names after version 0.55. This list does not need to be extended for new enums.

+ + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_CIPHER_CBC 
IMB_CIPHER_CNTR 
IMB_CIPHER_NULL 
IMB_CIPHER_DOCSIS_SEC_BPI 
IMB_CIPHER_GCM 
IMB_CIPHER_CUSTOM 
IMB_CIPHER_DES 
IMB_CIPHER_DOCSIS_DES 
IMB_CIPHER_CCM 
IMB_CIPHER_DES3 
IMB_CIPHER_PON_AES_CNTR 
IMB_CIPHER_ECB 
IMB_CIPHER_CNTR_BITLEN 

128-EEA2/NEA2 (3GPP)

+
IMB_CIPHER_ZUC_EEA3 

128-EEA3/NEA3 (3GPP)

+
IMB_CIPHER_SNOW3G_UEA2_BITLEN 

128-UEA2 (3GPP)

+
IMB_CIPHER_KASUMI_UEA1_BITLEN 

128-UEA1 (3GPP)

+
IMB_CIPHER_CBCS_1_9 

MPEG CENC (ISO 23001-7)

+
IMB_CIPHER_CHACHA20 
IMB_CIPHER_CHACHA20_POLY1305 

AEAD CHACHA20

+
IMB_CIPHER_CHACHA20_POLY1305_SGL 

AEAD CHACHA20 with SGL support

+
IMB_CIPHER_SNOW_V 
IMB_CIPHER_SNOW_V_AEAD 
IMB_CIPHER_GCM_SGL 
IMB_CIPHER_NUM 
+ +
+
+ +

◆ IMB_ERR

+ +
+
+ + + + +
enum IMB_ERR
+
+

Library error types

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_ERR_MIN 
IMB_ERR_NULL_MBMGR 
IMB_ERR_JOB_NULL_SRC 
IMB_ERR_JOB_NULL_DST 
IMB_ERR_JOB_NULL_KEY 
IMB_ERR_JOB_NULL_IV 
IMB_ERR_JOB_NULL_AUTH 
IMB_ERR_JOB_NULL_AAD 
IMB_ERR_JOB_CIPH_LEN 
IMB_ERR_JOB_AUTH_LEN 
IMB_ERR_JOB_IV_LEN 
IMB_ERR_JOB_KEY_LEN 
IMB_ERR_JOB_AUTH_TAG_LEN 
IMB_ERR_JOB_AAD_LEN 
IMB_ERR_JOB_SRC_OFFSET 
IMB_ERR_JOB_CHAIN_ORDER 
IMB_ERR_CIPH_MODE 
IMB_ERR_HASH_ALGO 
IMB_ERR_JOB_NULL_AUTH_KEY 
IMB_ERR_JOB_NULL_SGL_CTX 
IMB_ERR_JOB_NULL_NEXT_IV 
IMB_ERR_JOB_PON_PLI 
IMB_ERR_NULL_SRC 
IMB_ERR_NULL_DST 
IMB_ERR_NULL_KEY 
IMB_ERR_NULL_EXP_KEY 
IMB_ERR_NULL_IV 
IMB_ERR_NULL_AUTH 
IMB_ERR_NULL_AAD 
IMB_ERR_CIPH_LEN 
IMB_ERR_AUTH_LEN 
IMB_ERR_IV_LEN 
IMB_ERR_KEY_LEN 
IMB_ERR_AUTH_TAG_LEN 
IMB_ERR_AAD_LEN 
IMB_ERR_SRC_OFFSET 
IMB_ERR_NULL_AUTH_KEY 
IMB_ERR_NULL_CTX 
IMB_ERR_NO_AESNI_EMU 
IMB_ERR_JOB_NULL_HMAC_OPAD 
IMB_ERR_JOB_NULL_HMAC_IPAD 
IMB_ERR_JOB_NULL_XCBC_K1_EXP 
IMB_ERR_JOB_NULL_XCBC_K2 
IMB_ERR_JOB_NULL_XCBC_K3 
IMB_ERR_JOB_CIPH_DIR 
IMB_ERR_JOB_NULL_GHASH_INIT_TAG 
IMB_ERR_MISSING_CPUFLAGS_INIT_MGR 
IMB_ERR_NULL_JOB 
IMB_ERR_MAX 
+ +
+
+ +

◆ IMB_HASH_ALG

+ +
+
+ + + + +
enum IMB_HASH_ALG
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Enumerator
IMB_AUTH_HMAC_SHA_1 

HMAC-SHA1

+
IMB_AUTH_HMAC_SHA_224 

HMAC-SHA224

+
IMB_AUTH_HMAC_SHA_256 

HMAC-SHA256

+
IMB_AUTH_HMAC_SHA_384 

HMAC-SHA384

+
IMB_AUTH_HMAC_SHA_512 

HMAC-SHA512

+
IMB_AUTH_AES_XCBC 
IMB_AUTH_MD5 

HMAC-MD5

+
IMB_AUTH_NULL 
IMB_AUTH_AES_GMAC 
IMB_AUTH_CUSTOM 
IMB_AUTH_AES_CCM 

AES128-CCM

+
IMB_AUTH_AES_CMAC 

AES128-CMAC

+
IMB_AUTH_SHA_1 

SHA1

+
IMB_AUTH_SHA_224 

SHA224

+
IMB_AUTH_SHA_256 

SHA256

+
IMB_AUTH_SHA_384 

SHA384

+
IMB_AUTH_SHA_512 

SHA512

+
IMB_AUTH_AES_CMAC_BITLEN 

128-EIA2/NIA2 (3GPP)

+
IMB_AUTH_PON_CRC_BIP 
IMB_AUTH_ZUC_EIA3_BITLEN 

128-EIA3/NIA3 (3GPP)

+
IMB_AUTH_DOCSIS_CRC32 

with DOCSIS_SEC_BPI only

+
IMB_AUTH_SNOW3G_UIA2_BITLEN 

128-UIA2 (3GPP)

+
IMB_AUTH_KASUMI_UIA1 

128-UIA1 (3GPP)

+
IMB_AUTH_AES_GMAC_128 

AES-GMAC (128-bit key)

+
IMB_AUTH_AES_GMAC_192 

AES-GMAC (192-bit key)

+
IMB_AUTH_AES_GMAC_256 

AES-GMAC (256-bit key)

+
IMB_AUTH_AES_CMAC_256 

AES256-CMAC

+
IMB_AUTH_POLY1305 

POLY1305

+
IMB_AUTH_CHACHA20_POLY1305 

AEAD POLY1305

+
IMB_AUTH_CHACHA20_POLY1305_SGL 

AEAD CHACHA20 with SGL support

+
IMB_AUTH_ZUC256_EIA3_BITLEN 

256-EIA3/NIA3 (3GPP)

+
IMB_AUTH_SNOW_V_AEAD 

SNOW-V-AEAD

+
IMB_AUTH_GCM_SGL 

AES-GCM with SGL support

+
IMB_AUTH_CRC32_ETHERNET_FCS 

CRC32-ETHERNET-FCS

+
IMB_AUTH_CRC32_SCTP 

CRC32-SCTP

+
IMB_AUTH_CRC32_WIMAX_OFDMA_DATA 

CRC32-WIMAX-OFDMA-DATA

+
IMB_AUTH_CRC24_LTE_A 

CRC32-LTE-A

+
IMB_AUTH_CRC24_LTE_B 

CRC32-LTE-B

+
IMB_AUTH_CRC16_X25 

CRC16-X25

+
IMB_AUTH_CRC16_FP_DATA 

CRC16-FP-DATA

+
IMB_AUTH_CRC11_FP_HEADER 

CRC11-FP-HEADER

+
IMB_AUTH_CRC10_IUUP_DATA 

CRC10-IUUP-DATA

+
IMB_AUTH_CRC8_WIMAX_OFDMA_HCS 

CRC8-WIMAX-OFDMA-HCS

+
IMB_AUTH_CRC7_FP_HEADER 

CRC7-FP-HEADER

+
IMB_AUTH_CRC6_IUUP_HEADER 

CRC6-IUUP-HEADER

+
IMB_AUTH_GHASH 

GHASH

+
IMB_AUTH_NUM 
+ +
+
+ +

◆ IMB_KEY_SIZE_BYTES

+ +
+
+ + + + +
enum IMB_KEY_SIZE_BYTES
+
+ + + + +
Enumerator
IMB_KEY_128_BYTES 
IMB_KEY_192_BYTES 
IMB_KEY_256_BYTES 
+ +
+
+ +

◆ IMB_SGL_STATE

+ +
+
+ + + + +
enum IMB_SGL_STATE
+
+ + + + + +
Enumerator
IMB_SGL_INIT 
IMB_SGL_UPDATE 
IMB_SGL_COMPLETE 
IMB_SGL_ALL 
+ +
+
+ +

◆ IMB_STATUS

+ +
+
+ + + + +
enum IMB_STATUS
+
+

Job structure definitions

+ + + + + + + + +
Enumerator
IMB_STATUS_BEING_PROCESSED 
IMB_STATUS_COMPLETED_CIPHER 
IMB_STATUS_COMPLETED_AUTH 
IMB_STATUS_COMPLETED 

COMPLETED_CIPHER | COMPLETED_AUTH

+
IMB_STATUS_INVALID_ARGS 
IMB_STATUS_INTERNAL_ERROR 
IMB_STATUS_ERROR 
+ +
+
+

Function Documentation

+ +

◆ aes_cfb_128_one_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cfb_128_one_avx (void * out,
const void * in,
const void * iv,
const void * keys,
uint64_t len 
)
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + +
[out]outPlaintext/Ciphertext output
[in]inPlaintext/Ciphertext input
[in]ivPointer to 16 byte IV
[in]keysPointer to expanded AES keys
[in]lenLength of data in bytes
+
+
+ +
+
+ +

◆ aes_cfb_128_one_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cfb_128_one_avx2 (void * out,
const void * in,
const void * iv,
const void * keys,
uint64_t len 
)
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + +
[out]outPlaintext/Ciphertext output
[in]inPlaintext/Ciphertext input
[in]ivPointer to 16 byte IV
[in]keysPointer to expanded AES keys
[in]lenLength of data in bytes
+
+
+ +
+
+ +

◆ aes_cfb_128_one_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cfb_128_one_avx512 (void * out,
const void * in,
const void * iv,
const void * keys,
uint64_t len 
)
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + +
[out]outPlaintext/Ciphertext output
[in]inPlaintext/Ciphertext input
[in]ivPointer to 16 byte IV
[in]keysPointer to expanded AES keys
[in]lenLength of data in bytes
+
+
+ +
+
+ +

◆ aes_cfb_128_one_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cfb_128_one_sse (void * out,
const void * in,
const void * iv,
const void * keys,
uint64_t len 
)
+
+ +

AES-CFB-128 Encrypt/Decrypt up to one block.

+

Processes only one buffer at a time. Designed to manage partial blocks of DOCSIS 3.1 SEC BPI.

+
Parameters
+ + + + + + +
[out]outPlaintext/Ciphertext output
[in]inPlaintext/Ciphertext input
[in]ivPointer to 16 byte IV
[in]keysPointer to expanded AES keys
[in]lenLength of data in bytes
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2 (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512 (const void * key_exp,
void * key1,
void * key2 
)
+
+ +

+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_cmac_subkey_gen_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse (const void * key_exp,
void * key1,
void * key2 
)
+
+

Generate AES-128-CMAC subkeys.

+
Parameters
+ + + + +
[in]key_expInput expanded AES-128-CMAC key
[out]key1Subkey 1
[out]key2Subkey 2
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_128_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_192_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End decryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Decryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_dec_256_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Decrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outPlaintext output. Decrypt in-place is allowed
[in]inCiphertext input
[in]lenLength of data in bytes for decryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-128 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_128_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-128 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-192 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_192_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-192 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_finalize_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

End encryption of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8.
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
uint8_t const * in,
uint64_t len,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len,
uint8_t * auth_tag,
uint64_t auth_tag_len 
)
+
+ +

AES-GCM-256 Encryption.

+
Parameters
+ + + + + + + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authentication Data (AAD)
[in]aad_lenLength of AAD in bytes
[out]auth_tagAuthenticated Tag output
[in]auth_tag_lenAuthenticated Tag Length in bytes (must be a multiple of 4 bytes). Valid values are 16 (most likely), 12 or 8
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_enc_256_update_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
uint8_t * out,
const uint8_t * in,
uint64_t len 
)
+
+ +

Encrypt a block of a AES-GCM-256 encryption message.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[out]outCiphertext output. Encrypt in-place is allowed
[in]inPlaintext input
[in]lenLength of data in bytes for encryption
+
+
+ +
+
+ +

◆ aes_gcm_init_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_128_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_192_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4 (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_init_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_init_256_sse (const struct gcm_key_data * key_data,
struct gcm_context_data * context_data,
const uint8_t * iv,
uint8_t const * aad,
uint64_t aad_len 
)
+
+ +

Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.

+
Parameters
+ + + + + + +
[in]key_dataGCM expanded key data
[in,out]context_dataGCM operation context data
[in]ivPointer to 12 byte IV structure Internally, the library concatenates 0x00000001 to the IV
[in]aadAdditional Authenticated Data (AAD)
[in]aad_lenLength of AAD in bytes
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_128_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-128 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_192_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-192 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_avx_gen2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_avx_gen4()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4 (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_pre_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_pre_256_sse (const void * key,
struct gcm_key_data * key_data 
)
+
+ +

Pre-processes AES-GCM-256 key data.

+

Prefills the gcm key data with key values for each round and the initial sub hash key for tag encoding

+
Parameters
+ + + +
[in]keyPointer to key data
[out]key_dataGCM expanded key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_128_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-128 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_192_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-192 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_avx_gen2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_avx_gen4()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4 (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_gcm_precomp_256_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse (struct gcm_key_data * key_data)
+
+ +

Precomputation of AES-GCM-256 HashKey constants.

+

Precomputation of HashKey<<1 mod poly constants (shifted_hkey_X and shifted_hkey_X_k).

+
Parameters
+ + +
[in,out]key_dataGCM key data
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-128 expansion keys.

+
Parameters
+ + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_128_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_128_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-128 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-128 key
[out]enc_exp_keysAES-128 encryption expansion key
[out]dec_exp_keysAES-128 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-192 expansion keys.

+
Parameters
+ + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_192_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_192_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-192 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-192 key
[out]enc_exp_keysAES-192 encryption expansion key
[out]dec_exp_keysAES-192 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx2 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_avx512 (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+ +

+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512 (const void * key,
void * enc_exp_keys 
)
+
+ +

+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_enc_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse (const void * key,
void * enc_exp_keys 
)
+
+

Generate encryption AES-256 expansion keys.

+
Parameters
+ + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
+
+
+ +
+
+ +

◆ aes_keyexp_256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_keyexp_256_sse (const void * key,
void * enc_exp_keys,
void * dec_exp_keys 
)
+
+

Generate encryption/decryption AES-256 expansion keys.

+
Parameters
+ + + + +
[in]keyAES-256 key
[out]enc_exp_keysAES-256 encryption expansion key
[out]dec_exp_keysAES-256 decryption expansion key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2 (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512 (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+ +

+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ aes_xcbc_expand_key_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse (const void * key,
void * k1_exp,
void * k2,
void * k3 
)
+
+

Generate AES-128-XCBC expansion keys.

+
Parameters
+ + + + + +
[in]keyInput AES-128-XCBC key
[out]k1_expk1 expansion key
[out]k2k2 key
[out]k3k3 key
+
+
+ +
+
+ +

◆ alloc_mb_mgr()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr (uint64_t flags)
+
+ +

Allocates memory for multi-buffer manager instance.

+

get_next_job returns a job object. This must be filled in and returned via submit_job before get_next_job is called again. After submit_job is called, one should call get_completed_job() at least once (and preferably until it returns NULL). get_completed_job and flush_job returns a job object. This job object ceases to be usable at the next call to get_next_job

+

For binary compatibility between library versions it is recommended to use this API.

+
Parameters
+ + +
flagsmulti-buffer manager flags IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, currently SHANI is only available for SSE IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions.
+
+
+
Returns
Pointer to allocated memory for IMB_MGR structure
+
Return values
+ + +
NULLon allocation error
+
+
+ +
+
+ +

◆ des_key_schedule()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int des_key_schedule (uint64_t * ks,
const void * key 
)
+
+ +

DES key schedule set up.

+

ks buffer needs to accommodate DES_KEY_SCHED_SIZE (128) bytes of data.

+
Parameters
+ + + +
[out]ksDestination buffer to accommodate DES key schedule
[in]keyPointer to an 8 byte DES key
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
!0error
+
+
+ +
+
+ +

◆ flush_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2 (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512 (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ flush_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * flush_job_sse (IMB_MGR * state)
+
+ +

Force processing until next job in queue is completed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no more jobs to process
+ +
+
+ +

◆ free_mb_mgr()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void free_mb_mgr (IMB_MGR * ptr)
+
+ +

Frees memory allocated previously by alloc_mb_mgr()

+
Parameters
+ + +
[in]ptrPointer to allocated MB_MGR structure
+
+
+ +
+
+ +

◆ get_completed_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2 (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512 (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_completed_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse (IMB_MGR * state)
+
+ +

Get next completed job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if next job not complete
+ +
+
+ +

◆ get_next_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2 (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512 (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ get_next_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse (IMB_MGR * state)
+
+ +

Get next available job.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to next free IMB_JOB in the queue
+ +
+
+ +

◆ imb_clear_mem()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void imb_clear_mem (void * mem,
const size_t size 
)
+
+ +

Force clearing/zeroing of memory.

+
Parameters
+ + + +
[in]memPointer to memory address to clear
[in]sizeSize of memory to clear (in bytes)
+
+
+ +
+
+ +

◆ imb_get_errno()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT int imb_get_errno (IMB_MGR * mb_mgr)
+
+ +

API to get error status.

+
Parameters
+ + +
mb_mgrPointer to multi-buffer manager
+
+
+
Return values
+ + +
Integererror type
+
+
+ +
+
+ +

◆ imb_get_feature_flags()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint64_t imb_get_feature_flags (void )
+
+ +

Retrieves the bitmask with the features supported by the library, without having to allocate/initialize IMB_MGR;.

+
Returns
Bitmask containing feature flags
+ +
+
+ +

◆ imb_get_mb_mgr_size()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size (void )
+
+ +

Calculates necessary memory size for IMB_MGR.

+
Returns
Size for IMB_MGR (aligned to 64 bytes)
+ +
+
+ +

◆ imb_get_strerror()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT const char * imb_get_strerror (int errnum)
+
+ +

API to get description for errnum.

+
Parameters
+ + +
errnumerror type
+
+
+
Return values
+ + +
Stringdescription of errnum
+
+
+ +
+
+ +

◆ imb_get_version()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT unsigned imb_get_version (void )
+
+ +

Get library version in numerical format.

+

Use IMB_VERSION() macro to compare this numerical version against known library version.

+
Returns
library version number
+ +
+
+ +

◆ imb_get_version_str()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT const char * imb_get_version_str (void )
+
+ +

Get library version in string format.

+

API definitions

+
Returns
library version string
+ +
+
+ +

◆ imb_set_pointers_mb_mgr()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr (void * ptr,
const uint64_t flags,
const unsigned reset_mgr 
)
+
+ +

Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory.

+

imb_get_mb_mgr_size() should be called to know how much memory should be allocated externally.

+

init_mb_mgr_XXX() must be called after this function call, whereas XXX is the desired architecture.

+
Parameters
+ + + + +
[in]ptra pointer to allocated memory
[in]flagsmulti-buffer manager flags IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, currently SHANI is only available for SSE IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions.
[in]reset_mgrif 0, IMB_MGR structure is not cleared, else it is.
+
+
+
Returns
Pointer to IMB_MGR structure
+ +
+
+ +

◆ init_mb_mgr_auto()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_auto (IMB_MGR * state,
IMB_ARCH * arch 
)
+
+ +

Automatically initialize most performant Multi-buffer manager based on CPU features.

+
Parameters
+ + + +
[in]statePointer to MB_MGR struct
[out]archPointer to arch enum to be set (can be NULL)
+
+
+ +
+
+ +

◆ init_mb_mgr_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx2 (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_avx512 (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ init_mb_mgr_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT void init_mb_mgr_sse (IMB_MGR * state)
+
+ +

Initialize Multi-Buffer Manager structure.

+

Must be called before calling JOB/BURST API.

+
Parameters
+ + +
[in,out]statePointer to IMB_MGR structure For binary compatibility between library versions, it is recommended to allocate the IMB_MGR structure using the alloc_mb_mgr() API
+
+
+ +
+
+ +

◆ kasumi_f8_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int kasumi_f8_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of KASUMI F8 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ kasumi_f9_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int kasumi_f9_iv_gen (const uint32_t count,
const uint32_t fresh,
void * iv_ptr 
)
+
+ +

Generation of KASUMI F9 Initialization Vector.

+
Parameters
+ + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]freshFRESH (4 bytes in Little Endian)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ md5_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ md5_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void md5_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with MD5.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (16 bytes)
+
+
+ +
+
+ +

◆ queue_size_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx2 (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_avx512 (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ queue_size_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT uint32_t queue_size_sse (IMB_MGR * state)
+
+ +

Get number of jobs queued to be processed.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Number of jobs in the queue
+ +
+
+ +

◆ sha1_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_avx (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_avx2 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_avx512 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with SHA1.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha1_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha1_sse (const void * data,
const uint64_t length,
void * digest 
)
+
+

Authenticate variable sized data with SHA1.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (20 bytes)
+
+
+ +
+
+ +

◆ sha224_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_avx (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_avx2 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_avx512 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with SHA224.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha224_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha224_sse (const void * data,
const uint64_t length,
void * digest 
)
+
+

Authenticate variable sized data with SHA224.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (28 bytes)
+
+
+ +
+
+ +

◆ sha256_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_avx (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_avx2 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_avx512 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with SHA256.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha256_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha256_sse (const void * data,
const uint64_t length,
void * digest 
)
+
+

Authenticate variable sized data with SHA256.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (32 bytes)
+
+
+ +
+
+ +

◆ sha384_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_avx (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_avx2 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_avx512 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 128-byte data buffer with SHA384.

+
Parameters
+ + + +
[in]data64-byte data buffer
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha384_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha384_sse (const void * data,
const uint64_t length,
void * digest 
)
+
+

Authenticate variable sized data with SHA384.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (48 bytes)
+
+
+ +
+
+ +

◆ sha512_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_avx (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_avx2 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_avx512 (const void * data,
const uint64_t length,
void * digest 
)
+
+ +

+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_one_block_avx()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_one_block_avx (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA512.

+
Parameters
+ + + +
[in]data128-byte data buffer
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_one_block_avx2()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_one_block_avx2 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA512.

+
Parameters
+ + + +
[in]data128-byte data buffer
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_one_block_avx512()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_one_block_avx512 (const void * data,
void * digest 
)
+
+ +

+

Authenticate 64-byte data buffer with SHA512.

+
Parameters
+ + + +
[in]data128-byte data buffer
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_one_block_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_one_block_sse (const void * data,
void * digest 
)
+
+

Authenticate 64-byte data buffer with SHA512.

+
Parameters
+ + + +
[in]data128-byte data buffer
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ sha512_sse()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT void sha512_sse (const void * data,
const uint64_t length,
void * digest 
)
+
+

Authenticate variable sized data with SHA512.

+
Parameters
+ + + + +
[in]dataData buffer
[in]lengthLength of data in bytes for authentication.
[out]digestDigest output (64 bytes)
+
+
+ +
+
+ +

◆ snow3g_f8_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int snow3g_f8_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of SNOW3G F8 Initialization Vector.

+

Parameters are passed in Little Endian format and used to generate the IV in Big Endian format.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes) in Big Endian format
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ snow3g_f9_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int snow3g_f9_iv_gen (const uint32_t count,
const uint32_t fresh,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of SNOW3G F9 Initialization Vector.

+

Parameters are passed in Little Endian format and used to generate the IV in Big Endian format.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]freshFRESH (4 bytes in Little Endian)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes) in Big Endian format
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ submit_job_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2 (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512 (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ submit_job_nocheck_avx()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_avx2()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2 (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_avx512()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512 (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_nocheck_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse (IMB_MGR * state)
+
+ +

Submit job for processing without validating.

+

This is more performant but less secure than submit_job_xxx()

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed
+ +
+
+ +

◆ submit_job_sse()

+ +
+
+ + + + + + + + +
IMB_DLL_EXPORT IMB_JOB * submit_job_sse (IMB_MGR * state)
+
+ +

Submit job for processing after validating.

+
Parameters
+ + +
[in,out]statePointer to initialized IMB_MGR structure
+
+
+
Returns
Pointer to completed IMB_JOB or NULL if no job completed If NULL, imb_get_errno() can be used to check for potential error conditions
+ +
+
+ +

◆ zuc_eea3_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int zuc_eea3_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of ZUC-EEA3 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+ +

◆ zuc_eia3_iv_gen()

+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IMB_DLL_EXPORT int zuc_eia3_iv_gen (const uint32_t count,
const uint8_t bearer,
const uint8_t dir,
void * iv_ptr 
)
+
+ +

Generation of ZUC-EIA3 Initialization Vector.

+
Parameters
+ + + + + +
[in]countCOUNT (4 bytes in Little Endian)
[in]bearerBEARER (5 bits)
[in]dirDIRECTION (1 bit)
[out]iv_ptrPointer to generated IV (16 bytes)
+
+
+
Returns
Operation status
+
Return values
+ + + +
0success
-1if one or more parameters are invalid
+
+
+ +
+
+
+
+ + + + diff --git a/docs/intel-ipsec-mb_8h.js b/docs/intel-ipsec-mb_8h.js new file mode 100644 index 00000000..16684a79 --- /dev/null +++ b/docs/intel-ipsec-mb_8h.js @@ -0,0 +1,630 @@ +var intel_ipsec_mb_8h = +[ + [ "imb_uint128_t", "structimb__uint128__t.html", "structimb__uint128__t" ], + [ "IMB_SGL_IOV", "structIMB__SGL__IOV.html", "structIMB__SGL__IOV" ], + [ "IMB_JOB", "structIMB__JOB.html", "structIMB__JOB" ], + [ "kasumi_key_sched_s", "structkasumi__key__sched__s.html", "structkasumi__key__sched__s" ], + [ "gcm_context_data", "structgcm__context__data.html", "structgcm__context__data" ], + [ "chacha20_poly1305_context_data", "structchacha20__poly1305__context__data.html", "structchacha20__poly1305__context__data" ], + [ "gcm_key_data", "structgcm__key__data.html", "structgcm__key__data" ], + [ "snow3g_key_schedule_s", "structsnow3g__key__schedule__s.html", "structsnow3g__key__schedule__s" ], + [ "IMB_MGR", "structIMB__MGR.html", "structIMB__MGR" ], + [ "__forceinline", "intel-ipsec-mb_8h.html#af93b819ac40799ac392e16f6a90729fd", null ], + [ "DECLARE_ALIGNED", "intel-ipsec-mb_8h.html#ad920a47ed8f57af8b2cf6bcd18a3e41c", null ], + [ "IMB_AES128_CFB_ONE", "intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547", null ], + [ "IMB_AES128_GCM_DEC", "intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709", null ], + [ "IMB_AES128_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133", null ], + [ "IMB_AES128_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829", null ], + [ "IMB_AES128_GCM_ENC", "intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6", null ], + [ "IMB_AES128_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be", null ], + [ "IMB_AES128_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db", null ], + [ "IMB_AES128_GCM_INIT", "intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef", null ], + [ "IMB_AES128_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6", null ], + [ "IMB_AES128_GCM_PRE", "intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1", null ], + [ "IMB_AES128_GCM_PRECOMP", "intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e", null ], + [ "IMB_AES128_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad", null ], + [ "IMB_AES128_GMAC_INIT", "intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964", null ], + [ "IMB_AES128_GMAC_UPDATE", "intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a", null ], + [ "IMB_AES192_GCM_DEC", "intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047", null ], + [ "IMB_AES192_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b", null ], + [ "IMB_AES192_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f", null ], + [ "IMB_AES192_GCM_ENC", "intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9", null ], + [ "IMB_AES192_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd", null ], + [ "IMB_AES192_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957", null ], + [ "IMB_AES192_GCM_INIT", "intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f", null ], + [ "IMB_AES192_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262", null ], + [ "IMB_AES192_GCM_PRE", "intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471", null ], + [ "IMB_AES192_GCM_PRECOMP", "intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b", null ], + [ "IMB_AES192_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb", null ], + [ "IMB_AES192_GMAC_INIT", "intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231", null ], + [ "IMB_AES192_GMAC_UPDATE", "intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8", null ], + [ "IMB_AES256_GCM_DEC", "intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1", null ], + [ "IMB_AES256_GCM_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e", null ], + [ "IMB_AES256_GCM_DEC_UPDATE", "intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd", null ], + [ "IMB_AES256_GCM_ENC", "intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9", null ], + [ "IMB_AES256_GCM_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877", null ], + [ "IMB_AES256_GCM_ENC_UPDATE", "intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a", null ], + [ "IMB_AES256_GCM_INIT", "intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3", null ], + [ "IMB_AES256_GCM_INIT_VAR_IV", "intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded", null ], + [ "IMB_AES256_GCM_PRE", "intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754", null ], + [ "IMB_AES256_GCM_PRECOMP", "intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda", null ], + [ "IMB_AES256_GMAC_FINALIZE", "intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e", null ], + [ "IMB_AES256_GMAC_INIT", "intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092", null ], + [ "IMB_AES256_GMAC_UPDATE", "intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b", null ], + [ "IMB_AES_BLOCK_SIZE", "intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793", null ], + [ "IMB_AES_CMAC_SUBKEY_GEN_128", "intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5", null ], + [ "IMB_AES_CMAC_SUBKEY_GEN_256", "intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0", null ], + [ "IMB_AES_KEYEXP_128", "intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0", null ], + [ "IMB_AES_KEYEXP_192", "intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d", null ], + [ "IMB_AES_KEYEXP_256", "intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f", null ], + [ "IMB_AES_XCBC_KEYEXP", "intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda", null ], + [ "IMB_ASSERT", "intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d", null ], + [ "IMB_CHACHA20_POLY1305_DEC_FINALIZE", "intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0", null ], + [ "IMB_CHACHA20_POLY1305_DEC_UPDATE", "intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed", null ], + [ "IMB_CHACHA20_POLY1305_ENC_FINALIZE", "intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031", null ], + [ "IMB_CHACHA20_POLY1305_ENC_UPDATE", "intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b", null ], + [ "IMB_CHACHA20_POLY1305_INIT", "intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643", null ], + [ "IMB_CPUFLAGS_AVX", "intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576", null ], + [ "IMB_CPUFLAGS_AVX2", "intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d", null ], + [ "IMB_CPUFLAGS_AVX512", "intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d", null ], + [ "IMB_CPUFLAGS_NO_AESNI", "intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388", null ], + [ "IMB_CPUFLAGS_SSE", "intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c", null ], + [ "IMB_CRC10_IUUP_DATA", "intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961", null ], + [ "IMB_CRC11_FP_HEADER", "intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732", null ], + [ "IMB_CRC16_FP_DATA", "intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6", null ], + [ "IMB_CRC16_X25", "intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c", null ], + [ "IMB_CRC24_LTE_A", "intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b", null ], + [ "IMB_CRC24_LTE_B", "intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03", null ], + [ "IMB_CRC32_ETHERNET_FCS", "intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af", null ], + [ "IMB_CRC32_SCTP", "intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169", null ], + [ "IMB_CRC32_WIMAX_OFDMA_DATA", "intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e", null ], + [ "IMB_CRC6_IUUP_HEADER", "intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34", null ], + [ "IMB_CRC7_FP_HEADER", "intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b", null ], + [ "IMB_CRC8_WIMAX_OFDMA_HCS", "intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d", null ], + [ "IMB_DES_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649", null ], + [ "IMB_DES_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7", null ], + [ "IMB_DES_KEYSCHED", "intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6", null ], + [ "IMB_DIM", "intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d", null ], + [ "IMB_DLL_EXPORT", "intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40", null ], + [ "IMB_DLL_LOCAL", "intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e", null ], + [ "IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE", "intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af", null ], + [ "IMB_DOCSIS_CRC32_TAG_SIZE", "intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758", null ], + [ "IMB_FEATURE_AESNI", "intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05", null ], + [ "IMB_FEATURE_AESNI_EMU", "intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed", null ], + [ "IMB_FEATURE_AVX", "intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056", null ], + [ "IMB_FEATURE_AVX2", "intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530", null ], + [ "IMB_FEATURE_AVX512_IFMA", "intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48", null ], + [ "IMB_FEATURE_AVX512_SKX", "intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964", null ], + [ "IMB_FEATURE_AVX512BW", "intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f", null ], + [ "IMB_FEATURE_AVX512CD", "intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9", null ], + [ "IMB_FEATURE_AVX512DQ", "intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1", null ], + [ "IMB_FEATURE_AVX512F", "intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00", null ], + [ "IMB_FEATURE_AVX512VL", "intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8", null ], + [ "IMB_FEATURE_BMI2", "intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09", null ], + [ "IMB_FEATURE_CMOV", "intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c", null ], + [ "IMB_FEATURE_GFNI", "intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567", null ], + [ "IMB_FEATURE_PCLMULQDQ", "intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5", null ], + [ "IMB_FEATURE_SAFE_DATA", "intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b", null ], + [ "IMB_FEATURE_SAFE_PARAM", "intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080", null ], + [ "IMB_FEATURE_SHANI", "intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90", null ], + [ "IMB_FEATURE_SSE4_2", "intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99", null ], + [ "IMB_FEATURE_VAES", "intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3", null ], + [ "IMB_FEATURE_VPCLMULQDQ", "intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614", null ], + [ "IMB_FLAG_AESNI_OFF", "intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55", null ], + [ "IMB_FLAG_SHANI_OFF", "intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c", null ], + [ "IMB_FLUSH_JOB", "intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72", null ], + [ "IMB_GCM_128_KEY_LEN", "intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232", null ], + [ "IMB_GCM_192_KEY_LEN", "intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d", null ], + [ "IMB_GCM_256_KEY_LEN", "intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111", null ], + [ "IMB_GCM_BLOCK_LEN", "intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5", null ], + [ "IMB_GCM_ENC_KEY_LEN", "intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e", null ], + [ "IMB_GCM_IV_DATA_LEN", "intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702", null ], + [ "IMB_GCM_KEY_SETS", "intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b", null ], + [ "IMB_GET_COMPLETED_JOB", "intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117", null ], + [ "IMB_GET_NEXT_JOB", "intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382", null ], + [ "IMB_GHASH", "intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec", null ], + [ "IMB_GHASH_PRE", "intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2", null ], + [ "IMB_HEC_32", "intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac", null ], + [ "IMB_HEC_64", "intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4", null ], + [ "IMB_KASUMI_BLOCK_SIZE", "intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1", null ], + [ "IMB_KASUMI_DIGEST_SIZE", "intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf", null ], + [ "IMB_KASUMI_F8_1_BUFFER", "intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce", null ], + [ "IMB_KASUMI_F8_1_BUFFER_BIT", "intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c", null ], + [ "IMB_KASUMI_F8_2_BUFFER", "intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee", null ], + [ "IMB_KASUMI_F8_3_BUFFER", "intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41", null ], + [ "IMB_KASUMI_F8_4_BUFFER", "intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a", null ], + [ "IMB_KASUMI_F8_N_BUFFER", "intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d", null ], + [ "IMB_KASUMI_F9_1_BUFFER", "intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f", null ], + [ "IMB_KASUMI_F9_1_BUFFER_USER", "intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940", null ], + [ "IMB_KASUMI_INIT_F8_KEY_SCHED", "intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272", null ], + [ "IMB_KASUMI_INIT_F9_KEY_SCHED", "intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92", null ], + [ "IMB_KASUMI_IV_SIZE", "intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440", null ], + [ "IMB_KASUMI_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3", null ], + [ "IMB_KASUMI_KEY_SIZE", "intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102", null ], + [ "IMB_MAX_JOBS", "intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97", null ], + [ "IMB_MAX_TAG_LEN", "intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed", null ], + [ "IMB_MD5_ONE_BLOCK", "intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2", null ], + [ "IMB_QUEUE_SIZE", "intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365", null ], + [ "IMB_SHA1", "intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4", null ], + [ "IMB_SHA1_BLOCK_SIZE", "intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91", null ], + [ "IMB_SHA1_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066", null ], + [ "IMB_SHA1_ONE_BLOCK", "intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b", null ], + [ "IMB_SHA224", "intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360", null ], + [ "IMB_SHA224_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5", null ], + [ "IMB_SHA224_ONE_BLOCK", "intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6", null ], + [ "IMB_SHA256", "intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b", null ], + [ "IMB_SHA256_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731", null ], + [ "IMB_SHA256_ONE_BLOCK", "intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997", null ], + [ "IMB_SHA384", "intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f", null ], + [ "IMB_SHA384_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c", null ], + [ "IMB_SHA384_ONE_BLOCK", "intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b", null ], + [ "IMB_SHA512", "intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c", null ], + [ "IMB_SHA512_DIGEST_SIZE_IN_BYTES", "intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e", null ], + [ "IMB_SHA512_ONE_BLOCK", "intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9", null ], + [ "IMB_SHA_256_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef", null ], + [ "IMB_SHA_384_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855", null ], + [ "IMB_SHA_512_BLOCK_SIZE", "intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d", null ], + [ "IMB_SNOW3G_F8_1_BUFFER", "intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1", null ], + [ "IMB_SNOW3G_F8_1_BUFFER_BIT", "intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5", null ], + [ "IMB_SNOW3G_F8_2_BUFFER", "intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7", null ], + [ "IMB_SNOW3G_F8_4_BUFFER", "intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb", null ], + [ "IMB_SNOW3G_F8_8_BUFFER", "intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932", null ], + [ "IMB_SNOW3G_F8_8_BUFFER_MULTIKEY", "intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a", null ], + [ "IMB_SNOW3G_F8_N_BUFFER", "intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c", null ], + [ "IMB_SNOW3G_F8_N_BUFFER_MULTIKEY", "intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c", null ], + [ "IMB_SNOW3G_F9_1_BUFFER", "intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851", null ], + [ "IMB_SNOW3G_INIT_KEY_SCHED", "intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d", null ], + [ "IMB_SNOW3G_KEY_SCHED_SIZE", "intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2", null ], + [ "IMB_SUBMIT_BURST", "intel-ipsec-mb_8h.html#ad95138f7b201a9d7f8e57a155af720f4", null ], + [ "IMB_SUBMIT_BURST_NOCHECK", "intel-ipsec-mb_8h.html#ab043c084d20c19d9952dee899b693085", null ], + [ "IMB_SUBMIT_CIPHER_BURST", "intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e", null ], + [ "IMB_SUBMIT_CIPHER_BURST_NOCHECK", "intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb", null ], + [ "IMB_SUBMIT_HASH_BURST", "intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed", null ], + [ "IMB_SUBMIT_HASH_BURST_NOCHECK", "intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015", null ], + [ "IMB_SUBMIT_JOB", "intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f", null ], + [ "IMB_SUBMIT_JOB_NOCHECK", "intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d", null ], + [ "IMB_VERSION", "intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3", null ], + [ "IMB_VERSION_NUM", "intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4", null ], + [ "IMB_VERSION_STR", "intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc", null ], + [ "IMB_ZUC_EEA3_1_BUFFER", "intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4", null ], + [ "IMB_ZUC_EEA3_4_BUFFER", "intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf", null ], + [ "IMB_ZUC_EEA3_N_BUFFER", "intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991", null ], + [ "IMB_ZUC_EIA3_1_BUFFER", "intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4", null ], + [ "IMB_ZUC_EIA3_N_BUFFER", "intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c", null ], + [ "KASUMI_KEY_SCHEDULE_SIZE", "intel-ipsec-mb_8h.html#a071e04977525b67c8d1accc60937eb67", null ], + [ "aes_cfb_t", "intel-ipsec-mb_8h.html#aeb837fda81cdf56ffcfa4e68b5877a35", null ], + [ "aes_gcm_enc_dec_finalize_t", "intel-ipsec-mb_8h.html#a8c85430db7a8b695a2836dcb96653bc5", null ], + [ "aes_gcm_enc_dec_iv_t", "intel-ipsec-mb_8h.html#ac5bfb31d18fdc55451cd325c4af032c7", null ], + [ "aes_gcm_enc_dec_t", "intel-ipsec-mb_8h.html#a71fb9f25bcab533a98f3d1c5cc2c7f48", null ], + [ "aes_gcm_enc_dec_update_t", "intel-ipsec-mb_8h.html#af701adcd98fd95084d76e0cb0f926fef", null ], + [ "aes_gcm_init_t", "intel-ipsec-mb_8h.html#adeecd4bac067493d8465e8d7b6b7dd6d", null ], + [ "aes_gcm_init_var_iv_t", "intel-ipsec-mb_8h.html#a65f909001359ad986640155f8756a11d", null ], + [ "aes_gcm_pre_t", "intel-ipsec-mb_8h.html#a11e10a3ac7a08801dc68949fb0f428af", null ], + [ "aes_gcm_precomp_t", "intel-ipsec-mb_8h.html#ab50906d68121cb8e6edf2a3493a60c5c", null ], + [ "aes_gmac_finalize_t", "intel-ipsec-mb_8h.html#af3f75897a0ec8b3f22c4e2b01f7f48b7", null ], + [ "aes_gmac_init_t", "intel-ipsec-mb_8h.html#a31987251c968435d720e8656ae4621a0", null ], + [ "aes_gmac_update_t", "intel-ipsec-mb_8h.html#aba0ac59064802c6d0a182a5164568c18", null ], + [ "chacha_poly_enc_dec_update_t", "intel-ipsec-mb_8h.html#a437de868b8dba4f9b878d99a82f80548", null ], + [ "chacha_poly_finalize_t", "intel-ipsec-mb_8h.html#a1a96f063bf1bd07de158985b57cff4a0", null ], + [ "chacha_poly_init_t", "intel-ipsec-mb_8h.html#a4b0b9ffa331f8d675d24911639f0374f", null ], + [ "cmac_subkey_gen_t", "intel-ipsec-mb_8h.html#aab07996d622d682adc1db8286e633cb3", null ], + [ "crc32_fn_t", "intel-ipsec-mb_8h.html#a56dd197c301ad27dc9ec0862721cb73d", null ], + [ "des_keysched_t", "intel-ipsec-mb_8h.html#a60a93569304a978c798bb8c4a4f7f52f", null ], + [ "flush_job_t", "intel-ipsec-mb_8h.html#ae64d0f677e42b8f8d50289113cf5e01e", null ], + [ "get_completed_job_t", "intel-ipsec-mb_8h.html#ac8cb9a545cdef14f60f36fe8888e6e7c", null ], + [ "get_next_job_t", "intel-ipsec-mb_8h.html#a5a533d852e3b65e9e5978fec37da43b5", null ], + [ "ghash_t", "intel-ipsec-mb_8h.html#a12e0c05f0ecec8d75d4b288a7ec11c84", null ], + [ "hash_fn_t", "intel-ipsec-mb_8h.html#afacacc8b74c05c80a0ee255c68637490", null ], + [ "hash_one_block_t", "intel-ipsec-mb_8h.html#aade58eb541a45650ed7d875dc2be97fa", null ], + [ "hec_32_t", "intel-ipsec-mb_8h.html#aa34b11bf4be97c764b283aedc0589575", null ], + [ "hec_64_t", "intel-ipsec-mb_8h.html#aa9afd4ee91f0d82729afcf5f1cdcb808", null ], + [ "IMB_JOB", "intel-ipsec-mb_8h.html#aed2387ba36d72c3d148fcb9d2802ccf5", null ], + [ "IMB_MGR", "intel-ipsec-mb_8h.html#a6d466496b4adfea3f9bc0881f11fe551", null ], + [ "init_mb_mgr_t", "intel-ipsec-mb_8h.html#af0c38ca0a5aac81d8db7e0cc2811426b", null ], + [ "kasumi_f8_1_buffer_bit_t", "intel-ipsec-mb_8h.html#a81afa744fb5669fe37cf0dce2193d8a7", null ], + [ "kasumi_f8_1_buffer_t", "intel-ipsec-mb_8h.html#a89fe975f0388b71765be494d04b034f4", null ], + [ "kasumi_f8_2_buffer_t", "intel-ipsec-mb_8h.html#af1eb074f5da469fa655d1404be8701dd", null ], + [ "kasumi_f8_3_buffer_t", "intel-ipsec-mb_8h.html#a53ecac7e009fcbeee603f975a45d3012", null ], + [ "kasumi_f8_4_buffer_t", "intel-ipsec-mb_8h.html#accf08a616e22a2c56e6a37d56f38b354", null ], + [ "kasumi_f8_n_buffer_t", "intel-ipsec-mb_8h.html#a51bbde186cdd437b6f9448e19a58e8a1", null ], + [ "kasumi_f9_1_buffer_t", "intel-ipsec-mb_8h.html#a8b02114f003b5fe0e1ccbfdc13ac55e4", null ], + [ "kasumi_f9_1_buffer_user_t", "intel-ipsec-mb_8h.html#ac17147d04602964bea59322ff31ebf59", null ], + [ "kasumi_init_f8_key_sched_t", "intel-ipsec-mb_8h.html#a7bec62666456d8962676961cf187ac80", null ], + [ "kasumi_init_f9_key_sched_t", "intel-ipsec-mb_8h.html#a8b30db8462aca1c039c3fd20646f342c", null ], + [ "kasumi_key_sched_size_t", "intel-ipsec-mb_8h.html#ad7444c36024a63025e7682c16946a00a", null ], + [ "kasumi_key_sched_t", "intel-ipsec-mb_8h.html#a82d6866ebcf1bd46b684fe473b4b73f2", null ], + [ "keyexp_t", "intel-ipsec-mb_8h.html#a60de89a391c8d6d9c51c742a87effb6f", null ], + [ "queue_size_t", "intel-ipsec-mb_8h.html#a9884ae92b598dd504f7d0ef8be5051d6", null ], + [ "snow3g_f8_1_buffer_bit_t", "intel-ipsec-mb_8h.html#a7bd00b31d12260305accf0d37b60c958", null ], + [ "snow3g_f8_1_buffer_t", "intel-ipsec-mb_8h.html#a281315fb8b441d837ebed541b3a23380", null ], + [ "snow3g_f8_2_buffer_t", "intel-ipsec-mb_8h.html#a4e8912e7e55a2ac891f85c19e7f29328", null ], + [ "snow3g_f8_4_buffer_t", "intel-ipsec-mb_8h.html#ae13b1154dcc13ca562e81471659d6099", null ], + [ "snow3g_f8_8_buffer_multikey_t", "intel-ipsec-mb_8h.html#a109b2a72d05bbe05a373ed5cf3c1f201", null ], + [ "snow3g_f8_8_buffer_t", "intel-ipsec-mb_8h.html#a6155f472c6794e88289235752d83154a", null ], + [ "snow3g_f8_n_buffer_multikey_t", "intel-ipsec-mb_8h.html#a4049b601f9da6542e25d903efe524bc1", null ], + [ "snow3g_f8_n_buffer_t", "intel-ipsec-mb_8h.html#a6a4a1da950207c9ed04f3f18ae1b7e43", null ], + [ "snow3g_f9_1_buffer_t", "intel-ipsec-mb_8h.html#ae7a48386f7190b4e46cd113eee413733", null ], + [ "snow3g_init_key_sched_t", "intel-ipsec-mb_8h.html#adfa182361f1c179d75129b4e46841af5", null ], + [ "snow3g_key_sched_size_t", "intel-ipsec-mb_8h.html#ad78a4f1c0d6de808f7046c0fbcd98d8f", null ], + [ "snow3g_key_schedule_t", "intel-ipsec-mb_8h.html#ae50b48832b41b4b468cf933b44849213", null ], + [ "submit_burst_t", "intel-ipsec-mb_8h.html#a0cf6acb86385bc984ec5f916fe6e1066", null ], + [ "submit_cipher_burst_t", "intel-ipsec-mb_8h.html#ac4bd89ab3cb1341eb4f30c4c501a9009", null ], + [ "submit_hash_burst_t", "intel-ipsec-mb_8h.html#a4f6fca648737cec9ee869786e9c240b5", null ], + [ "submit_job_t", "intel-ipsec-mb_8h.html#a41297d789b434157e746e338f0af96ec", null ], + [ "xcbc_keyexp_t", "intel-ipsec-mb_8h.html#abb36f71b9fb4d928c2ad9fa462b80ae6", null ], + [ "zuc_eea3_1_buffer_t", "intel-ipsec-mb_8h.html#a0a3032f0698c42bd486d053e90c44fdd", null ], + [ "zuc_eea3_4_buffer_t", "intel-ipsec-mb_8h.html#a7df2e3f063333d677ef7668626126c4d", null ], + [ "zuc_eea3_n_buffer_t", "intel-ipsec-mb_8h.html#a303f2dd11b1193938596b4ec69edf8cc", null ], + [ "zuc_eia3_1_buffer_t", "intel-ipsec-mb_8h.html#a95ed5580686f708ba2313179cd6064a8", null ], + [ "zuc_eia3_n_buffer_t", "intel-ipsec-mb_8h.html#ada45082bb04dd727c0deec5c4fc96fed", null ], + [ "IMB_ARCH", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6", [ + [ "IMB_ARCH_NONE", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733", null ], + [ "IMB_ARCH_NOAESNI", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44", null ], + [ "IMB_ARCH_SSE", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16", null ], + [ "IMB_ARCH_AVX", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d", null ], + [ "IMB_ARCH_AVX2", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6", null ], + [ "IMB_ARCH_AVX512", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8", null ], + [ "IMB_ARCH_NUM", "intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda", null ] + ] ], + [ "IMB_CHAIN_ORDER", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5", [ + [ "IMB_ORDER_CIPHER_HASH", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52", null ], + [ "IMB_ORDER_HASH_CIPHER", "intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d", null ] + ] ], + [ "IMB_CIPHER_DIRECTION", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9ee", [ + [ "IMB_DIR_ENCRYPT", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce", null ], + [ "IMB_DIR_DECRYPT", "intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59", null ] + ] ], + [ "IMB_CIPHER_MODE", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334c", [ + [ "IMB_CIPHER_CBC", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f", null ], + [ "IMB_CIPHER_CNTR", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea", null ], + [ "IMB_CIPHER_NULL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6", null ], + [ "IMB_CIPHER_DOCSIS_SEC_BPI", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7", null ], + [ "IMB_CIPHER_GCM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa", null ], + [ "IMB_CIPHER_CUSTOM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8", null ], + [ "IMB_CIPHER_DES", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851", null ], + [ "IMB_CIPHER_DOCSIS_DES", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b", null ], + [ "IMB_CIPHER_CCM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f", null ], + [ "IMB_CIPHER_DES3", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f", null ], + [ "IMB_CIPHER_PON_AES_CNTR", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea", null ], + [ "IMB_CIPHER_ECB", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56", null ], + [ "IMB_CIPHER_CNTR_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a", null ], + [ "IMB_CIPHER_ZUC_EEA3", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc", null ], + [ "IMB_CIPHER_SNOW3G_UEA2_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e", null ], + [ "IMB_CIPHER_KASUMI_UEA1_BITLEN", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b", null ], + [ "IMB_CIPHER_CBCS_1_9", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8", null ], + [ "IMB_CIPHER_CHACHA20", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741", null ], + [ "IMB_CIPHER_CHACHA20_POLY1305", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a", null ], + [ "IMB_CIPHER_CHACHA20_POLY1305_SGL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb", null ], + [ "IMB_CIPHER_SNOW_V", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465", null ], + [ "IMB_CIPHER_SNOW_V_AEAD", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409", null ], + [ "IMB_CIPHER_GCM_SGL", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918", null ], + [ "IMB_CIPHER_NUM", "intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75", null ] + ] ], + [ "IMB_ERR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0b", [ + [ "IMB_ERR_MIN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659", null ], + [ "IMB_ERR_NULL_MBMGR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd", null ], + [ "IMB_ERR_JOB_NULL_SRC", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d", null ], + [ "IMB_ERR_JOB_NULL_DST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927", null ], + [ "IMB_ERR_JOB_NULL_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0", null ], + [ "IMB_ERR_JOB_NULL_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2", null ], + [ "IMB_ERR_JOB_NULL_AUTH", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb", null ], + [ "IMB_ERR_JOB_NULL_AAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c", null ], + [ "IMB_ERR_JOB_CIPH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920", null ], + [ "IMB_ERR_JOB_AUTH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608", null ], + [ "IMB_ERR_JOB_IV_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4", null ], + [ "IMB_ERR_JOB_KEY_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285", null ], + [ "IMB_ERR_JOB_AUTH_TAG_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2", null ], + [ "IMB_ERR_JOB_AAD_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6", null ], + [ "IMB_ERR_JOB_SRC_OFFSET", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22", null ], + [ "IMB_ERR_JOB_CHAIN_ORDER", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950", null ], + [ "IMB_ERR_CIPH_MODE", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd", null ], + [ "IMB_ERR_HASH_ALGO", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814", null ], + [ "IMB_ERR_JOB_NULL_AUTH_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9", null ], + [ "IMB_ERR_JOB_NULL_SGL_CTX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0", null ], + [ "IMB_ERR_JOB_NULL_NEXT_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039", null ], + [ "IMB_ERR_JOB_PON_PLI", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303", null ], + [ "IMB_ERR_NULL_SRC", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302", null ], + [ "IMB_ERR_NULL_DST", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02", null ], + [ "IMB_ERR_NULL_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef", null ], + [ "IMB_ERR_NULL_EXP_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382", null ], + [ "IMB_ERR_NULL_IV", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419", null ], + [ "IMB_ERR_NULL_AUTH", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1", null ], + [ "IMB_ERR_NULL_AAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543", null ], + [ "IMB_ERR_CIPH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94", null ], + [ "IMB_ERR_AUTH_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d", null ], + [ "IMB_ERR_IV_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2", null ], + [ "IMB_ERR_KEY_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768", null ], + [ "IMB_ERR_AUTH_TAG_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e", null ], + [ "IMB_ERR_AAD_LEN", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa", null ], + [ "IMB_ERR_SRC_OFFSET", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09", null ], + [ "IMB_ERR_NULL_AUTH_KEY", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c", null ], + [ "IMB_ERR_NULL_CTX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d", null ], + [ "IMB_ERR_NO_AESNI_EMU", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc", null ], + [ "IMB_ERR_JOB_NULL_HMAC_OPAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a", null ], + [ "IMB_ERR_JOB_NULL_HMAC_IPAD", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K1_EXP", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K2", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428", null ], + [ "IMB_ERR_JOB_NULL_XCBC_K3", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3", null ], + [ "IMB_ERR_JOB_CIPH_DIR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db", null ], + [ "IMB_ERR_JOB_NULL_GHASH_INIT_TAG", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9", null ], + [ "IMB_ERR_MISSING_CPUFLAGS_INIT_MGR", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60", null ], + [ "IMB_ERR_NULL_JOB", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38", null ], + [ "IMB_ERR_MAX", "intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6", null ] + ] ], + [ "IMB_HASH_ALG", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006", [ + [ "IMB_AUTH_HMAC_SHA_1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58", null ], + [ "IMB_AUTH_HMAC_SHA_224", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca", null ], + [ "IMB_AUTH_HMAC_SHA_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995", null ], + [ "IMB_AUTH_HMAC_SHA_384", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9", null ], + [ "IMB_AUTH_HMAC_SHA_512", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1", null ], + [ "IMB_AUTH_AES_XCBC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454", null ], + [ "IMB_AUTH_MD5", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe", null ], + [ "IMB_AUTH_NULL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0", null ], + [ "IMB_AUTH_AES_GMAC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d", null ], + [ "IMB_AUTH_CUSTOM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b", null ], + [ "IMB_AUTH_AES_CCM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e", null ], + [ "IMB_AUTH_AES_CMAC", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48", null ], + [ "IMB_AUTH_SHA_1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77", null ], + [ "IMB_AUTH_SHA_224", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325", null ], + [ "IMB_AUTH_SHA_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88", null ], + [ "IMB_AUTH_SHA_384", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc", null ], + [ "IMB_AUTH_SHA_512", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d", null ], + [ "IMB_AUTH_AES_CMAC_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83", null ], + [ "IMB_AUTH_PON_CRC_BIP", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048", null ], + [ "IMB_AUTH_ZUC_EIA3_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b", null ], + [ "IMB_AUTH_DOCSIS_CRC32", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc", null ], + [ "IMB_AUTH_SNOW3G_UIA2_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b", null ], + [ "IMB_AUTH_KASUMI_UIA1", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a", null ], + [ "IMB_AUTH_AES_GMAC_128", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0", null ], + [ "IMB_AUTH_AES_GMAC_192", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66", null ], + [ "IMB_AUTH_AES_GMAC_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af", null ], + [ "IMB_AUTH_AES_CMAC_256", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205", null ], + [ "IMB_AUTH_POLY1305", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56", null ], + [ "IMB_AUTH_CHACHA20_POLY1305", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01", null ], + [ "IMB_AUTH_CHACHA20_POLY1305_SGL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712", null ], + [ "IMB_AUTH_ZUC256_EIA3_BITLEN", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0", null ], + [ "IMB_AUTH_SNOW_V_AEAD", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6", null ], + [ "IMB_AUTH_GCM_SGL", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2", null ], + [ "IMB_AUTH_CRC32_ETHERNET_FCS", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2", null ], + [ "IMB_AUTH_CRC32_SCTP", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44", null ], + [ "IMB_AUTH_CRC32_WIMAX_OFDMA_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab", null ], + [ "IMB_AUTH_CRC24_LTE_A", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31", null ], + [ "IMB_AUTH_CRC24_LTE_B", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395", null ], + [ "IMB_AUTH_CRC16_X25", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3", null ], + [ "IMB_AUTH_CRC16_FP_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc", null ], + [ "IMB_AUTH_CRC11_FP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc", null ], + [ "IMB_AUTH_CRC10_IUUP_DATA", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9", null ], + [ "IMB_AUTH_CRC8_WIMAX_OFDMA_HCS", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42", null ], + [ "IMB_AUTH_CRC7_FP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0", null ], + [ "IMB_AUTH_CRC6_IUUP_HEADER", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f", null ], + [ "IMB_AUTH_GHASH", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3", null ], + [ "IMB_AUTH_NUM", "intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af", null ] + ] ], + [ "IMB_KEY_SIZE_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cc", [ + [ "IMB_KEY_128_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d", null ], + [ "IMB_KEY_192_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598", null ], + [ "IMB_KEY_256_BYTES", "intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a", null ] + ] ], + [ "IMB_SGL_STATE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3d", [ + [ "IMB_SGL_INIT", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e", null ], + [ "IMB_SGL_UPDATE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977", null ], + [ "IMB_SGL_COMPLETE", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975", null ], + [ "IMB_SGL_ALL", "intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408", null ] + ] ], + [ "IMB_STATUS", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066ab", [ + [ "IMB_STATUS_BEING_PROCESSED", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141", null ], + [ "IMB_STATUS_COMPLETED_CIPHER", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089", null ], + [ "IMB_STATUS_COMPLETED_AUTH", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792", null ], + [ "IMB_STATUS_COMPLETED", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341", null ], + [ "IMB_STATUS_INVALID_ARGS", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7", null ], + [ "IMB_STATUS_INTERNAL_ERROR", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb", null ], + [ "IMB_STATUS_ERROR", "intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d", null ] + ] ], + [ "aes_cfb_128_one_avx", "intel-ipsec-mb_8h.html#a5a330bd4ed945f830c88dd6e1066652d", null ], + [ "aes_cfb_128_one_avx2", "intel-ipsec-mb_8h.html#a7046666312bdba42b05bfbf63d0e1c6a", null ], + [ "aes_cfb_128_one_avx512", "intel-ipsec-mb_8h.html#abb7c6395420af2addd92a3fa46e0fd28", null ], + [ "aes_cfb_128_one_sse", "intel-ipsec-mb_8h.html#a7bc28362ae2fb8a4d624b8a7a00adebd", null ], + [ "aes_cmac_subkey_gen_avx", "intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83", null ], + [ "aes_cmac_subkey_gen_avx2", "intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b", null ], + [ "aes_cmac_subkey_gen_avx512", "intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc", null ], + [ "aes_cmac_subkey_gen_sse", "intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d", null ], + [ "aes_gcm_dec_128_avx_gen2", "intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b", null ], + [ "aes_gcm_dec_128_avx_gen4", "intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d", null ], + [ "aes_gcm_dec_128_finalize_avx_gen2", "intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39", null ], + [ "aes_gcm_dec_128_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0", null ], + [ "aes_gcm_dec_128_finalize_sse", "intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5", null ], + [ "aes_gcm_dec_128_sse", "intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d", null ], + [ "aes_gcm_dec_128_update_avx_gen2", "intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628", null ], + [ "aes_gcm_dec_128_update_avx_gen4", "intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6", null ], + [ "aes_gcm_dec_128_update_sse", "intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f", null ], + [ "aes_gcm_dec_192_avx_gen2", "intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac", null ], + [ "aes_gcm_dec_192_avx_gen4", "intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee", null ], + [ "aes_gcm_dec_192_finalize_avx_gen2", "intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798", null ], + [ "aes_gcm_dec_192_finalize_avx_gen4", "intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001", null ], + [ "aes_gcm_dec_192_finalize_sse", "intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce", null ], + [ "aes_gcm_dec_192_sse", "intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1", null ], + [ "aes_gcm_dec_192_update_avx_gen2", "intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc", null ], + [ "aes_gcm_dec_192_update_avx_gen4", "intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363", null ], + [ "aes_gcm_dec_192_update_sse", "intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63", null ], + [ "aes_gcm_dec_256_avx_gen2", "intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c", null ], + [ "aes_gcm_dec_256_avx_gen4", "intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da", null ], + [ "aes_gcm_dec_256_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e", null ], + [ "aes_gcm_dec_256_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b", null ], + [ "aes_gcm_dec_256_finalize_sse", "intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0", null ], + [ "aes_gcm_dec_256_sse", "intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43", null ], + [ "aes_gcm_dec_256_update_avx_gen2", "intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a", null ], + [ "aes_gcm_dec_256_update_avx_gen4", "intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296", null ], + [ "aes_gcm_dec_256_update_sse", "intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68", null ], + [ "aes_gcm_enc_128_avx_gen2", "intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164", null ], + [ "aes_gcm_enc_128_avx_gen4", "intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601", null ], + [ "aes_gcm_enc_128_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d", null ], + [ "aes_gcm_enc_128_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44", null ], + [ "aes_gcm_enc_128_finalize_sse", "intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886", null ], + [ "aes_gcm_enc_128_sse", "intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50", null ], + [ "aes_gcm_enc_128_update_avx_gen2", "intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5", null ], + [ "aes_gcm_enc_128_update_avx_gen4", "intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b", null ], + [ "aes_gcm_enc_128_update_sse", "intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f", null ], + [ "aes_gcm_enc_192_avx_gen2", "intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc", null ], + [ "aes_gcm_enc_192_avx_gen4", "intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7", null ], + [ "aes_gcm_enc_192_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06", null ], + [ "aes_gcm_enc_192_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339", null ], + [ "aes_gcm_enc_192_finalize_sse", "intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c", null ], + [ "aes_gcm_enc_192_sse", "intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076", null ], + [ "aes_gcm_enc_192_update_avx_gen2", "intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb", null ], + [ "aes_gcm_enc_192_update_avx_gen4", "intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81", null ], + [ "aes_gcm_enc_192_update_sse", "intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7", null ], + [ "aes_gcm_enc_256_avx_gen2", "intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672", null ], + [ "aes_gcm_enc_256_avx_gen4", "intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8", null ], + [ "aes_gcm_enc_256_finalize_avx_gen2", "intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f", null ], + [ "aes_gcm_enc_256_finalize_avx_gen4", "intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c", null ], + [ "aes_gcm_enc_256_finalize_sse", "intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8", null ], + [ "aes_gcm_enc_256_sse", "intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c", null ], + [ "aes_gcm_enc_256_update_avx_gen2", "intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f", null ], + [ "aes_gcm_enc_256_update_avx_gen4", "intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547", null ], + [ "aes_gcm_enc_256_update_sse", "intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce", null ], + [ "aes_gcm_init_128_avx_gen2", "intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76", null ], + [ "aes_gcm_init_128_avx_gen4", "intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6", null ], + [ "aes_gcm_init_128_sse", "intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c", null ], + [ "aes_gcm_init_192_avx_gen2", "intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb", null ], + [ "aes_gcm_init_192_avx_gen4", "intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4", null ], + [ "aes_gcm_init_192_sse", "intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b", null ], + [ "aes_gcm_init_256_avx_gen2", "intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701", null ], + [ "aes_gcm_init_256_avx_gen4", "intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1", null ], + [ "aes_gcm_init_256_sse", "intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033", null ], + [ "aes_gcm_pre_128_avx_gen2", "intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339", null ], + [ "aes_gcm_pre_128_avx_gen4", "intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1", null ], + [ "aes_gcm_pre_128_sse", "intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd", null ], + [ "aes_gcm_pre_192_avx_gen2", "intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f", null ], + [ "aes_gcm_pre_192_avx_gen4", "intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c", null ], + [ "aes_gcm_pre_192_sse", "intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a", null ], + [ "aes_gcm_pre_256_avx_gen2", "intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4", null ], + [ "aes_gcm_pre_256_avx_gen4", "intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c", null ], + [ "aes_gcm_pre_256_sse", "intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f", null ], + [ "aes_gcm_precomp_128_avx_gen2", "intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942", null ], + [ "aes_gcm_precomp_128_avx_gen4", "intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36", null ], + [ "aes_gcm_precomp_128_sse", "intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f", null ], + [ "aes_gcm_precomp_192_avx_gen2", "intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7", null ], + [ "aes_gcm_precomp_192_avx_gen4", "intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f", null ], + [ "aes_gcm_precomp_192_sse", "intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6", null ], + [ "aes_gcm_precomp_256_avx_gen2", "intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08", null ], + [ "aes_gcm_precomp_256_avx_gen4", "intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564", null ], + [ "aes_gcm_precomp_256_sse", "intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767", null ], + [ "aes_keyexp_128_avx", "intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04", null ], + [ "aes_keyexp_128_avx2", "intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930", null ], + [ "aes_keyexp_128_avx512", "intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d", null ], + [ "aes_keyexp_128_enc_avx", "intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63", null ], + [ "aes_keyexp_128_enc_avx2", "intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312", null ], + [ "aes_keyexp_128_enc_avx512", "intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c", null ], + [ "aes_keyexp_128_enc_sse", "intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817", null ], + [ "aes_keyexp_128_sse", "intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b", null ], + [ "aes_keyexp_192_avx", "intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086", null ], + [ "aes_keyexp_192_avx2", "intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2", null ], + [ "aes_keyexp_192_avx512", "intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831", null ], + [ "aes_keyexp_192_enc_avx", "intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51", null ], + [ "aes_keyexp_192_enc_avx2", "intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1", null ], + [ "aes_keyexp_192_enc_avx512", "intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127", null ], + [ "aes_keyexp_192_enc_sse", "intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2", null ], + [ "aes_keyexp_192_sse", "intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562", null ], + [ "aes_keyexp_256_avx", "intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91", null ], + [ "aes_keyexp_256_avx2", "intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37", null ], + [ "aes_keyexp_256_avx512", "intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45", null ], + [ "aes_keyexp_256_enc_avx", "intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9", null ], + [ "aes_keyexp_256_enc_avx2", "intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54", null ], + [ "aes_keyexp_256_enc_avx512", "intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d", null ], + [ "aes_keyexp_256_enc_sse", "intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356", null ], + [ "aes_keyexp_256_sse", "intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb", null ], + [ "aes_xcbc_expand_key_avx", "intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121", null ], + [ "aes_xcbc_expand_key_avx2", "intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1", null ], + [ "aes_xcbc_expand_key_avx512", "intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564", null ], + [ "aes_xcbc_expand_key_sse", "intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40", null ], + [ "alloc_mb_mgr", "intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e", null ], + [ "des_key_schedule", "intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440", null ], + [ "flush_job_avx", "intel-ipsec-mb_8h.html#abc6b0f1c857f26b834f22b6bd8f51846", null ], + [ "flush_job_avx2", "intel-ipsec-mb_8h.html#ae242d142065172898129f30e6bc40d8c", null ], + [ "flush_job_avx512", "intel-ipsec-mb_8h.html#ac7aec54f653aeee2554ba980d62fd491", null ], + [ "flush_job_sse", "intel-ipsec-mb_8h.html#a96b880b8a32b0466f880d4f22d95cc3b", null ], + [ "free_mb_mgr", "intel-ipsec-mb_8h.html#a238b96cc36c0dc64798a1244b94add8b", null ], + [ "get_completed_job_avx", "intel-ipsec-mb_8h.html#a9d115c47670330bd01b960f2188169d9", null ], + [ "get_completed_job_avx2", "intel-ipsec-mb_8h.html#ad85bdbf56038fd72ceda7e50deea586e", null ], + [ "get_completed_job_avx512", "intel-ipsec-mb_8h.html#a693afd528fcc0809cc667b6c38de84dc", null ], + [ "get_completed_job_sse", "intel-ipsec-mb_8h.html#a267a718472e0c1b6585402e81e44e7e9", null ], + [ "get_next_job_avx", "intel-ipsec-mb_8h.html#a3518b02f06a82e9f83cd0435f3cf25c9", null ], + [ "get_next_job_avx2", "intel-ipsec-mb_8h.html#a49a3d54979188b3c68bdd621043125f7", null ], + [ "get_next_job_avx512", "intel-ipsec-mb_8h.html#a40c08ba2c387e24f2096b1213c46c812", null ], + [ "get_next_job_sse", "intel-ipsec-mb_8h.html#a055db1f6f16bc17724bdc64d0f313871", null ], + [ "imb_clear_mem", "intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6", null ], + [ "imb_get_errno", "intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10", null ], + [ "imb_get_feature_flags", "intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf", null ], + [ "imb_get_mb_mgr_size", "intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459", null ], + [ "imb_get_strerror", "intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4", null ], + [ "imb_get_version", "intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d", null ], + [ "imb_get_version_str", "intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2", null ], + [ "imb_set_pointers_mb_mgr", "intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81", null ], + [ "init_mb_mgr_auto", "intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638", null ], + [ "init_mb_mgr_avx", "intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c", null ], + [ "init_mb_mgr_avx2", "intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa", null ], + [ "init_mb_mgr_avx512", "intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f", null ], + [ "init_mb_mgr_sse", "intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5", null ], + [ "kasumi_f8_iv_gen", "intel-ipsec-mb_8h.html#a3876dca52c5c210dd0827b2a472e6d0b", null ], + [ "kasumi_f9_iv_gen", "intel-ipsec-mb_8h.html#ac886170fc4e70b467383fdf09f0724cf", null ], + [ "md5_one_block_avx", "intel-ipsec-mb_8h.html#a7addeb4d0c6ddcdaa031b9e7b4dd35c2", null ], + [ "md5_one_block_avx2", "intel-ipsec-mb_8h.html#a8b912812c65f4d3b592875cdd233704d", null ], + [ "md5_one_block_avx512", "intel-ipsec-mb_8h.html#ac7f4da2fc0601ad50851e7e43f53e0cf", null ], + [ "md5_one_block_sse", "intel-ipsec-mb_8h.html#a9d883a3a629240fe86efedd180294433", null ], + [ "queue_size_avx", "intel-ipsec-mb_8h.html#a800c395d326e718a2b520d0bb91cbb89", null ], + [ "queue_size_avx2", "intel-ipsec-mb_8h.html#a8a540000c08366ad47d95e23ed5b2918", null ], + [ "queue_size_avx512", "intel-ipsec-mb_8h.html#a51c0426a4b57c9e7db2b091692dff472", null ], + [ "queue_size_sse", "intel-ipsec-mb_8h.html#a80f792744558de173ea97d722c58e5c1", null ], + [ "sha1_avx", "intel-ipsec-mb_8h.html#a9478828dde5161c2a1b43cb0da0f9f67", null ], + [ "sha1_avx2", "intel-ipsec-mb_8h.html#ac13d60c1ddbd3c770f9d85d59821e520", null ], + [ "sha1_avx512", "intel-ipsec-mb_8h.html#ad9b42865aff95b5bcde6efa4b4e83fe8", null ], + [ "sha1_one_block_avx", "intel-ipsec-mb_8h.html#a532582e6d066b9c6b1c2189ba1b36f27", null ], + [ "sha1_one_block_avx2", "intel-ipsec-mb_8h.html#af1a63afff082c9d14bbe22f14bd6b5d9", null ], + [ "sha1_one_block_avx512", "intel-ipsec-mb_8h.html#a745c97ce8263ebc67f4d8070f5c9ee50", null ], + [ "sha1_one_block_sse", "intel-ipsec-mb_8h.html#a99f608f81f6b692eaeea1d68cac25a28", null ], + [ "sha1_sse", "intel-ipsec-mb_8h.html#a37491294dc799c5477e3979e89e0cf92", null ], + [ "sha224_avx", "intel-ipsec-mb_8h.html#af0e48b520099db5a76190aa5395c90d3", null ], + [ "sha224_avx2", "intel-ipsec-mb_8h.html#ad5945d85d451ccf9b3ba06c06c10d4e9", null ], + [ "sha224_avx512", "intel-ipsec-mb_8h.html#a51c522a828b36d9c69dd7c357b556fc9", null ], + [ "sha224_one_block_avx", "intel-ipsec-mb_8h.html#a76d64c2094522011f741de4ac8a03125", null ], + [ "sha224_one_block_avx2", "intel-ipsec-mb_8h.html#a58916afb9e72f42a1c8c2658e248703a", null ], + [ "sha224_one_block_avx512", "intel-ipsec-mb_8h.html#a16f7bd134f4a667f14261a89ca541126", null ], + [ "sha224_one_block_sse", "intel-ipsec-mb_8h.html#a6a15b62259d7475edd02b11b775b7a38", null ], + [ "sha224_sse", "intel-ipsec-mb_8h.html#a2ef2c1513632e69da882a895a075aa2c", null ], + [ "sha256_avx", "intel-ipsec-mb_8h.html#a7976461101f011a77d19f4fa480d5be3", null ], + [ "sha256_avx2", "intel-ipsec-mb_8h.html#aec411a6b29d7adff67e6eaa1b13b3975", null ], + [ "sha256_avx512", "intel-ipsec-mb_8h.html#a1020a70f15b89b7146733d8cbc2bcee8", null ], + [ "sha256_one_block_avx", "intel-ipsec-mb_8h.html#a2e481e815eb9c8109460e8a8dac62ce9", null ], + [ "sha256_one_block_avx2", "intel-ipsec-mb_8h.html#a0a3c025cfbe9b6e72e9b9f4805f0ca1f", null ], + [ "sha256_one_block_avx512", "intel-ipsec-mb_8h.html#ac2bfb5dca839523118362b48115d7c18", null ], + [ "sha256_one_block_sse", "intel-ipsec-mb_8h.html#ae5ce647166e6f9f8b1d616584ee5d1d6", null ], + [ "sha256_sse", "intel-ipsec-mb_8h.html#a4d9d41b8311d2649ec1b43d6e6edd398", null ], + [ "sha384_avx", "intel-ipsec-mb_8h.html#a7b656d085ad03ff43a6c3a4751bfbf56", null ], + [ "sha384_avx2", "intel-ipsec-mb_8h.html#a389b88f838e79306acea6c8bd157c712", null ], + [ "sha384_avx512", "intel-ipsec-mb_8h.html#a908b9e33df6e434c4bb0c74fa20ceece", null ], + [ "sha384_one_block_avx", "intel-ipsec-mb_8h.html#aef8af95a47ec01a8c280802a16d0f0c7", null ], + [ "sha384_one_block_avx2", "intel-ipsec-mb_8h.html#ae5cf8bca3e0f1c7c40c110b88168669e", null ], + [ "sha384_one_block_avx512", "intel-ipsec-mb_8h.html#a39fa05d7d02e8911e3d882f379d8661e", null ], + [ "sha384_one_block_sse", "intel-ipsec-mb_8h.html#aa3bf3d1bd2a5d62035c972fe38827549", null ], + [ "sha384_sse", "intel-ipsec-mb_8h.html#ad63e21495f9b798250f5c68371af4f37", null ], + [ "sha512_avx", "intel-ipsec-mb_8h.html#a17c4f20de186e1f04580d04d6551e844", null ], + [ "sha512_avx2", "intel-ipsec-mb_8h.html#afc6b16cbe9cf1f840723aa6be3913131", null ], + [ "sha512_avx512", "intel-ipsec-mb_8h.html#a7fa788fbcd353c13fd1eac251f658f1c", null ], + [ "sha512_one_block_avx", "intel-ipsec-mb_8h.html#a7d95ff83cbadc815cbe8c11b0af74b7e", null ], + [ "sha512_one_block_avx2", "intel-ipsec-mb_8h.html#ada14329d2331d02241efaafc3ba26ffd", null ], + [ "sha512_one_block_avx512", "intel-ipsec-mb_8h.html#a5d2e35fbfc14d979d87b5b9a63bf19c9", null ], + [ "sha512_one_block_sse", "intel-ipsec-mb_8h.html#aa5e8c4861339d101c718449cf544d5c4", null ], + [ "sha512_sse", "intel-ipsec-mb_8h.html#a392a99d0622122fe0e3c17611ef276a7", null ], + [ "snow3g_f8_iv_gen", "intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43", null ], + [ "snow3g_f9_iv_gen", "intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd", null ], + [ "submit_job_avx", "intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18", null ], + [ "submit_job_avx2", "intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf", null ], + [ "submit_job_avx512", "intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68", null ], + [ "submit_job_nocheck_avx", "intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a", null ], + [ "submit_job_nocheck_avx2", "intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e", null ], + [ "submit_job_nocheck_avx512", "intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd", null ], + [ "submit_job_nocheck_sse", "intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e", null ], + [ "submit_job_sse", "intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0", null ], + [ "zuc_eea3_iv_gen", "intel-ipsec-mb_8h.html#a4931dd6b1b91456bfe885e28ada0aaf3", null ], + [ "zuc_eia3_iv_gen", "intel-ipsec-mb_8h.html#ae3cefd0e992f1390110f0ef9a3aa510f", null ] +]; \ No newline at end of file diff --git a/docs/intel-ipsec-mb_8h_source.html b/docs/intel-ipsec-mb_8h_source.html new file mode 100644 index 00000000..cf67490d --- /dev/null +++ b/docs/intel-ipsec-mb_8h_source.html @@ -0,0 +1,2867 @@ + + + + + + + +intel-ipsec-mb: intel-ipsec-mb.h Source File + + + + + + + + + + + + + + +
+
+ + + + + + + +
+
intel-ipsec-mb +
+
Documentation of the Intel(R) IPSec Multi-Buffer library
+
+ + + + + + +
+
+
+ + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+ +
+ +
+
intel-ipsec-mb.h
+
+
+Go to the documentation of this file.
1/*******************************************************************************
+
2 Copyright (c) 2012-2022, Intel Corporation
+
3
+
4 Redistribution and use in source and binary forms, with or without
+
5 modification, are permitted provided that the following conditions are met:
+
6
+
7 * Redistributions of source code must retain the above copyright notice,
+
8 this list of conditions and the following disclaimer.
+
9 * Redistributions in binary form must reproduce the above copyright
+
10 notice, this list of conditions and the following disclaimer in the
+
11 documentation and/or other materials provided with the distribution.
+
12 * Neither the name of Intel Corporation nor the names of its contributors
+
13 may be used to endorse or promote products derived from this software
+
14 without specific prior written permission.
+
15
+
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+
17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+
18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+
20 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+
21 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+
22 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+
23 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+
24 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+
25 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
26*******************************************************************************/
+
27
+
28#ifndef IMB_IPSEC_MB_H
+
29#define IMB_IPSEC_MB_H
+
30
+
31#include <stdlib.h>
+
32#include <stdint.h>
+
33#include <errno.h>
+
34
+
35#ifdef __cplusplus
+
36extern "C" {
+
37#endif
+
38
+
39/* 128-bit data type that is not in sdtint.h */
+
40typedef struct {
+
41 uint64_t low;
+
42 uint64_t high;
+
43} imb_uint128_t;
+
44
+
48#if defined __linux__ || defined __FreeBSD__
+
50#define DECLARE_ALIGNED(decl, alignval) \
+
51 decl __attribute__((aligned(alignval)))
+
52#define __forceinline \
+
53 static inline __attribute__((always_inline))
+
54
+
55#if __GNUC__ >= 4
+
56#define IMB_DLL_EXPORT __attribute__((visibility("default")))
+
57#define IMB_DLL_LOCAL __attribute__((visibility("hidden")))
+
58#else /* GNU C 4.0 and later */
+
59#define IMB_DLL_EXPORT
+
60#define IMB_DLL_LOCAL
+
61#endif
+
63#else
+
64/* Windows */
+
65
+
66#ifdef __MINGW32__
+
67/* MinGW-w64 */
+
68#define DECLARE_ALIGNED(decl, alignval) \
+
69 decl __attribute__((aligned(alignval)))
+
70#undef __forceinline
+
71#define __forceinline \
+
72 static inline __attribute__((always_inline))
+
73
+
74#else
+
75/* MSVS */
+
76#define DECLARE_ALIGNED(decl, alignval) \
+
77 __declspec(align(alignval)) decl
+
78#define __forceinline \
+
79 static __forceinline
+
80
+
81#endif /* __MINGW__ */
+
82
+
86#define IMB_DLL_EXPORT
+
87#define IMB_DLL_LOCAL
+
88
+
89#endif /* defined __linux__ || defined __FreeBSD__ */
+
90
+
94#define IMB_VERSION_STR "1.2.0"
+
95#define IMB_VERSION_NUM 0x10200
+
96
+
100#define IMB_VERSION(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+
101
+
105#ifdef DEBUG
+
106#include <assert.h>
+
107#define IMB_ASSERT(x) assert(x)
+
108#else
+
109#define IMB_ASSERT(x)
+
110#endif
+
111
+
112#ifndef IMB_DIM
+
113#define IMB_DIM(x) (sizeof(x) / sizeof(x[0]))
+
114#endif
+
115
+
119typedef enum {
+
120 IMB_ARCH_NONE = 0,
+
121 IMB_ARCH_NOAESNI,
+
122 IMB_ARCH_SSE,
+
123 IMB_ARCH_AVX,
+
124 IMB_ARCH_AVX2,
+
125 IMB_ARCH_AVX512,
+
126 IMB_ARCH_NUM,
+
127} IMB_ARCH;
+
128
+
132#define IMB_DES_KEY_SCHED_SIZE (16 * 8)
+
133#define IMB_DES_BLOCK_SIZE 8
+
134
+
135#define IMB_AES_BLOCK_SIZE 16
+
136
+
137#define IMB_SHA1_DIGEST_SIZE_IN_BYTES 20
+
138#define IMB_SHA224_DIGEST_SIZE_IN_BYTES 28
+
139#define IMB_SHA256_DIGEST_SIZE_IN_BYTES 32
+
140#define IMB_SHA384_DIGEST_SIZE_IN_BYTES 48
+
141#define IMB_SHA512_DIGEST_SIZE_IN_BYTES 64
+
142
+
143#define IMB_SHA1_BLOCK_SIZE 64
+
144#define IMB_SHA_256_BLOCK_SIZE 64
+
145#define IMB_SHA_384_BLOCK_SIZE 128
+
146#define IMB_SHA_512_BLOCK_SIZE 128
+
147
+
148#define IMB_KASUMI_KEY_SIZE 16
+
149#define IMB_KASUMI_IV_SIZE 8
+
150#define IMB_KASUMI_BLOCK_SIZE 8
+
151#define IMB_KASUMI_DIGEST_SIZE 4
+
152
+
157#define IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE 14
+
158#define IMB_DOCSIS_CRC32_TAG_SIZE 4
+
159
+
164typedef enum {
+
165 IMB_STATUS_BEING_PROCESSED = 0,
+
166 IMB_STATUS_COMPLETED_CIPHER = 1,
+
167 IMB_STATUS_COMPLETED_AUTH = 2,
+
168 IMB_STATUS_COMPLETED = 3,
+
170 IMB_STATUS_INVALID_ARGS = 4,
+
171 IMB_STATUS_INTERNAL_ERROR,
+
172 IMB_STATUS_ERROR
+
173} IMB_STATUS;
+
174
+
178typedef enum {
+
179 IMB_ERR_MIN = 2000,
+
180 IMB_ERR_NULL_MBMGR,
+
181 IMB_ERR_JOB_NULL_SRC,
+
182 IMB_ERR_JOB_NULL_DST,
+
183 IMB_ERR_JOB_NULL_KEY,
+
184 IMB_ERR_JOB_NULL_IV,
+
185 IMB_ERR_JOB_NULL_AUTH,
+
186 IMB_ERR_JOB_NULL_AAD,
+
187 IMB_ERR_JOB_CIPH_LEN,
+
188 IMB_ERR_JOB_AUTH_LEN,
+
189 IMB_ERR_JOB_IV_LEN,
+
190 IMB_ERR_JOB_KEY_LEN,
+
191 IMB_ERR_JOB_AUTH_TAG_LEN,
+
192 IMB_ERR_JOB_AAD_LEN,
+
193 IMB_ERR_JOB_SRC_OFFSET,
+
194 IMB_ERR_JOB_CHAIN_ORDER,
+
195 IMB_ERR_CIPH_MODE,
+
196 IMB_ERR_HASH_ALGO,
+
197 IMB_ERR_JOB_NULL_AUTH_KEY,
+
198 IMB_ERR_JOB_NULL_SGL_CTX,
+
199 IMB_ERR_JOB_NULL_NEXT_IV,
+
200 IMB_ERR_JOB_PON_PLI,
+
201 IMB_ERR_NULL_SRC,
+
202 IMB_ERR_NULL_DST,
+
203 IMB_ERR_NULL_KEY,
+
204 IMB_ERR_NULL_EXP_KEY,
+
205 IMB_ERR_NULL_IV,
+
206 IMB_ERR_NULL_AUTH,
+
207 IMB_ERR_NULL_AAD,
+
208 IMB_ERR_CIPH_LEN,
+
209 IMB_ERR_AUTH_LEN,
+
210 IMB_ERR_IV_LEN,
+
211 IMB_ERR_KEY_LEN,
+
212 IMB_ERR_AUTH_TAG_LEN,
+
213 IMB_ERR_AAD_LEN,
+
214 IMB_ERR_SRC_OFFSET,
+
215 IMB_ERR_NULL_AUTH_KEY,
+
216 IMB_ERR_NULL_CTX,
+
217 IMB_ERR_NO_AESNI_EMU,
+
218 IMB_ERR_JOB_NULL_HMAC_OPAD,
+
219 IMB_ERR_JOB_NULL_HMAC_IPAD,
+
220 IMB_ERR_JOB_NULL_XCBC_K1_EXP,
+
221 IMB_ERR_JOB_NULL_XCBC_K2,
+
222 IMB_ERR_JOB_NULL_XCBC_K3,
+
223 IMB_ERR_JOB_CIPH_DIR,
+
224 IMB_ERR_JOB_NULL_GHASH_INIT_TAG,
+
225 IMB_ERR_MISSING_CPUFLAGS_INIT_MGR,
+
226 IMB_ERR_NULL_JOB,
+
227 /* add new error types above this comment */
+
228 IMB_ERR_MAX /* don't move this one */
+
229} IMB_ERR;
+
230
+
235#ifdef __ELASTERROR
+
236#if __ELASTERROR > 2000
+
237#error "Library error codes conflict with errno.h - please update IMB_ERR_MIN!"
+
238#endif
+
239#endif
+
240
+
247#ifndef NO_COMPAT_IMB_API_053
+
248/* Previous cipher mode enums */
+
249#define CBC IMB_CIPHER_CBC
+
250#define CNTR IMB_CIPHER_CNTR
+
251#define NULL_CIPHER IMB_CIPHER_NULL
+
252#define DOCSIS_SEC_BPI IMB_CIPHER_DOCSIS_SEC_BPI
+
253#define GCM IMB_CIPHER_GCM
+
254#define CUSTOM_CIPHER IMB_CIPHER_CUSTOM
+
255#define DES IMB_CIPHER_DES
+
256#define DOCSIS_DES IMB_CIPHER_DOCSIS_DES
+
257#define CCM IMB_CIPHER_CCM
+
258#define DES3 IMB_CIPHER_DES3
+
259#define PON_AES_CNTR IMB_CIPHER_PON_AES_CNTR
+
260#define ECB IMB_CIPHER_ECB
+
261#define CNTR_BITLEN IMB_CIPHER_CNTR_BITLEN
+
262
+
263/* Previous hash algo enums */
+
264#define SHA1 IMB_AUTH_HMAC_SHA_1
+
265#define SHA_224 IMB_AUTH_HMAC_SHA_224
+
266#define SHA_256 IMB_AUTH_HMAC_SHA_256
+
267#define SHA_384 IMB_AUTH_HMAC_SHA_384
+
268#define SHA_512 IMB_AUTH_HMAC_SHA_512
+
269#define AES_XCBC IMB_AUTH_AES_XCBC
+
270#define MD5 IMB_AUTH_MD5
+
271#define NULL_HASH IMB_AUTH_NULL
+
272#define AES_GMAC IMB_AUTH_AES_GMAC
+
273#define CUSTOM_HASH IMB_AUTH_CUSTOM
+
274#define AES_CCM IMB_AUTH_AES_CCM
+
275#define AES_CMAC IMB_AUTH_AES_CMAC
+
276#define PLAIN_SHA1 IMB_AUTH_SHA_1
+
277#define PLAIN_SHA_224 IMB_AUTH_SHA_224
+
278#define PLAIN_SHA_256 IMB_AUTH_SHA_256
+
279#define PLAIN_SHA_384 IMB_AUTH_SHA_384
+
280#define PLAIN_SHA_512 IMB_AUTH_SHA_512
+
281#define AES_CMAC_BITLEN IMB_AUTH_AES_CMAC_BITLEN
+
282#define PON_CRC_BIP IMB_AUTH_PON_CRC_BIP
+
283
+
284/* Previous cipher direction enums */
+
285#define ENCRYPT IMB_DIR_ENCRYPT
+
286#define DECRYPT IMB_DIR_DECRYPT
+
287
+
288/* Previous chain order enums */
+
289#define HASH_CIPHER IMB_ORDER_HASH_CIPHER
+
290#define CIPHER_HASH IMB_ORDER_CIPHER_HASH
+
291
+
292/* Previous key size enums */
+
293#define AES_128_BYTES IMB_KEY_128_BYTES
+
294#define AES_192_BYTES IMB_KEY_192_BYTES
+
295#define AES_256_BYTES IMB_KEY_256_BYTES
+
296#define IMB_KEY_AES_128_BYTES IMB_KEY_128_BYTES
+
297#define IMB_KEY_AES_192_BYTES IMB_KEY_192_BYTES
+
298#define IMB_KEY_AES_256_BYTES IMB_KEY_256_BYTES
+
299#define AES_KEY_SIZE_BYTES IMB_KEY_SIZE_BYTES
+
300
+
301#define MB_MGR IMB_MGR
+
302#define JOB_AES_HMAC IMB_JOB
+
303#define JOB_STS IMB_STATUS
+
304#define IMB_JOB_STS IMB_STATUS
+
305#define JOB_CIPHER_MODE IMB_CIPHER_MODE
+
306#define JOB_CIPHER_DIRECTION IMB_CIPHER_DIRECTION
+
307#define JOB_HASH_ALG IMB_HASH_ALG
+
308#define JOB_CHAIN_ORDER IMB_CHAIN_ORDER
+
309#define MAX_JOBS IMB_MAX_JOBS
+
310
+
311#define STS_BEING_PROCESSED IMB_STATUS_BEING_PROCESSED
+
312#define STS_COMPLETED_AES IMB_STATUS_COMPLETED_CIPHER
+
313#define STS_COMPLETED_HMAC IMB_STATUS_COMPLETED_AUTH
+
314#define STS_COMPLETED IMB_STATUS_COMPLETED
+
315#define STS_INVALID_ARGS IMB_STATUS_INVALID_ARGS
+
316#define STS_INTERNAL_ERROR IMB_STATUS_INTERNAL_ERROR
+
317#define STS_ERROR IMB_STATUS_ERROR
+
318
+
319#define MAX_TAG_LEN IMB_MAX_TAG_LEN
+
320#define GCM_IV_DATA_LEN IMB_GCM_IV_DATA_LEN
+
321#define GCM_128_KEY_LEN IMB_GCM_128_KEY_LEN
+
322#define GCM_192_KEY_LEN IMB_GCM_192_KEY_LEN
+
323#define GCM_256_KEY_LEN IMB_GCM_256_KEY_LEN
+
324
+
325#define DES_KEY_SCHED_SIZE IMB_DES_KEY_SCHED_SIZE
+
326#define DES_BLOCK_SIZE IMB_DES_BLOCK_SIZE
+
327
+
328#define AES_BLOCK_SIZE IMB_AES_BLOCK_SIZE
+
329
+
330#define SHA1_DIGEST_SIZE_IN_BYTES IMB_SHA1_DIGEST_SIZE_IN_BYTES
+
331#define SHA224_DIGEST_SIZE_IN_BYTES IMB_SHA224_DIGEST_SIZE_IN_BYTES
+
332#define SHA256_DIGEST_SIZE_IN_BYTES IMB_SHA256_DIGEST_SIZE_IN_BYTES
+
333#define SHA384_DIGEST_SIZE_IN_BYTES IMB_SHA384_DIGEST_SIZE_IN_BYTES
+
334#define SHA512_DIGEST_SIZE_IN_BYTES IMB_SHA512_DIGEST_SIZE_IN_BYTES
+
335
+
336#define SHA1_BLOCK_SIZE IMB_SHA1_BLOCK_SIZE
+
337#define SHA_256_BLOCK_SIZE IMB_SHA_256_BLOCK_SIZE
+
338#define SHA_384_BLOCK_SIZE IMB_SHA_384_BLOCK_SIZE
+
339#define SHA_512_BLOCK_SIZE IMB_SHA_512_BLOCK_SIZE
+
340
+
341#define KASUMI_KEY_SIZE IMB_KASUMI_KEY_SIZE
+
342#define KASUMI_IV_SIZE IMB_KASUMI_IV_SIZE
+
343#define KASUMI_BLOCK_SIZE IMB_KASUMI_BLOCK_SIZE
+
344#define KASUMI_DIGEST_SIZE IMB_KASUMI_DIGEST_SIZE
+
345
+
346#define DOCSIS_CRC32_MIN_ETH_PDU_SIZE IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE
+
347#define DOCSIS_CRC32_TAG_SIZE IMB_DOCSIS_CRC32_TAG_SIZE
+
348
+
349/* Previous fields in IMB_JOB/JOB_AES_HMAC */
+
350#define aes_enc_key_expanded enc_keys
+
351#define aes_dec_key_expanded dec_keys
+
352#define aes_key_len_in_bytes key_len_in_bytes
+
353#endif /* !NO_COMPAT_IMB_API_053 */
+
354
+
355typedef enum {
+
356 IMB_CIPHER_CBC = 1,
+
357 IMB_CIPHER_CNTR,
+
358 IMB_CIPHER_NULL,
+
359 IMB_CIPHER_DOCSIS_SEC_BPI,
+
360 IMB_CIPHER_GCM,
+
361 IMB_CIPHER_CUSTOM,
+
362 IMB_CIPHER_DES,
+
363 IMB_CIPHER_DOCSIS_DES,
+
364 IMB_CIPHER_CCM,
+
365 IMB_CIPHER_DES3,
+
366 IMB_CIPHER_PON_AES_CNTR,
+
367 IMB_CIPHER_ECB,
+
368 IMB_CIPHER_CNTR_BITLEN,
+
369 IMB_CIPHER_ZUC_EEA3,
+
370 IMB_CIPHER_SNOW3G_UEA2_BITLEN,
+
371 IMB_CIPHER_KASUMI_UEA1_BITLEN,
+
372 IMB_CIPHER_CBCS_1_9,
+
373 IMB_CIPHER_CHACHA20,
+
374 IMB_CIPHER_CHACHA20_POLY1305,
+
375 IMB_CIPHER_CHACHA20_POLY1305_SGL,
+
376 IMB_CIPHER_SNOW_V,
+
377 IMB_CIPHER_SNOW_V_AEAD,
+
378 IMB_CIPHER_GCM_SGL,
+
379 IMB_CIPHER_NUM
+
380} IMB_CIPHER_MODE;
+
381
+
382typedef enum {
+
383 IMB_DIR_ENCRYPT = 1,
+
384 IMB_DIR_DECRYPT
+
385} IMB_CIPHER_DIRECTION;
+
386
+
387typedef enum {
+
388 IMB_AUTH_HMAC_SHA_1 = 1,
+
389 IMB_AUTH_HMAC_SHA_224,
+
390 IMB_AUTH_HMAC_SHA_256,
+
391 IMB_AUTH_HMAC_SHA_384,
+
392 IMB_AUTH_HMAC_SHA_512,
+
393 IMB_AUTH_AES_XCBC,
+
394 IMB_AUTH_MD5,
+
395 IMB_AUTH_NULL,
+
396 IMB_AUTH_AES_GMAC,
+
397 IMB_AUTH_CUSTOM,
+
398 IMB_AUTH_AES_CCM,
+
399 IMB_AUTH_AES_CMAC,
+
400 IMB_AUTH_SHA_1,
+
401 IMB_AUTH_SHA_224,
+
402 IMB_AUTH_SHA_256,
+
403 IMB_AUTH_SHA_384,
+
404 IMB_AUTH_SHA_512,
+
405 IMB_AUTH_AES_CMAC_BITLEN,
+
406 IMB_AUTH_PON_CRC_BIP,
+
407 IMB_AUTH_ZUC_EIA3_BITLEN,
+
408 IMB_AUTH_DOCSIS_CRC32,
+
409 IMB_AUTH_SNOW3G_UIA2_BITLEN,
+
410 IMB_AUTH_KASUMI_UIA1,
+
411 IMB_AUTH_AES_GMAC_128,
+
412 IMB_AUTH_AES_GMAC_192,
+
413 IMB_AUTH_AES_GMAC_256,
+
414 IMB_AUTH_AES_CMAC_256,
+
415 IMB_AUTH_POLY1305,
+
416 IMB_AUTH_CHACHA20_POLY1305,
+
417 IMB_AUTH_CHACHA20_POLY1305_SGL,
+
418 IMB_AUTH_ZUC256_EIA3_BITLEN,
+
419 IMB_AUTH_SNOW_V_AEAD,
+
420 IMB_AUTH_GCM_SGL,
+
421 IMB_AUTH_CRC32_ETHERNET_FCS,
+
422 IMB_AUTH_CRC32_SCTP,
+
423 IMB_AUTH_CRC32_WIMAX_OFDMA_DATA,
+
424 IMB_AUTH_CRC24_LTE_A,
+
425 IMB_AUTH_CRC24_LTE_B,
+
426 IMB_AUTH_CRC16_X25,
+
427 IMB_AUTH_CRC16_FP_DATA,
+
428 IMB_AUTH_CRC11_FP_HEADER,
+
429 IMB_AUTH_CRC10_IUUP_DATA,
+
430 IMB_AUTH_CRC8_WIMAX_OFDMA_HCS,
+
431 IMB_AUTH_CRC7_FP_HEADER,
+
432 IMB_AUTH_CRC6_IUUP_HEADER,
+
433 IMB_AUTH_GHASH,
+
434 IMB_AUTH_NUM
+
435} IMB_HASH_ALG;
+
436
+
437typedef enum {
+
438 IMB_ORDER_CIPHER_HASH = 1,
+
439 IMB_ORDER_HASH_CIPHER
+
440} IMB_CHAIN_ORDER;
+
441
+
442typedef enum {
+
443 IMB_KEY_128_BYTES = 16,
+
444 IMB_KEY_192_BYTES = 24,
+
445 IMB_KEY_256_BYTES = 32
+
446} IMB_KEY_SIZE_BYTES;
+
447
+
448typedef enum {
+
449 IMB_SGL_INIT = 0,
+
450 IMB_SGL_UPDATE,
+
451 IMB_SGL_COMPLETE,
+
452 IMB_SGL_ALL
+
453} IMB_SGL_STATE;
+
454
+
458struct IMB_SGL_IOV {
+
459 const void *in;
+
460 void *out;
+
461 uint64_t len;
+
462};
+
463
+
483typedef struct IMB_JOB {
+
484 const void *enc_keys;
+
485 const void *dec_keys;
+
486 uint64_t key_len_in_bytes;
+
487 union {
+
488 const uint8_t *src;
+
491 const struct IMB_SGL_IOV *sgl_io_segs;
+
493 };
+
494 union {
+
495 uint8_t *dst;
+
498 uint64_t num_sgl_io_segs;
+
500 };
+
501 union {
+
502 uint64_t cipher_start_src_offset_in_bytes;
+
504 uint64_t cipher_start_src_offset_in_bits;
+
506 uint64_t cipher_start_offset_in_bits;
+
508 };
+
509 union {
+
510 uint64_t msg_len_to_cipher_in_bytes;
+
512 uint64_t msg_len_to_cipher_in_bits;
+
514 };
+
515 uint64_t hash_start_src_offset_in_bytes;
+
516 union {
+
517 uint64_t msg_len_to_hash_in_bytes;
+
519 uint64_t msg_len_to_hash_in_bits;
+
521 };
+
522 const uint8_t *iv;
+
523 uint64_t iv_len_in_bytes;
+
524 uint8_t *auth_tag_output;
+
525 uint64_t auth_tag_output_len_in_bytes;
+
528 /* Start hash algorithm-specific fields */
+
529 union {
+
530 struct _HMAC_specific_fields {
+
531 const uint8_t *_hashed_auth_key_xor_ipad;
+
534 const uint8_t *_hashed_auth_key_xor_opad;
+
537 } HMAC;
+
538 struct _AES_XCBC_specific_fields {
+
539 const uint32_t *_k1_expanded;
+
541 const uint8_t *_k2;
+
543 const uint8_t *_k3;
+
545 } XCBC;
+
546 struct _AES_CCM_specific_fields {
+
547 const void *aad;
+
549 uint64_t aad_len_in_bytes;
+
550 } CCM;
+
551 struct _AES_CMAC_specific_fields {
+
552 const void *_key_expanded;
+
554 const void *_skey1;
+
555 const void *_skey2;
+
556 } CMAC;
+
557 struct _AES_GCM_specific_fields {
+
558 const void *aad;
+
560 uint64_t aad_len_in_bytes;
+
561 struct gcm_context_data *ctx;
+
563 } GCM;
+
564 struct _ZUC_EIA3_specific_fields {
+
565 const uint8_t *_key;
+
567 const uint8_t *_iv;
+
569 const uint8_t *_iv23;
+
571 } ZUC_EIA3;
+
572 struct _SNOW3G_UIA2_specific_fields {
+
573 const void *_key;
+
575 const void *_iv;
+
577 } SNOW3G_UIA2;
+
578 struct _KASUMI_UIA1_specific_fields {
+
579 const void *_key;
+
581 } KASUMI_UIA1;
+
582 struct _AES_GMAC_specific_fields {
+
583 const struct gcm_key_data *_key;
+
585 const void *_iv;
+
587 uint64_t iv_len_in_bytes;
+
589 } GMAC;
+
590 struct _GHASH_specific_fields {
+
591 const struct gcm_key_data *_key;
+
593 const void *_init_tag;
+
594 } GHASH;
+
595 struct _POLY1305_specific_fields {
+
596 const void *_key;
+
598 } POLY1305;
+
599 struct _CHACHA20_POLY1305_specific_fields {
+
600 const void *aad;
+
602 uint64_t aad_len_in_bytes;
+
604 struct chacha20_poly1305_context_data *ctx;
+
606 } CHACHA20_POLY1305;
+
607 struct _SNOW_V_AEAD_specific_fields {
+
608 const void *aad;
+
610 uint64_t aad_len_in_bytes;
+
612 void *reserved;
+
614 } SNOW_V_AEAD;
+
615 } u;
+
617 IMB_STATUS status;
+
618 IMB_CIPHER_MODE cipher_mode;
+
619 IMB_CIPHER_DIRECTION cipher_direction;
+
620 IMB_HASH_ALG hash_alg;
+
621 IMB_CHAIN_ORDER chain_order;
+
624 void *user_data;
+
625 void *user_data2;
+
627 int (*cipher_func)(struct IMB_JOB *);
+
629 int (*hash_func)(struct IMB_JOB *);
+
632 IMB_SGL_STATE sgl_state;
+
636 union {
+
637 struct _CBCS_specific_fields {
+
638 void *next_iv;
+
640 } CBCS;
+
641 } cipher_fields;
+
642} IMB_JOB;
+
643
+
644
+
645/* KASUMI */
+
646
+
647/* 64 precomputed words for key schedule */
+
648#define KASUMI_KEY_SCHEDULE_SIZE 64
+
649
+
653typedef struct kasumi_key_sched_s {
+
655 uint16_t sk16[KASUMI_KEY_SCHEDULE_SIZE];
+
656 uint16_t msk16[KASUMI_KEY_SCHEDULE_SIZE];
+
657} kasumi_key_sched_t;
+
658
+
659/* GCM data structures */
+
660#define IMB_GCM_BLOCK_LEN 16
+
661
+
667struct gcm_context_data {
+
668 uint8_t aad_hash[IMB_GCM_BLOCK_LEN];
+
669 uint64_t aad_length;
+
670 uint64_t in_length;
+
671 uint8_t partial_block_enc_key[IMB_GCM_BLOCK_LEN];
+
672 uint8_t orig_IV[IMB_GCM_BLOCK_LEN];
+
673 uint8_t current_counter[IMB_GCM_BLOCK_LEN];
+
674 uint64_t partial_block_length;
+
675};
+
676#undef IMB_GCM_BLOCK_LEN
+
677
+
681struct chacha20_poly1305_context_data {
+
682 uint64_t hash[3];
+
683 uint64_t aad_len;
+
684 uint64_t hash_len;
+
685 uint8_t last_ks[64];
+
686 uint8_t poly_key[32];
+
687 uint8_t poly_scratch[16];
+
688 uint64_t last_block_count;
+
689 uint64_t remain_ks_bytes;
+
691 uint64_t remain_ct_bytes;
+
694 uint8_t IV[12];
+
695};
+
696
+
701#define IMB_MAX_TAG_LEN (16)
+
702
+
712#define IMB_GCM_IV_DATA_LEN (12)
+
713
+
714#define IMB_GCM_128_KEY_LEN (16)
+
715#define IMB_GCM_192_KEY_LEN (24)
+
716#define IMB_GCM_256_KEY_LEN (32)
+
717
+
718#define IMB_GCM_ENC_KEY_LEN 16
+
719#define IMB_GCM_KEY_SETS (15)
+
726#ifdef __WIN32
+
727__declspec(align(64))
+
728#endif /* WIN32 */
+
729struct gcm_key_data {
+
730 uint8_t expanded_keys[IMB_GCM_ENC_KEY_LEN * IMB_GCM_KEY_SETS];
+
731 union {
+
733 struct {
+
739 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 8];
+
748 uint8_t shifted_hkey_k[IMB_GCM_ENC_KEY_LEN * 8];
+
749 } sse_avx;
+
750 struct {
+
756 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 8];
+
757 } avx2_avx512;
+
758 struct {
+
763 uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN * 48];
+
764 } vaes_avx512;
+
765 } ghash_keys;
+
766}
+
767#ifdef LINUX
+
768__attribute__((aligned(64)));
+
769#else
+
770;
+
771#endif
+
772
+
773#undef IMB_GCM_ENC_KEY_LEN
+
774#undef IMB_GCM_KEY_SETS
+
775
+
776/* API data type definitions */
+
777struct IMB_MGR;
+
778
+
779typedef void (*init_mb_mgr_t)(struct IMB_MGR *);
+
780typedef IMB_JOB *(*get_next_job_t)(struct IMB_MGR *);
+
781typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *);
+
782typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *);
+
783typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *);
+
784typedef uint32_t (*queue_size_t)(struct IMB_MGR *);
+
785typedef uint32_t (*submit_burst_t)(struct IMB_MGR *,
+
786 struct IMB_JOB *, const uint32_t);
+
787typedef uint32_t (*submit_cipher_burst_t)(struct IMB_MGR *,
+
788 struct IMB_JOB *,
+
789 const uint32_t,
+
790 const IMB_CIPHER_MODE cipher,
+
791 const IMB_CIPHER_DIRECTION dir,
+
792 const IMB_KEY_SIZE_BYTES key_size);
+
793typedef uint32_t (*submit_hash_burst_t)(struct IMB_MGR *,
+
794 struct IMB_JOB *,
+
795 const uint32_t,
+
796 const IMB_HASH_ALG hash);
+
797typedef void (*keyexp_t)(const void *, void *, void *);
+
798typedef void (*cmac_subkey_gen_t)(const void *, void *, void *);
+
799typedef void (*hash_one_block_t)(const void *, void *);
+
800typedef void (*hash_fn_t)(const void *, const uint64_t, void *);
+
801typedef void (*xcbc_keyexp_t)(const void *, void *, void *, void *);
+
802typedef int (*des_keysched_t)(uint64_t *, const void *);
+
803typedef void (*aes_cfb_t)(void *, const void *, const void *, const void *,
+
804 uint64_t);
+
805typedef void (*aes_gcm_enc_dec_t)(const struct gcm_key_data *,
+
806 struct gcm_context_data *,
+
807 uint8_t *, uint8_t const *, uint64_t,
+
808 const uint8_t *, uint8_t const *, uint64_t,
+
809 uint8_t *, uint64_t);
+
810typedef void (*aes_gcm_enc_dec_iv_t)(const struct gcm_key_data *,
+
811 struct gcm_context_data *, uint8_t *,
+
812 uint8_t const *, const uint64_t,
+
813 const uint8_t *, uint8_t const *,
+
814 const uint64_t, uint8_t *,
+
815 const uint64_t, const uint64_t);
+
816typedef void (*aes_gcm_init_t)(const struct gcm_key_data *,
+
817 struct gcm_context_data *,
+
818 const uint8_t *, uint8_t const *, uint64_t);
+
819typedef void (*aes_gcm_init_var_iv_t)(const struct gcm_key_data *,
+
820 struct gcm_context_data *,
+
821 const uint8_t *, const uint64_t,
+
822 const uint8_t *, const uint64_t);
+
823typedef void (*aes_gcm_enc_dec_update_t)(const struct gcm_key_data *,
+
824 struct gcm_context_data *,
+
825 uint8_t *, const uint8_t *, uint64_t);
+
826typedef void (*aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *,
+
827 struct gcm_context_data *,
+
828 uint8_t *, uint64_t);
+
829typedef void (*aes_gcm_precomp_t)(struct gcm_key_data *);
+
830typedef void (*aes_gcm_pre_t)(const void *, struct gcm_key_data *);
+
831
+
832typedef void (*aes_gmac_init_t)(const struct gcm_key_data *,
+
833 struct gcm_context_data *,
+
834 const uint8_t *, const uint64_t);
+
835typedef void (*aes_gmac_update_t)(const struct gcm_key_data *,
+
836 struct gcm_context_data *,
+
837 const uint8_t *, const uint64_t);
+
838typedef void (*aes_gmac_finalize_t)(const struct gcm_key_data *,
+
839 struct gcm_context_data *,
+
840 uint8_t *, const uint64_t);
+
841
+
842typedef void (*chacha_poly_init_t)(const void *,
+
843 struct chacha20_poly1305_context_data *,
+
844 const void *, const void *, const uint64_t);
+
845typedef void (*chacha_poly_enc_dec_update_t)(const void *,
+
846 struct chacha20_poly1305_context_data *,
+
847 void *, const void *, const uint64_t);
+
848typedef void (*chacha_poly_finalize_t)(struct chacha20_poly1305_context_data *,
+
849 void *, const uint64_t);
+
850typedef void (*ghash_t)(const struct gcm_key_data *, const void *,
+
851 const uint64_t, void *, const uint64_t);
+
852
+
853typedef void (*zuc_eea3_1_buffer_t)(const void *, const void *, const void *,
+
854 void *, const uint32_t);
+
855
+
856typedef void (*zuc_eea3_4_buffer_t)(const void * const *, const void * const *,
+
857 const void * const *, void **,
+
858 const uint32_t *);
+
859
+
860typedef void (*zuc_eea3_n_buffer_t)(const void * const *, const void * const *,
+
861 const void * const *, void **,
+
862 const uint32_t *, const uint32_t);
+
863
+
864typedef void (*zuc_eia3_1_buffer_t)(const void *, const void *, const void *,
+
865 const uint32_t, uint32_t *);
+
866
+
867typedef void (*zuc_eia3_n_buffer_t)(const void * const *, const void * const *,
+
868 const void * const *,
+
869 const uint32_t *, uint32_t **,
+
870 const uint32_t);
+
871
+
872
+
873typedef void (*kasumi_f8_1_buffer_t)(const kasumi_key_sched_t *,
+
874 const uint64_t, const void *, void *,
+
875 const uint32_t);
+
876typedef void (*kasumi_f8_1_buffer_bit_t)(const kasumi_key_sched_t *,
+
877 const uint64_t, const void *,
+
878 void *,
+
879 const uint32_t, const uint32_t);
+
880typedef void (*kasumi_f8_2_buffer_t)(const kasumi_key_sched_t *,
+
881 const uint64_t, const uint64_t,
+
882 const void *, void *,
+
883 const uint32_t,
+
884 const void *, void *,
+
885 const uint32_t);
+
886typedef void (*kasumi_f8_3_buffer_t)(const kasumi_key_sched_t *,
+
887 const uint64_t, const uint64_t,
+
888 const uint64_t,
+
889 const void *, void *,
+
890 const void *, void *,
+
891 const void *, void *,
+
892 const uint32_t);
+
893typedef void (*kasumi_f8_4_buffer_t)(const kasumi_key_sched_t *,
+
894 const uint64_t, const uint64_t,
+
895 const uint64_t, const uint64_t,
+
896 const void *, void *,
+
897 const void *, void *,
+
898 const void *, void *,
+
899 const void *, void *,
+
900 const uint32_t);
+
901typedef void (*kasumi_f8_n_buffer_t)(const kasumi_key_sched_t *,
+
902 const uint64_t *, const void * const *,
+
903 void **, const uint32_t *,
+
904 const uint32_t);
+
905typedef void (*kasumi_f9_1_buffer_user_t)(const kasumi_key_sched_t *,
+
906 const uint64_t, const void *,
+
907 const uint32_t, void *,
+
908 const uint32_t);
+
909typedef void (*kasumi_f9_1_buffer_t)(const kasumi_key_sched_t *,
+
910 const void *,
+
911 const uint32_t, void *);
+
912typedef int (*kasumi_init_f8_key_sched_t)(const void *,
+
913 kasumi_key_sched_t *);
+
914typedef int (*kasumi_init_f9_key_sched_t)(const void *,
+
915 kasumi_key_sched_t *);
+
916typedef size_t (*kasumi_key_sched_size_t)(void);
+
917
+
918
+
922typedef struct snow3g_key_schedule_s {
+
923 /* KEY */
+
924 uint32_t k[4];
+
925} snow3g_key_schedule_t;
+
926
+
927typedef void (*snow3g_f8_1_buffer_t)(const snow3g_key_schedule_t *,
+
928 const void *, const void *,
+
929 void *, const uint32_t);
+
930
+
931typedef void (*snow3g_f8_1_buffer_bit_t)(const snow3g_key_schedule_t *,
+
932 const void *, const void *, void *,
+
933 const uint32_t, const uint32_t);
+
934
+
935typedef void (*snow3g_f8_2_buffer_t)(const snow3g_key_schedule_t *,
+
936 const void *, const void *,
+
937 const void *, void *, const uint32_t,
+
938 const void *, void *, const uint32_t);
+
939
+
940typedef void (*snow3g_f8_4_buffer_t)(const snow3g_key_schedule_t *,
+
941 const void *, const void *, const void *,
+
942 const void *, const void *, void *,
+
943 const uint32_t, const void *, void *,
+
944 const uint32_t, const void *, void *,
+
945 const uint32_t, const void *, void *,
+
946 const uint32_t);
+
947
+
948typedef void (*snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *,
+
949 const void *, const void *, const void *,
+
950 const void *, const void *, const void *,
+
951 const void *, const void *, const void *,
+
952 void *, const uint32_t, const void *,
+
953 void *, const uint32_t, const void *,
+
954 void *, const uint32_t, const void *,
+
955 void *, const uint32_t, const void *,
+
956 void *, const uint32_t, const void *,
+
957 void *, const uint32_t, const void *,
+
958 void *, const uint32_t, const void *,
+
959 void *, const uint32_t);
+
960
+
961typedef void
+
962(*snow3g_f8_8_buffer_multikey_t)(const snow3g_key_schedule_t * const [],
+
963 const void * const [], const void * const [],
+
964 void *[], const uint32_t[]);
+
965
+
966typedef void (*snow3g_f8_n_buffer_t)(const snow3g_key_schedule_t *,
+
967 const void * const [],
+
968 const void * const [],
+
969 void *[], const uint32_t[],
+
970 const uint32_t);
+
971
+
972typedef void
+
973(*snow3g_f8_n_buffer_multikey_t)(const snow3g_key_schedule_t * const [],
+
974 const void * const [],
+
975 const void * const [],
+
976 void *[], const uint32_t[],
+
977 const uint32_t);
+
978
+
979typedef void (*snow3g_f9_1_buffer_t)(const snow3g_key_schedule_t *,
+
980 const void *, const void *,
+
981 const uint64_t, void *);
+
982
+
983typedef int (*snow3g_init_key_sched_t)(const void *,
+
984 snow3g_key_schedule_t *);
+
985
+
986typedef size_t (*snow3g_key_sched_size_t)(void);
+
987
+
988typedef uint32_t (*hec_32_t)(const uint8_t *);
+
989typedef uint64_t (*hec_64_t)(const uint8_t *);
+
990
+
991typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t);
+
992/* Multi-buffer manager flags passed to alloc_mb_mgr() */
+
993
+
994#define IMB_FLAG_SHANI_OFF (1ULL << 0)
+
995#define IMB_FLAG_AESNI_OFF (1ULL << 1)
+
1004#define IMB_FEATURE_SHANI (1ULL << 0)
+
1005#define IMB_FEATURE_AESNI (1ULL << 1)
+
1006#define IMB_FEATURE_PCLMULQDQ (1ULL << 2)
+
1007#define IMB_FEATURE_CMOV (1ULL << 3)
+
1008#define IMB_FEATURE_SSE4_2 (1ULL << 4)
+
1009#define IMB_FEATURE_AVX (1ULL << 5)
+
1010#define IMB_FEATURE_AVX2 (1ULL << 6)
+
1011#define IMB_FEATURE_AVX512F (1ULL << 7)
+
1012#define IMB_FEATURE_AVX512DQ (1ULL << 8)
+
1013#define IMB_FEATURE_AVX512CD (1ULL << 9)
+
1014#define IMB_FEATURE_AVX512BW (1ULL << 10)
+
1015#define IMB_FEATURE_AVX512VL (1ULL << 11)
+
1016#define IMB_FEATURE_AVX512_SKX (IMB_FEATURE_AVX512F | IMB_FEATURE_AVX512DQ | \
+
1017 IMB_FEATURE_AVX512CD | IMB_FEATURE_AVX512BW | \
+
1018 IMB_FEATURE_AVX512VL)
+
1019#define IMB_FEATURE_VAES (1ULL << 12)
+
1020#define IMB_FEATURE_VPCLMULQDQ (1ULL << 13)
+
1021#define IMB_FEATURE_SAFE_DATA (1ULL << 14)
+
1022#define IMB_FEATURE_SAFE_PARAM (1ULL << 15)
+
1023#define IMB_FEATURE_GFNI (1ULL << 16)
+
1024#define IMB_FEATURE_AVX512_IFMA (1ULL << 17)
+
1025#define IMB_FEATURE_BMI2 (1ULL << 18)
+
1026#define IMB_FEATURE_AESNI_EMU (1ULL << 19)
+
1027
+
1031#define IMB_CPUFLAGS_NO_AESNI (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV)
+
1032#define IMB_CPUFLAGS_SSE (IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \
+
1033 IMB_FEATURE_PCLMULQDQ)
+
1034#define IMB_CPUFLAGS_AVX (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX)
+
1035#define IMB_CPUFLAGS_AVX2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \
+
1036 IMB_FEATURE_BMI2)
+
1037#define IMB_CPUFLAGS_AVX512 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX)
+
1038
+
1039/* TOP LEVEL (IMB_MGR) Data structure fields */
+
1040
+
1041#define IMB_MAX_JOBS 128
+
1042
+
1043typedef struct IMB_MGR {
+
1044
+
1045 uint64_t flags;
+
1046 uint64_t features;
+
1048 uint64_t reserved[5];
+
1049 uint32_t used_arch;
+
1051 int imb_errno;
+
1059 get_next_job_t get_next_job;
+
1060 submit_job_t submit_job;
+
1061 submit_job_t submit_job_nocheck;
+
1062 get_completed_job_t get_completed_job;
+
1063 flush_job_t flush_job;
+
1064 queue_size_t queue_size;
+
1065 keyexp_t keyexp_128;
+
1066 keyexp_t keyexp_192;
+
1067 keyexp_t keyexp_256;
+
1068 cmac_subkey_gen_t cmac_subkey_gen_128;
+
1069 xcbc_keyexp_t xcbc_keyexp;
+
1070 des_keysched_t des_key_sched;
+
1071 hash_one_block_t sha1_one_block;
+
1072 hash_one_block_t sha224_one_block;
+
1073 hash_one_block_t sha256_one_block;
+
1074 hash_one_block_t sha384_one_block;
+
1075 hash_one_block_t sha512_one_block;
+
1076 hash_one_block_t md5_one_block;
+
1077 hash_fn_t sha1;
+
1078 hash_fn_t sha224;
+
1079 hash_fn_t sha256;
+
1080 hash_fn_t sha384;
+
1081 hash_fn_t sha512;
+
1082 aes_cfb_t aes128_cfb_one;
+
1083
+
1084 aes_gcm_enc_dec_t gcm128_enc;
+
1085 aes_gcm_enc_dec_t gcm192_enc;
+
1086 aes_gcm_enc_dec_t gcm256_enc;
+
1087 aes_gcm_enc_dec_t gcm128_dec;
+
1088 aes_gcm_enc_dec_t gcm192_dec;
+
1089 aes_gcm_enc_dec_t gcm256_dec;
+
1090 aes_gcm_init_t gcm128_init;
+
1091 aes_gcm_init_t gcm192_init;
+
1092 aes_gcm_init_t gcm256_init;
+
1093 aes_gcm_enc_dec_update_t gcm128_enc_update;
+
1094 aes_gcm_enc_dec_update_t gcm192_enc_update;
+
1095 aes_gcm_enc_dec_update_t gcm256_enc_update;
+
1096 aes_gcm_enc_dec_update_t gcm128_dec_update;
+
1097 aes_gcm_enc_dec_update_t gcm192_dec_update;
+
1098 aes_gcm_enc_dec_update_t gcm256_dec_update;
+
1099 aes_gcm_enc_dec_finalize_t gcm128_enc_finalize;
+
1100 aes_gcm_enc_dec_finalize_t gcm192_enc_finalize;
+
1101 aes_gcm_enc_dec_finalize_t gcm256_enc_finalize;
+
1102 aes_gcm_enc_dec_finalize_t gcm128_dec_finalize;
+
1103 aes_gcm_enc_dec_finalize_t gcm192_dec_finalize;
+
1104 aes_gcm_enc_dec_finalize_t gcm256_dec_finalize;
+
1105 aes_gcm_precomp_t gcm128_precomp;
+
1106 aes_gcm_precomp_t gcm192_precomp;
+
1107 aes_gcm_precomp_t gcm256_precomp;
+
1108 aes_gcm_pre_t gcm128_pre;
+
1109 aes_gcm_pre_t gcm192_pre;
+
1110 aes_gcm_pre_t gcm256_pre;
+
1111
+
1112 zuc_eea3_1_buffer_t eea3_1_buffer;
+
1113 zuc_eea3_4_buffer_t eea3_4_buffer;
+
1114 zuc_eea3_n_buffer_t eea3_n_buffer;
+
1115 zuc_eia3_1_buffer_t eia3_1_buffer;
+
1116
+
1117 kasumi_f8_1_buffer_t f8_1_buffer;
+
1118 kasumi_f8_1_buffer_bit_t f8_1_buffer_bit;
+
1119 kasumi_f8_2_buffer_t f8_2_buffer;
+
1120 kasumi_f8_3_buffer_t f8_3_buffer;
+
1121 kasumi_f8_4_buffer_t f8_4_buffer;
+
1122 kasumi_f8_n_buffer_t f8_n_buffer;
+
1123 kasumi_f9_1_buffer_t f9_1_buffer;
+
1124 kasumi_f9_1_buffer_user_t f9_1_buffer_user;
+
1125 kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched;
+
1126 kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched;
+
1127 kasumi_key_sched_size_t kasumi_key_sched_size;
+
1128
+
1129 snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit;
+
1130 snow3g_f8_1_buffer_t snow3g_f8_1_buffer;
+
1131 snow3g_f8_2_buffer_t snow3g_f8_2_buffer;
+
1132 snow3g_f8_4_buffer_t snow3g_f8_4_buffer;
+
1133 snow3g_f8_8_buffer_t snow3g_f8_8_buffer;
+
1134 snow3g_f8_n_buffer_t snow3g_f8_n_buffer;
+
1135 snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey;
+
1136 snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey;
+
1137 snow3g_f9_1_buffer_t snow3g_f9_1_buffer;
+
1138 snow3g_init_key_sched_t snow3g_init_key_sched;
+
1139 snow3g_key_sched_size_t snow3g_key_sched_size;
+
1140
+
1141 ghash_t ghash;
+
1142 zuc_eia3_n_buffer_t eia3_n_buffer;
+
1143 aes_gcm_init_var_iv_t gcm128_init_var_iv;
+
1144 aes_gcm_init_var_iv_t gcm192_init_var_iv;
+
1145 aes_gcm_init_var_iv_t gcm256_init_var_iv;
+
1146
+
1147 aes_gmac_init_t gmac128_init;
+
1148 aes_gmac_init_t gmac192_init;
+
1149 aes_gmac_init_t gmac256_init;
+
1150 aes_gmac_update_t gmac128_update;
+
1151 aes_gmac_update_t gmac192_update;
+
1152 aes_gmac_update_t gmac256_update;
+
1153 aes_gmac_finalize_t gmac128_finalize;
+
1154 aes_gmac_finalize_t gmac192_finalize;
+
1155 aes_gmac_finalize_t gmac256_finalize;
+
1156 hec_32_t hec_32;
+
1157 hec_64_t hec_64;
+
1158 cmac_subkey_gen_t cmac_subkey_gen_256;
+
1159 aes_gcm_pre_t ghash_pre;
+
1160 crc32_fn_t crc32_ethernet_fcs;
+
1161 crc32_fn_t crc16_x25;
+
1162 crc32_fn_t crc32_sctp;
+
1163 crc32_fn_t crc24_lte_a;
+
1164 crc32_fn_t crc24_lte_b;
+
1165 crc32_fn_t crc16_fp_data;
+
1166 crc32_fn_t crc11_fp_header;
+
1167 crc32_fn_t crc7_fp_header;
+
1168 crc32_fn_t crc10_iuup_data;
+
1169 crc32_fn_t crc6_iuup_header;
+
1170 crc32_fn_t crc32_wimax_ofdma_data;
+
1171 crc32_fn_t crc8_wimax_ofdma_hcs;
+
1172
+
1173 chacha_poly_init_t chacha20_poly1305_init;
+
1174 chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update;
+
1175 chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update;
+
1176 chacha_poly_finalize_t chacha20_poly1305_finalize;
+
1177
+
1178 submit_burst_t submit_burst;
+
1179 submit_burst_t submit_burst_nocheck;
+
1180 submit_cipher_burst_t submit_cipher_burst;
+
1181 submit_cipher_burst_t submit_cipher_burst_nocheck;
+
1182 submit_hash_burst_t submit_hash_burst;
+
1183 submit_hash_burst_t submit_hash_burst_nocheck;
+
1184
+
1185 /* in-order scheduler fields */
+
1186 int earliest_job;
+
1187 int next_job;
+
1188 IMB_JOB jobs[IMB_MAX_JOBS];
+
1189
+
1190 /* out of order managers */
+
1191 void *aes128_ooo;
+
1192 void *aes192_ooo;
+
1193 void *aes256_ooo;
+
1194 void *docsis128_sec_ooo;
+
1195 void *docsis128_crc32_sec_ooo;
+
1196 void *docsis256_sec_ooo;
+
1197 void *docsis256_crc32_sec_ooo;
+
1198 void *des_enc_ooo;
+
1199 void *des_dec_ooo;
+
1200 void *des3_enc_ooo;
+
1201 void *des3_dec_ooo;
+
1202 void *docsis_des_enc_ooo;
+
1203 void *docsis_des_dec_ooo;
+
1204
+
1205 void *hmac_sha_1_ooo;
+
1206 void *hmac_sha_224_ooo;
+
1207 void *hmac_sha_256_ooo;
+
1208 void *hmac_sha_384_ooo;
+
1209 void *hmac_sha_512_ooo;
+
1210 void *hmac_md5_ooo;
+
1211 void *aes_xcbc_ooo;
+
1212 void *aes_ccm_ooo;
+
1213 void *aes_cmac_ooo;
+
1214 void *zuc_eea3_ooo;
+
1215 void *zuc_eia3_ooo;
+
1216 void *aes128_cbcs_ooo;
+
1217 void *zuc256_eea3_ooo;
+
1218 void *zuc256_eia3_ooo;
+
1219 void *aes256_ccm_ooo;
+
1220 void *aes256_cmac_ooo;
+
1221 void *snow3g_uea2_ooo;
+
1222 void *snow3g_uia2_ooo;
+
1223 void *sha_1_ooo;
+
1224 void *sha_224_ooo;
+
1225 void *sha_256_ooo;
+
1226 void *sha_384_ooo;
+
1227 void *sha_512_ooo;
+
1228 void *end_ooo; /* add new out-of-order managers above this line */
+
1229} IMB_MGR;
+
1230
+
1241IMB_DLL_EXPORT const char *imb_get_version_str(void);
+
1242
+
1251IMB_DLL_EXPORT unsigned imb_get_version(void);
+
1252
+
1253
+
1261IMB_DLL_EXPORT int imb_get_errno(IMB_MGR *mb_mgr);
+
1262
+
1270IMB_DLL_EXPORT const char *imb_get_strerror(int errnum);
+
1271
+
1295IMB_DLL_EXPORT IMB_MGR *alloc_mb_mgr(uint64_t flags);
+
1296
+
1303IMB_DLL_EXPORT void free_mb_mgr(IMB_MGR *ptr);
+
1304
+
1310IMB_DLL_EXPORT size_t imb_get_mb_mgr_size(void);
+
1311
+
1332IMB_DLL_EXPORT IMB_MGR *imb_set_pointers_mb_mgr(void *ptr, const uint64_t flags,
+
1333 const unsigned reset_mgr);
+
1334
+
1341IMB_DLL_EXPORT uint64_t imb_get_feature_flags(void);
+
1342
+
1353IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state);
+
1357IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state);
+
1361IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state);
+
1365IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state);
+
1366
+
1367
+
1377IMB_DLL_EXPORT IMB_JOB *submit_job_avx(IMB_MGR *state);
+
1381IMB_DLL_EXPORT IMB_JOB *submit_job_avx2(IMB_MGR *state);
+
1385IMB_DLL_EXPORT IMB_JOB *submit_job_avx512(IMB_MGR *state);
+
1389IMB_DLL_EXPORT IMB_JOB *submit_job_sse(IMB_MGR *state);
+
1390
+
1400IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state);
+
1404IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state);
+
1408IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state);
+
1412IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse(IMB_MGR *state);
+
1413
+
1421IMB_DLL_EXPORT IMB_JOB *flush_job_avx(IMB_MGR *state);
+
1425IMB_DLL_EXPORT IMB_JOB *flush_job_avx2(IMB_MGR *state);
+
1429IMB_DLL_EXPORT IMB_JOB *flush_job_avx512(IMB_MGR *state);
+
1433IMB_DLL_EXPORT IMB_JOB *flush_job_sse(IMB_MGR *state);
+
1434
+
1442IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state);
+
1446IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state);
+
1450IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state);
+
1454IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state);
+
1455
+
1463IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx(IMB_MGR *state);
+
1467IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2(IMB_MGR *state);
+
1471IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512(IMB_MGR *state);
+
1475IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse(IMB_MGR *state);
+
1476
+
1484IMB_DLL_EXPORT IMB_JOB *get_next_job_avx(IMB_MGR *state);
+
1488IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2(IMB_MGR *state);
+
1492IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512(IMB_MGR *state);
+
1496IMB_DLL_EXPORT IMB_JOB *get_next_job_sse(IMB_MGR *state);
+
1497
+
1506IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch);
+
1507
+
1508/*
+
1509 * Wrapper macros to call arch API's set up
+
1510 * at init phase of multi-buffer manager.
+
1511 *
+
1512 * For example, after calling init_mb_mgr_sse(&mgr)
+
1513 * The 'mgr' structure be set up so that:
+
1514 * mgr.get_next_job will point to get_next_job_sse(),
+
1515 * mgr.submit_job will point to submit_job_sse(),
+
1516 * mgr.submit_job_nocheck will point to submit_job_nocheck_sse(),
+
1517 * mgr.get_completed_job will point to get_completed_job_sse(),
+
1518 * mgr.flush_job will point to flush_job_sse(),
+
1519 * mgr.queue_size will point to queue_size_sse()
+
1520 * mgr.keyexp_128 will point to aes_keyexp_128_sse()
+
1521 * mgr.keyexp_192 will point to aes_keyexp_192_sse()
+
1522 * mgr.keyexp_256 will point to aes_keyexp_256_sse()
+
1523 * etc.
+
1524 *
+
1525 * Direct use of arch API's may result in better performance.
+
1526 * Using below indirect interface may produce slightly worse performance but
+
1527 * it can simplify application implementation.
+
1528 * The test app provides example of using the indirect interface.
+
1529 */
+
1530
+
1538#define IMB_GET_NEXT_JOB(_mgr) ((_mgr)->get_next_job((_mgr)))
+
1539
+
1549#define IMB_SUBMIT_JOB(_mgr) ((_mgr)->submit_job((_mgr)))
+
1550
+
1560#define IMB_SUBMIT_JOB_NOCHECK(_mgr) ((_mgr)->submit_job_nocheck((_mgr)))
+
1561
+
1569#define IMB_GET_COMPLETED_JOB(_mgr) ((_mgr)->get_completed_job((_mgr)))
+
1570
+
1578#define IMB_FLUSH_JOB(_mgr) ((_mgr)->flush_job((_mgr)))
+
1579
+
1587#define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr)))
+
1588
+
1598#define IMB_SUBMIT_BURST(_mgr, _jobs, _n_jobs) \
+
1599 ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs)))
+
1600
+
1612#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs) \
+
1613 ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs)))
+
1614
+
1627#define IMB_SUBMIT_CIPHER_BURST(_mgr, _jobs, _n_jobs, _cipher, \
+
1628 _dir, _key_size) \
+
1629 ((_mgr)->submit_cipher_burst((_mgr), (_jobs), (_n_jobs), \
+
1630 (_cipher), (_dir), (_key_size)))
+
1645#define IMB_SUBMIT_CIPHER_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _cipher, \
+
1646 _dir, _key_size) \
+
1647 ((_mgr)->submit_cipher_burst_nocheck((_mgr), (_jobs), (_n_jobs),\
+
1648 (_cipher), (_dir), (_key_size)))
+
1659#define IMB_SUBMIT_HASH_BURST(_mgr, _jobs, _n_jobs, _hash) \
+
1660 ((_mgr)->submit_hash_burst((_mgr), (_jobs), (_n_jobs), (_hash)))
+
1661
+
1674#define IMB_SUBMIT_HASH_BURST_NOCHECK(_mgr, _jobs, _n_jobs, _hash) \
+
1675 ((_mgr)->submit_hash_burst_nocheck((_mgr), (_jobs), (_n_jobs), (_hash)))
+
1676
+
1677/* Key expansion and generation API's */
+
1678
+
1687#define IMB_AES_KEYEXP_128(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1688 ((_mgr)->keyexp_128((_key), (_enc_exp_key), (_dec_exp_key)))
+
1697#define IMB_AES_KEYEXP_192(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1698 ((_mgr)->keyexp_192((_key), (_enc_exp_key), (_dec_exp_key)))
+
1707#define IMB_AES_KEYEXP_256(_mgr, _key, _enc_exp_key, _dec_exp_key) \
+
1708 ((_mgr)->keyexp_256((_key), (_enc_exp_key), (_dec_exp_key)))
+
1709
+
1718#define IMB_AES_CMAC_SUBKEY_GEN_128(_mgr, _exp_key, _key1, _key2) \
+
1719 ((_mgr)->cmac_subkey_gen_128((_exp_key), (_key1), (_key2)))
+
1720
+
1729#define IMB_AES_CMAC_SUBKEY_GEN_256(_mgr, _exp_key, _key1, _key2) \
+
1730 ((_mgr)->cmac_subkey_gen_256((_exp_key), (_key1), (_key2)))
+
1731
+
1741#define IMB_AES_XCBC_KEYEXP(_mgr, _key, _exp_key, _exp_key2, _exp_key3) \
+
1742 ((_mgr)->xcbc_keyexp((_key), (_exp_key), (_exp_key2), (_exp_key3)))
+
1743
+
1744#define IMB_DES_KEYSCHED(_mgr, _exp_key, _key) \
+
1745 ((_mgr)->des_key_sched((_exp_key), (_key)))
+
1746
+
1747/* Hash API's */
+
1748
+
1756#define IMB_SHA1_ONE_BLOCK(_mgr, _src, _tag) \
+
1757 ((_mgr)->sha1_one_block((_src), (_tag)))
+
1758
+
1767#define IMB_SHA1(_mgr, _src, _length, _tag) \
+
1768 ((_mgr)->sha1((_src), (_length), (_tag)))
+
1776#define IMB_SHA224_ONE_BLOCK(_mgr, _src, _tag) \
+
1777 ((_mgr)->sha224_one_block((_src), (_tag)))
+
1778
+
1787#define IMB_SHA224(_mgr, _src, _length, _tag) \
+
1788 ((_mgr)->sha224((_src), (_length), (_tag)))
+
1796#define IMB_SHA256_ONE_BLOCK(_mgr, _src, _tag) \
+
1797 ((_mgr)->sha256_one_block((_src), (_tag)))
+
1806#define IMB_SHA256(_mgr, _src, _length, _tag) \
+
1807 ((_mgr)->sha256((_src), (_length), (_tag)))
+
1815#define IMB_SHA384_ONE_BLOCK(_mgr, _src, _tag) \
+
1816 ((_mgr)->sha384_one_block((_src), (_tag)))
+
1825#define IMB_SHA384(_mgr, _src, _length, _tag) \
+
1826 ((_mgr)->sha384((_src), (_length), (_tag)))
+
1834#define IMB_SHA512_ONE_BLOCK(_mgr, _src, _tag) \
+
1835 ((_mgr)->sha512_one_block((_src), (_tag)))
+
1844#define IMB_SHA512(_mgr, _src, _length, _tag) \
+
1845 ((_mgr)->sha512((_src), (_length), (_tag)))
+
1853#define IMB_MD5_ONE_BLOCK(_mgr, _src, _tag) \
+
1854 ((_mgr)->md5_one_block((_src), (_tag)))
+
1855
+
1869#define IMB_AES128_CFB_ONE(_mgr, _dst, _src, _iv, _exp_key, _len) \
+
1870 ((_mgr)->aes128_cfb_one((_dst), (_src), (_iv), (_exp_key), (_len)))
+
1871
+
1872/* AES-GCM API's */
+
1873#define IMB_AES128_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1874 _aadl, _tag, _tagl) \
+
1875 ((_mgr)->gcm128_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1876 (_aad), (_aadl), (_tag), (_tagl)))
+
1877#define IMB_AES192_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1878 _aadl, _tag, _tagl) \
+
1879 ((_mgr)->gcm192_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1880 (_aad), (_aadl), (_tag), (_tagl)))
+
1881#define IMB_AES256_GCM_ENC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1882 _aadl, _tag, _tagl) \
+
1883 ((_mgr)->gcm256_enc((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1884 (_aad), (_aadl), (_tag), (_tagl)))
+
1885
+
1886#define IMB_AES128_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, _aad, \
+
1887 _aadl, _tag, _tagl) \
+
1888 ((_mgr)->gcm128_dec((_exp_key), (_ctx), (_dst), (_src), (_len), (_iv), \
+
1889 (_aad), (_aadl), (_tag), (_tagl)))
+
1890#define IMB_AES192_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, \
+
1891 _aad, _aadl, _tag, _tagl) \
+
1892 ((_mgr)->gcm192_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
1893 (_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
1894#define IMB_AES256_GCM_DEC(_mgr, _exp_key, _ctx, _dst, _src, _len, _iv, \
+
1895 _aad, _aadl, _tag, _tagl) \
+
1896 ((_mgr)->gcm256_dec((_exp_key), (_ctx), (_dst), (_src), (_len), \
+
1897 (_iv), (_aad), (_aadl), (_tag), (_tagl)))
+
1898
+
1899#define IMB_AES128_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1900 ((_mgr)->gcm128_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1901#define IMB_AES192_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1902 ((_mgr)->gcm192_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1903#define IMB_AES256_GCM_INIT(_mgr, _exp_key, _ctx, _iv, _aad, _aadl) \
+
1904 ((_mgr)->gcm256_init((_exp_key), (_ctx), (_iv), (_aad), (_aadl)))
+
1905
+
1906#define IMB_AES128_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1907 _aadl) \
+
1908 ((_mgr)->gcm128_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1909 (_aad), (_aadl)))
+
1910#define IMB_AES192_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1911 _aadl) \
+
1912 ((_mgr)->gcm192_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1913 (_aad), (_aadl)))
+
1914#define IMB_AES256_GCM_INIT_VAR_IV(_mgr, _exp_key, _ctx, _iv, _ivl, _aad, \
+
1915 _aadl) \
+
1916 ((_mgr)->gcm256_init_var_iv((_exp_key), (_ctx), (_iv), (_ivl), \
+
1917 (_aad), (_aadl)))
+
1918
+
1919#define IMB_AES128_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1920 ((_mgr)->gcm128_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1921#define IMB_AES192_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1922 ((_mgr)->gcm192_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1923#define IMB_AES256_GCM_ENC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1924 ((_mgr)->gcm256_enc_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1925
+
1926#define IMB_AES128_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1927 ((_mgr)->gcm128_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1928#define IMB_AES192_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1929 ((_mgr)->gcm192_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1930#define IMB_AES256_GCM_DEC_UPDATE(_mgr, _exp_key, _ctx, _dst, _src, _len) \
+
1931 ((_mgr)->gcm256_dec_update((_exp_key), (_ctx), (_dst), (_src), (_len)))
+
1932
+
1933#define IMB_AES128_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1934 ((_mgr)->gcm128_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1935#define IMB_AES192_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1936 ((_mgr)->gcm192_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1937#define IMB_AES256_GCM_ENC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1938 ((_mgr)->gcm256_enc_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1939
+
1940#define IMB_AES128_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1941 ((_mgr)->gcm128_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1942#define IMB_AES192_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1943 ((_mgr)->gcm192_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1944#define IMB_AES256_GCM_DEC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1945 ((_mgr)->gcm256_dec_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1946
+
1947#define IMB_AES128_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1948 ((_mgr)->gmac128_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1949#define IMB_AES192_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1950 ((_mgr)->gmac192_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1951#define IMB_AES256_GMAC_INIT(_mgr, _exp_key, _ctx, _iv, _ivl) \
+
1952 ((_mgr)->gmac256_init((_exp_key), (_ctx), (_iv), (_ivl)))
+
1953
+
1954#define IMB_AES128_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1955 ((_mgr)->gmac128_update((_exp_key), (_ctx), (_src), (_len)))
+
1956#define IMB_AES192_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1957 ((_mgr)->gmac192_update((_exp_key), (_ctx), (_src), (_len)))
+
1958#define IMB_AES256_GMAC_UPDATE(_mgr, _exp_key, _ctx, _src, _len) \
+
1959 ((_mgr)->gmac256_update((_exp_key), (_ctx), (_src), (_len)))
+
1960
+
1961#define IMB_AES128_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1962 ((_mgr)->gmac128_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1963#define IMB_AES192_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1964 ((_mgr)->gmac192_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1965#define IMB_AES256_GMAC_FINALIZE(_mgr, _exp_key, _ctx, _tag, _tagl) \
+
1966 ((_mgr)->gmac256_finalize((_exp_key), (_ctx), (_tag), (_tagl)))
+
1967
+
1968#define IMB_AES128_GCM_PRECOMP(_mgr, _key) \
+
1969 ((_mgr)->gcm128_precomp((_key)))
+
1970#define IMB_AES192_GCM_PRECOMP(_mgr, _key) \
+
1971 ((_mgr)->gcm192_precomp((_key)))
+
1972#define IMB_AES256_GCM_PRECOMP(_mgr, _key) \
+
1973 ((_mgr)->gcm256_precomp((_key)))
+
1974
+
1975#define IMB_AES128_GCM_PRE(_mgr, _key, _exp_key) \
+
1976 ((_mgr)->gcm128_pre((_key), (_exp_key)))
+
1977#define IMB_AES192_GCM_PRE(_mgr, _key, _exp_key) \
+
1978 ((_mgr)->gcm192_pre((_key), (_exp_key)))
+
1979#define IMB_AES256_GCM_PRE(_mgr, _key, _exp_key) \
+
1980 ((_mgr)->gcm256_pre((_key), (_exp_key)))
+
1981
+
1982#define IMB_GHASH_PRE(_mgr, _key, _exp_key) \
+
1983 ((_mgr)->ghash_pre((_key), (_exp_key)))
+
1984#define IMB_GHASH(_mgr, _exp_key, _src, _len, _tag, _tagl) \
+
1985 ((_mgr)->ghash((_exp_key), (_src), (_len), (_tag), (_tagl)))
+
1986
+
1987/* Chacha20-Poly1305 direct API's */
+
1988#define IMB_CHACHA20_POLY1305_INIT(_mgr, _key, _ctx, _iv, _aad, _aadl) \
+
1989 ((_mgr)->chacha20_poly1305_init((_key), (_ctx), (_iv), (_aad), \
+
1990 (_aadl)))
+
1991
+
1992#define IMB_CHACHA20_POLY1305_ENC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len) \
+
1993 ((_mgr)->chacha20_poly1305_enc_update((_key), (_ctx), (_dst), (_src), \
+
1994 (_len)))
+
1995#define IMB_CHACHA20_POLY1305_DEC_UPDATE(_mgr, _key, _ctx, _dst, _src, _len) \
+
1996 ((_mgr)->chacha20_poly1305_dec_update((_key), (_ctx), (_dst), (_src), \
+
1997 (_len)))
+
1998
+
1999#define IMB_CHACHA20_POLY1305_ENC_FINALIZE(_mgr, _ctx, _tag, _tagl) \
+
2000 ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
2001
+
2002#define IMB_CHACHA20_POLY1305_DEC_FINALIZE(_mgr, _ctx, _tag, _tagl) \
+
2003 ((_mgr)->chacha20_poly1305_finalize((_ctx), (_tag), (_tagl)))
+
2004
+
2005/* ZUC EEA3/EIA3 functions */
+
2006
+
2017#define IMB_ZUC_EEA3_1_BUFFER(_mgr, _key, _iv, _src, _dst, _len) \
+
2018 ((_mgr)->eea3_1_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
2019#define IMB_ZUC_EEA3_4_BUFFER(_mgr, _key, _iv, _src, _dst, _len) \
+
2020 ((_mgr)->eea3_4_buffer((_key), (_iv), (_src), (_dst), (_len)))
+
2021#define IMB_ZUC_EEA3_N_BUFFER(_mgr, _key, _iv, _src, _dst, _len, _count) \
+
2022 ((_mgr)->eea3_n_buffer((_key), (_iv), (_src), (_dst), (_len), (_count)))
+
2023
+
2024
+
2035#define IMB_ZUC_EIA3_1_BUFFER(_mgr, _key, _iv, _src, _len, _tag) \
+
2036 ((_mgr)->eia3_1_buffer((_key), (_iv), (_src), (_len), (_tag)))
+
2037#define IMB_ZUC_EIA3_N_BUFFER(_mgr, _key, _iv, _src, _len, _tag, _count) \
+
2038 ((_mgr)->eia3_n_buffer((_key), (_iv), (_src), (_len), (_tag), (_count)))
+
2039
+
2040
+
2041/* KASUMI F8/F9 functions */
+
2042
+
2058#define IMB_KASUMI_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2059 ((_mgr)->f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
2060
+
2077#define IMB_KASUMI_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, _len, \
+
2078 _offset) \
+
2079 ((_mgr)->f8_1_buffer_bit((_exp_key), (_iv), (_src), (_dst), (_len), \
+
2080 (_offset)))
+
2081
+
2101#define IMB_KASUMI_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, _src1, _dst1, \
+
2102 _len1, _src2, _dst2, _len2) \
+
2103 ((_mgr)->f8_2_buffer((_exp_key), (_iv1), (_iv2), (_src1), (_dst1), \
+
2104 (_len1), (_src2), (_dst2), (_len2)))
+
2126#define IMB_KASUMI_F8_3_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _src1, _dst1, \
+
2127 _src2, _dst2, _src3, _dst3, _len) \
+
2128 ((_mgr)->f8_3_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_src1), \
+
2129 (_dst1), (_src2), (_dst2), (_src3), (_dst3), \
+
2130 (_len)))
+
2155#define IMB_KASUMI_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2156 _src1, _dst1, _src2, _dst2, _src3, _dst3, \
+
2157 _src4, _dst4, _len) \
+
2158 ((_mgr)->f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), (_iv4), \
+
2159 (_src1), (_dst1), (_src2), (_dst2), \
+
2160 (_src3), (_dst3), (_src4), (_dst4), (_len)))
+
2176#define IMB_KASUMI_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count) \
+
2177 ((_mgr)->f8_n_buffer((_exp_key), (_iv), (_src), (_dst), (_len), \
+
2178 (_count)))
+
2194#define IMB_KASUMI_F9_1_BUFFER(_mgr, _exp_key, _src, _len, _tag) \
+
2195 ((_mgr)->f9_1_buffer((_exp_key), (_src), (_len), (_tag)))
+
2196
+
2211#define IMB_KASUMI_F9_1_BUFFER_USER(_mgr, _exp_key, _iv, _src, _len, _tag, \
+
2212 _dir) \
+
2213 ((_mgr)->f9_1_buffer_user((_exp_key), (_iv), (_src), (_len), \
+
2214 (_tag), (_dir)))
+
2215
+
2225#define IMB_KASUMI_INIT_F8_KEY_SCHED(_mgr, _key, _exp_key) \
+
2226 ((_mgr)->kasumi_init_f8_key_sched((_key), (_exp_key)))
+
2227
+
2237#define IMB_KASUMI_INIT_F9_KEY_SCHED(_mgr, _key, _exp_key) \
+
2238 ((_mgr)->kasumi_init_f9_key_sched((_key), (_exp_key)))
+
2239
+
2249#define IMB_KASUMI_KEY_SCHED_SIZE(_mgr)((_mgr)->kasumi_key_sched_size())
+
2250
+
2251
+
2252/* SNOW3G F8/F9 functions */
+
2253
+
2269#define IMB_SNOW3G_F8_1_BUFFER_BIT(_mgr, _exp_key, _iv, _src, _dst, \
+
2270 _len, _offset) \
+
2271 ((_mgr)->snow3g_f8_1_buffer_bit((_exp_key), (_iv), (_src), \
+
2272 (_dst), (_len), (_offset)))
+
2273
+
2288#define IMB_SNOW3G_F8_1_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2289 ((_mgr)->snow3g_f8_1_buffer((_exp_key), (_iv), (_src), (_dst), (_len)))
+
2290
+
2307#define IMB_SNOW3G_F8_2_BUFFER(_mgr, _exp_key, _iv1, _iv2, \
+
2308 _src1, _dst1, _len1, \
+
2309 _src2, _dst2, _len2) \
+
2310 ((_mgr)->snow3g_f8_2_buffer((_exp_key), (_iv1), (_iv2), \
+
2311 (_src1), (_dst1), (_len1), \
+
2312 (_src2), (_dst2), (_len2)))
+
2313
+
2339#define IMB_SNOW3G_F8_4_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2340 _src1, _dst1, _len1, \
+
2341 _src2, _dst2, _len2, \
+
2342 _src3, _dst3, _len3, \
+
2343 _src4, _dst4, _len4) \
+
2344 ((_mgr)->snow3g_f8_4_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
2345 (_iv4), (_src1), (_dst1), (_len1), \
+
2346 (_src2), (_dst2), (_len2), \
+
2347 (_src3), (_dst3), (_len3), \
+
2348 (_src4), (_dst4), (_len4)))
+
2349
+
2391#define IMB_SNOW3G_F8_8_BUFFER(_mgr, _exp_key, _iv1, _iv2, _iv3, _iv4, \
+
2392 _iv5, _iv6, _iv7, _iv8, \
+
2393 _src1, _dst1, _len1, \
+
2394 _src2, _dst2, _len2, \
+
2395 _src3, _dst3, _len3, \
+
2396 _src4, _dst4, _len4, \
+
2397 _src5, _dst5, _len5, \
+
2398 _src6, _dst6, _len6, \
+
2399 _src7, _dst7, _len7, \
+
2400 _src8, _dst8, _len8) \
+
2401 ((_mgr)->snow3g_f8_8_buffer((_exp_key), (_iv1), (_iv2), (_iv3), \
+
2402 (_iv4), (_iv5), (_iv6), (_iv7), \
+
2403 (_iv8), (_src1), (_dst1), (_len1), \
+
2404 (_src2), (_dst2), (_len2), \
+
2405 (_src3), (_dst3), (_len3), \
+
2406 (_src4), (_dst4), (_len4), \
+
2407 (_src5), (_dst5), (_len5), \
+
2408 (_src6), (_dst6), (_len6), \
+
2409 (_src7), (_dst7), (_len7), \
+
2410 (_src8), (_dst8), (_len8)))
+
2424#define IMB_SNOW3G_F8_8_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, _dst, _len) \
+
2425 ((_mgr)->snow3g_f8_8_buffer_multikey((_exp_key), (_iv), (_src), (_dst),\
+
2426 (_len)))
+
2427
+
2442#define IMB_SNOW3G_F8_N_BUFFER(_mgr, _exp_key, _iv, _src, _dst, _len, _count) \
+
2443 ((_mgr)->snow3g_f8_n_buffer((_exp_key), (_iv), (_src), \
+
2444 (_dst), (_len), (_count)))
+
2445
+
2460#define IMB_SNOW3G_F8_N_BUFFER_MULTIKEY(_mgr, _exp_key, _iv, _src, \
+
2461 _dst, _len, _count) \
+
2462 ((_mgr)->snow3g_f8_n_buffer_multikey((_exp_key), (_iv), (_src), \
+
2463 (_dst), (_len), (_count)))
+
2464
+
2480#define IMB_SNOW3G_F9_1_BUFFER(_mgr, _exp_key, _iv, _src, _len, _tag) \
+
2481 ((_mgr)->snow3g_f9_1_buffer((_exp_key), (_iv), (_src), (_len), (_tag)))
+
2482
+
2493#define IMB_SNOW3G_INIT_KEY_SCHED(_mgr, _key, _exp_key) \
+
2494 ((_mgr)->snow3g_init_key_sched((_key), (_exp_key)))
+
2495
+
2505#define IMB_SNOW3G_KEY_SCHED_SIZE(_mgr)((_mgr)->snow3g_key_sched_size())
+
2506
+
2510#define IMB_HEC_32(_mgr, _src)((_mgr)->hec_32(_src))
+
2511#define IMB_HEC_64(_mgr, _src)((_mgr)->hec_64(_src))
+
2512
+
2516#define IMB_CRC32_ETHERNET_FCS(_mgr, _src, _len) \
+
2517 (_mgr)->crc32_ethernet_fcs(_src, _len)
+
2518
+
2522#define IMB_CRC16_X25(_mgr, _src, _len) \
+
2523 (_mgr)->crc16_x25(_src, _len)
+
2524
+
2528#define IMB_CRC32_SCTP(_mgr, _src, _len) \
+
2529 (_mgr)->crc32_sctp(_src, _len)
+
2530
+
2534#define IMB_CRC24_LTE_A(_mgr, _src, _len) \
+
2535 (_mgr)->crc24_lte_a(_src, _len)
+
2536
+
2540#define IMB_CRC24_LTE_B(_mgr, _src, _len) \
+
2541 (_mgr)->crc24_lte_b(_src, _len)
+
2542
+
2546#define IMB_CRC16_FP_DATA(_mgr, _src, _len) \
+
2547 (_mgr)->crc16_fp_data(_src, _len)
+
2548
+
2552#define IMB_CRC11_FP_HEADER(_mgr, _src, _len) \
+
2553 (_mgr)->crc11_fp_header(_src, _len)
+
2554
+
2558#define IMB_CRC7_FP_HEADER(_mgr, _src, _len) \
+
2559 (_mgr)->crc7_fp_header(_src, _len)
+
2560
+
2564#define IMB_CRC10_IUUP_DATA(_mgr, _src, _len) \
+
2565 (_mgr)->crc10_iuup_data(_src, _len)
+
2566
+
2570#define IMB_CRC6_IUUP_HEADER(_mgr, _src, _len) \
+
2571 (_mgr)->crc6_iuup_header(_src, _len)
+
2572
+
2576#define IMB_CRC32_WIMAX_OFDMA_DATA(_mgr, _src, _len) \
+
2577 (_mgr)->crc32_wimax_ofdma_data(_src, _len)
+
2578
+
2582#define IMB_CRC8_WIMAX_OFDMA_HCS(_mgr, _src, _len) \
+
2583 (_mgr)->crc8_wimax_ofdma_hcs(_src, _len)
+
2584
+
2585/* Auxiliary functions */
+
2586
+
2599IMB_DLL_EXPORT int
+
2600des_key_schedule(uint64_t *ks, const void *key);
+
2601
+
2609IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length,
+
2610 void *digest);
+
2611
+
2615IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length,
+
2616 void *digest);
+
2620IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length,
+
2621 void *digest);
+
2625IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length,
+
2626 void *digest);
+
2627
+
2634IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest);
+
2638IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest);
+
2642IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest);
+
2646IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest);
+
2647
+
2655IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length,
+
2656 void *digest);
+
2660IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length,
+
2661 void *digest);
+
2665IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length,
+
2666 void *digest);
+
2670IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length,
+
2671 void *digest);
+
2672
+
2679IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest);
+
2683IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest);
+
2687IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest);
+
2691IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest);
+
2692
+
2700IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length,
+
2701 void *digest);
+
2705IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length,
+
2706 void *digest);
+
2710IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length,
+
2711 void *digest);
+
2715IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length,
+
2716 void *digest);
+
2717
+
2724IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest);
+
2728IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest);
+
2732IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest);
+
2736IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest);
+
2737
+
2745IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length,
+
2746 void *digest);
+
2750IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length,
+
2751 void *digest);
+
2755IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length,
+
2756 void *digest);
+
2760IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length,
+
2761 void *digest);
+
2762
+
2769IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest);
+
2773IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest);
+
2777IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest);
+
2781IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest);
+
2782
+
2790IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length,
+
2791 void *digest);
+
2795IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length,
+
2796 void *digest);
+
2800IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length,
+
2801 void *digest);
+
2805IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length,
+
2806 void *digest);
+
2807
+
2814IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest);
+
2818IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest);
+
2822IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest);
+
2826IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest);
+
2827
+
2834IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest);
+
2838IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest);
+
2842IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest);
+
2846IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest);
+
2847
+
2848
+
2856IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys,
+
2857 void *dec_exp_keys);
+
2861IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys,
+
2862 void *dec_exp_keys);
+
2866IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys,
+
2867 void *dec_exp_keys);
+
2871IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys,
+
2872 void *dec_exp_keys);
+
2873
+
2881IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys,
+
2882 void *dec_exp_keys);
+
2886IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys,
+
2887 void *dec_exp_keys);
+
2891IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys,
+
2892 void *dec_exp_keys);
+
2896IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys,
+
2897 void *dec_exp_keys);
+
2898
+
2906IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys,
+
2907 void *dec_exp_keys);
+
2911IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys,
+
2912 void *dec_exp_keys);
+
2916IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys,
+
2917 void *dec_exp_keys);
+
2921IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys,
+
2922 void *dec_exp_keys);
+
2923
+
2930IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key,
+
2931 void *enc_exp_keys);
+
2935IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key,
+
2936 void *enc_exp_keys);
+
2940IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key,
+
2941 void *enc_exp_keys);
+
2945IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key,
+
2946 void *enc_exp_keys);
+
2947
+
2954IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key,
+
2955 void *enc_exp_keys);
+
2959IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key,
+
2960 void *enc_exp_keys);
+
2964IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key,
+
2965 void *enc_exp_keys);
+
2969IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key,
+
2970 void *enc_exp_keys);
+
2971
+
2978IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key,
+
2979 void *enc_exp_keys);
+
2983IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key,
+
2984 void *enc_exp_keys);
+
2988IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key,
+
2989 void *enc_exp_keys);
+
2993IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key,
+
2994 void *enc_exp_keys);
+
2995
+
3004IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp,
+
3005 void *k2, void *k3);
+
3009IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp,
+
3010 void *k2, void *k3);
+
3014IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp,
+
3015 void *k2, void *k3);
+
3019IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp,
+
3020 void *k2, void *k3);
+
3021
+
3029IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1,
+
3030 void *key2);
+
3034IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1,
+
3035 void *key2);
+
3039IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1,
+
3040 void *key2);
+
3044IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1,
+
3045 void *key2);
+
3058IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in,
+
3059 const void *iv, const void *keys,
+
3060 uint64_t len);
+
3064IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in,
+
3065 const void *iv, const void *keys,
+
3066 uint64_t len);
+
3070IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in,
+
3071 const void *iv, const void *keys,
+
3072 uint64_t len);
+
3076IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in,
+
3077 const void *iv, const void *keys,
+
3078 uint64_t len);
+
3079
+
3080/*
+
3081 * Direct GCM API.
+
3082 * Note that GCM is also available through job API.
+
3083 */
+
3084
+
3103IMB_DLL_EXPORT void
+
3104aes_gcm_enc_128_sse(const struct gcm_key_data *key_data,
+
3105 struct gcm_context_data *context_data,
+
3106 uint8_t *out, uint8_t const *in, uint64_t len,
+
3107 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3108 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3112IMB_DLL_EXPORT void
+
3113aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data,
+
3114 struct gcm_context_data *context_data,
+
3115 uint8_t *out, uint8_t const *in, uint64_t len,
+
3116 const uint8_t *iv,
+
3117 uint8_t const *aad, uint64_t aad_len,
+
3118 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3122IMB_DLL_EXPORT void
+
3123aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data,
+
3124 struct gcm_context_data *context_data,
+
3125 uint8_t *out, uint8_t const *in, uint64_t len,
+
3126 const uint8_t *iv,
+
3127 uint8_t const *aad, uint64_t aad_len,
+
3128 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3147IMB_DLL_EXPORT void
+
3148aes_gcm_enc_192_sse(const struct gcm_key_data *key_data,
+
3149 struct gcm_context_data *context_data,
+
3150 uint8_t *out, uint8_t const *in, uint64_t len,
+
3151 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3152 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3156IMB_DLL_EXPORT void
+
3157aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data,
+
3158 struct gcm_context_data *context_data,
+
3159 uint8_t *out, uint8_t const *in, uint64_t len,
+
3160 const uint8_t *iv,
+
3161 uint8_t const *aad, uint64_t aad_len,
+
3162 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3166IMB_DLL_EXPORT void
+
3167aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data,
+
3168 struct gcm_context_data *context_data,
+
3169 uint8_t *out, uint8_t const *in, uint64_t len,
+
3170 const uint8_t *iv,
+
3171 uint8_t const *aad, uint64_t aad_len,
+
3172 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3173
+
3192IMB_DLL_EXPORT void
+
3193aes_gcm_enc_256_sse(const struct gcm_key_data *key_data,
+
3194 struct gcm_context_data *context_data,
+
3195 uint8_t *out, uint8_t const *in, uint64_t len,
+
3196 const uint8_t *iv,
+
3197 uint8_t const *aad, uint64_t aad_len,
+
3198 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3202IMB_DLL_EXPORT void
+
3203aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data,
+
3204 struct gcm_context_data *context_data,
+
3205 uint8_t *out, uint8_t const *in, uint64_t len,
+
3206 const uint8_t *iv,
+
3207 uint8_t const *aad, uint64_t aad_len,
+
3208 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3212IMB_DLL_EXPORT void
+
3213aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data,
+
3214 struct gcm_context_data *context_data,
+
3215 uint8_t *out, uint8_t const *in, uint64_t len,
+
3216 const uint8_t *iv,
+
3217 uint8_t const *aad, uint64_t aad_len,
+
3218 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3219
+
3238IMB_DLL_EXPORT void
+
3239aes_gcm_dec_128_sse(const struct gcm_key_data *key_data,
+
3240 struct gcm_context_data *context_data,
+
3241 uint8_t *out, uint8_t const *in, uint64_t len,
+
3242 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3243 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3247IMB_DLL_EXPORT void
+
3248aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data,
+
3249 struct gcm_context_data *context_data,
+
3250 uint8_t *out, uint8_t const *in, uint64_t len,
+
3251 const uint8_t *iv,
+
3252 uint8_t const *aad, uint64_t aad_len,
+
3253 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3257IMB_DLL_EXPORT void
+
3258aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data,
+
3259 struct gcm_context_data *context_data,
+
3260 uint8_t *out, uint8_t const *in, uint64_t len,
+
3261 const uint8_t *iv,
+
3262 uint8_t const *aad, uint64_t aad_len,
+
3263 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3264
+
3283IMB_DLL_EXPORT void
+
3284aes_gcm_dec_192_sse(const struct gcm_key_data *key_data,
+
3285 struct gcm_context_data *context_data,
+
3286 uint8_t *out, uint8_t const *in, uint64_t len,
+
3287 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3288 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3292IMB_DLL_EXPORT void
+
3293aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data,
+
3294 struct gcm_context_data *context_data,
+
3295 uint8_t *out, uint8_t const *in, uint64_t len,
+
3296 const uint8_t *iv,
+
3297 uint8_t const *aad, uint64_t aad_len,
+
3298 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3302IMB_DLL_EXPORT void
+
3303aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data,
+
3304 struct gcm_context_data *context_data,
+
3305 uint8_t *out, uint8_t const *in, uint64_t len,
+
3306 const uint8_t *iv,
+
3307 uint8_t const *aad, uint64_t aad_len,
+
3308 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3309
+
3328IMB_DLL_EXPORT void
+
3329aes_gcm_dec_256_sse(const struct gcm_key_data *key_data,
+
3330 struct gcm_context_data *context_data,
+
3331 uint8_t *out, uint8_t const *in, uint64_t len,
+
3332 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len,
+
3333 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3337IMB_DLL_EXPORT void
+
3338aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data,
+
3339 struct gcm_context_data *context_data,
+
3340 uint8_t *out, uint8_t const *in, uint64_t len,
+
3341 const uint8_t *iv,
+
3342 uint8_t const *aad, uint64_t aad_len,
+
3343 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3347IMB_DLL_EXPORT void
+
3348aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data,
+
3349 struct gcm_context_data *context_data,
+
3350 uint8_t *out, uint8_t const *in, uint64_t len,
+
3351 const uint8_t *iv,
+
3352 uint8_t const *aad, uint64_t aad_len,
+
3353 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3354
+
3367IMB_DLL_EXPORT void
+
3368aes_gcm_init_128_sse(const struct gcm_key_data *key_data,
+
3369 struct gcm_context_data *context_data,
+
3370 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3374IMB_DLL_EXPORT void
+
3375aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data,
+
3376 struct gcm_context_data *context_data,
+
3377 const uint8_t *iv,
+
3378 uint8_t const *aad, uint64_t aad_len);
+
3382IMB_DLL_EXPORT void
+
3383aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data,
+
3384 struct gcm_context_data *context_data,
+
3385 const uint8_t *iv,
+
3386 uint8_t const *aad, uint64_t aad_len);
+
3399IMB_DLL_EXPORT void
+
3400aes_gcm_init_192_sse(const struct gcm_key_data *key_data,
+
3401 struct gcm_context_data *context_data,
+
3402 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3406IMB_DLL_EXPORT void
+
3407aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data,
+
3408 struct gcm_context_data *context_data,
+
3409 const uint8_t *iv,
+
3410 uint8_t const *aad, uint64_t aad_len);
+
3414IMB_DLL_EXPORT void
+
3415aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data,
+
3416 struct gcm_context_data *context_data,
+
3417 const uint8_t *iv,
+
3418 uint8_t const *aad, uint64_t aad_len);
+
3431IMB_DLL_EXPORT void
+
3432aes_gcm_init_256_sse(const struct gcm_key_data *key_data,
+
3433 struct gcm_context_data *context_data,
+
3434 const uint8_t *iv, uint8_t const *aad, uint64_t aad_len);
+
3438IMB_DLL_EXPORT void
+
3439aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data,
+
3440 struct gcm_context_data *context_data,
+
3441 const uint8_t *iv,
+
3442 uint8_t const *aad, uint64_t aad_len);
+
3446IMB_DLL_EXPORT void
+
3447aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data,
+
3448 struct gcm_context_data *context_data,
+
3449 const uint8_t *iv,
+
3450 uint8_t const *aad, uint64_t aad_len);
+
3451
+
3461IMB_DLL_EXPORT void
+
3462aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data,
+
3463 struct gcm_context_data *context_data,
+
3464 uint8_t *out, const uint8_t *in, uint64_t len);
+
3468IMB_DLL_EXPORT void
+
3469aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data,
+
3470 struct gcm_context_data *context_data,
+
3471 uint8_t *out, const uint8_t *in, uint64_t len);
+
3475IMB_DLL_EXPORT void
+
3476aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data,
+
3477 struct gcm_context_data *context_data,
+
3478 uint8_t *out, const uint8_t *in, uint64_t len);
+
3479
+
3489IMB_DLL_EXPORT void
+
3490aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data,
+
3491 struct gcm_context_data *context_data,
+
3492 uint8_t *out, const uint8_t *in, uint64_t len);
+
3496IMB_DLL_EXPORT void
+
3497aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data,
+
3498 struct gcm_context_data *context_data,
+
3499 uint8_t *out, const uint8_t *in, uint64_t len);
+
3503IMB_DLL_EXPORT void
+
3504aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data,
+
3505 struct gcm_context_data *context_data,
+
3506 uint8_t *out, const uint8_t *in, uint64_t len);
+
3507
+
3517IMB_DLL_EXPORT void
+
3518aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data,
+
3519 struct gcm_context_data *context_data,
+
3520 uint8_t *out, const uint8_t *in, uint64_t len);
+
3524IMB_DLL_EXPORT void
+
3525aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data,
+
3526 struct gcm_context_data *context_data,
+
3527 uint8_t *out, const uint8_t *in, uint64_t len);
+
3531IMB_DLL_EXPORT void
+
3532aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data,
+
3533 struct gcm_context_data *context_data,
+
3534 uint8_t *out, const uint8_t *in, uint64_t len);
+
3535
+
3545IMB_DLL_EXPORT void
+
3546aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data,
+
3547 struct gcm_context_data *context_data,
+
3548 uint8_t *out, const uint8_t *in, uint64_t len);
+
3552IMB_DLL_EXPORT void
+
3553aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data,
+
3554 struct gcm_context_data *context_data,
+
3555 uint8_t *out, const uint8_t *in, uint64_t len);
+
3559IMB_DLL_EXPORT void
+
3560aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data,
+
3561 struct gcm_context_data *context_data,
+
3562 uint8_t *out, const uint8_t *in, uint64_t len);
+
3563
+
3573IMB_DLL_EXPORT void
+
3574aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data,
+
3575 struct gcm_context_data *context_data,
+
3576 uint8_t *out, const uint8_t *in, uint64_t len);
+
3580IMB_DLL_EXPORT void
+
3581aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data,
+
3582 struct gcm_context_data *context_data,
+
3583 uint8_t *out, const uint8_t *in, uint64_t len);
+
3587IMB_DLL_EXPORT void
+
3588aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data,
+
3589 struct gcm_context_data *context_data,
+
3590 uint8_t *out, const uint8_t *in, uint64_t len);
+
3591
+
3601IMB_DLL_EXPORT void
+
3602aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data,
+
3603 struct gcm_context_data *context_data,
+
3604 uint8_t *out, const uint8_t *in, uint64_t len);
+
3608IMB_DLL_EXPORT void
+
3609aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data,
+
3610 struct gcm_context_data *context_data,
+
3611 uint8_t *out, const uint8_t *in, uint64_t len);
+
3615IMB_DLL_EXPORT void
+
3616aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data,
+
3617 struct gcm_context_data *context_data,
+
3618 uint8_t *out, const uint8_t *in, uint64_t len);
+
3619
+
3630IMB_DLL_EXPORT void
+
3631aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data,
+
3632 struct gcm_context_data *context_data,
+
3633 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3637IMB_DLL_EXPORT void
+
3638aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3639 struct gcm_context_data *context_data,
+
3640 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3644IMB_DLL_EXPORT void
+
3645aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3646 struct gcm_context_data *context_data,
+
3647 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3648
+
3659IMB_DLL_EXPORT void
+
3660aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data,
+
3661 struct gcm_context_data *context_data,
+
3662 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3666IMB_DLL_EXPORT void
+
3667aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3668 struct gcm_context_data *context_data,
+
3669 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3673IMB_DLL_EXPORT void
+
3674aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3675 struct gcm_context_data *context_data,
+
3676 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3677
+
3688IMB_DLL_EXPORT void
+
3689aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data,
+
3690 struct gcm_context_data *context_data,
+
3691 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3695IMB_DLL_EXPORT void
+
3696aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3697 struct gcm_context_data *context_data,
+
3698 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3702IMB_DLL_EXPORT void
+
3703aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3704 struct gcm_context_data *context_data,
+
3705 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3706
+
3717IMB_DLL_EXPORT void
+
3718aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data,
+
3719 struct gcm_context_data *context_data,
+
3720 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3724IMB_DLL_EXPORT void
+
3725aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3726 struct gcm_context_data *context_data,
+
3727 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3731IMB_DLL_EXPORT void
+
3732aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3733 struct gcm_context_data *context_data,
+
3734 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3735
+
3746IMB_DLL_EXPORT void
+
3747aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data,
+
3748 struct gcm_context_data *context_data,
+
3749 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3753IMB_DLL_EXPORT void
+
3754aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3755 struct gcm_context_data *context_data,
+
3756 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3760IMB_DLL_EXPORT void
+
3761aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3762 struct gcm_context_data *context_data,
+
3763 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3764
+
3775IMB_DLL_EXPORT void
+
3776aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data,
+
3777 struct gcm_context_data *context_data,
+
3778 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3782IMB_DLL_EXPORT void
+
3783aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data,
+
3784 struct gcm_context_data *context_data,
+
3785 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3789IMB_DLL_EXPORT void
+
3790aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data,
+
3791 struct gcm_context_data *context_data,
+
3792 uint8_t *auth_tag, uint64_t auth_tag_len);
+
3793
+
3802IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data);
+
3803
+
3807IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data);
+
3808
+
3812IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data);
+
3813
+
3822IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data);
+
3823
+
3827IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data);
+
3828
+
3832IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data);
+
3833
+
3842IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data);
+
3843
+
3847IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data);
+
3848
+
3852IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data);
+
3853
+
3863IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key,
+
3864 struct gcm_key_data *key_data);
+
3868IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key,
+
3869 struct gcm_key_data *key_data);
+
3873IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key,
+
3874 struct gcm_key_data *key_data);
+
3884IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key,
+
3885 struct gcm_key_data *key_data);
+
3889IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key,
+
3890 struct gcm_key_data *key_data);
+
3894IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key,
+
3895 struct gcm_key_data *key_data);
+
3905IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key,
+
3906 struct gcm_key_data *key_data);
+
3910IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key,
+
3911 struct gcm_key_data *key_data);
+
3915IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key,
+
3916 struct gcm_key_data *key_data);
+
3917
+
3930IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count,
+
3931 const uint8_t bearer,
+
3932 const uint8_t dir,
+
3933 void *iv_ptr);
+
3946IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count,
+
3947 const uint8_t bearer,
+
3948 const uint8_t dir,
+
3949 void *iv_ptr);
+
3950
+
3963IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count,
+
3964 const uint8_t bearer,
+
3965 const uint8_t dir,
+
3966 void *iv_ptr);
+
3978IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count,
+
3979 const uint32_t fresh,
+
3980 void *iv_ptr);
+
3981
+
3997IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count,
+
3998 const uint8_t bearer,
+
3999 const uint8_t dir,
+
4000 void *iv_ptr);
+
4016IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count,
+
4017 const uint32_t fresh,
+
4018 const uint8_t dir,
+
4019 void *iv_ptr);
+
4026IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size);
+
4027
+
4028#ifdef __cplusplus
+
4029}
+
4030#endif
+
4031
+
4032#endif /* IMB_IPSEC_MB_H */
+
aes_gcm_enc_128_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
get_next_job_sse
IMB_DLL_EXPORT IMB_JOB * get_next_job_sse(IMB_MGR *state)
Get next available job.
+
KASUMI_KEY_SCHEDULE_SIZE
#define KASUMI_KEY_SCHEDULE_SIZE
Definition: intel-ipsec-mb.h:648
+
aes_gcm_pre_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
aes_gcm_dec_128_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
aes_gcm_init_256_sse
IMB_DLL_EXPORT void aes_gcm_init_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
zuc_eea3_1_buffer_t
void(* zuc_eea3_1_buffer_t)(const void *, const void *, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:853
+
sha256_one_block_avx2
IMB_DLL_EXPORT void sha256_one_block_avx2(const void *data, void *digest)
+
aes_keyexp_192_enc_sse
IMB_DLL_EXPORT void aes_keyexp_192_enc_sse(const void *key, void *enc_exp_keys)
+
aes_gcm_dec_256_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
IMB_GCM_ENC_KEY_LEN
#define IMB_GCM_ENC_KEY_LEN
Definition: intel-ipsec-mb.h:718
+
submit_burst_t
uint32_t(* submit_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t)
Definition: intel-ipsec-mb.h:785
+
aes_gcm_dec_192_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_gcm_enc_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
sha256_avx512
IMB_DLL_EXPORT void sha256_avx512(const void *data, const uint64_t length, void *digest)
+
snow3g_f8_8_buffer_multikey_t
void(* snow3g_f8_8_buffer_multikey_t)(const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[])
Definition: intel-ipsec-mb.h:962
+
aes_gcm_pre_t
void(* aes_gcm_pre_t)(const void *, struct gcm_key_data *)
Definition: intel-ipsec-mb.h:830
+
ghash_t
void(* ghash_t)(const struct gcm_key_data *, const void *, const uint64_t, void *, const uint64_t)
Definition: intel-ipsec-mb.h:850
+
aes_xcbc_expand_key_avx512
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx512(const void *key, void *k1_exp, void *k2, void *k3)
+
aes_gcm_dec_256_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
sha224_one_block_avx512
IMB_DLL_EXPORT void sha224_one_block_avx512(const void *data, void *digest)
+
aes_gcm_enc_192_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
sha512_avx
IMB_DLL_EXPORT void sha512_avx(const void *data, const uint64_t length, void *digest)
+
aes_gcm_precomp_256_sse
IMB_DLL_EXPORT void aes_gcm_precomp_256_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
aes_gcm_pre_256_sse
IMB_DLL_EXPORT void aes_gcm_pre_256_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
chacha_poly_finalize_t
void(* chacha_poly_finalize_t)(struct chacha20_poly1305_context_data *, void *, const uint64_t)
Definition: intel-ipsec-mb.h:848
+
aes_keyexp_192_avx512
IMB_DLL_EXPORT void aes_keyexp_192_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
IMB_KEY_SIZE_BYTES
IMB_KEY_SIZE_BYTES
Definition: intel-ipsec-mb.h:442
+
IMB_KEY_192_BYTES
@ IMB_KEY_192_BYTES
Definition: intel-ipsec-mb.h:444
+
IMB_KEY_256_BYTES
@ IMB_KEY_256_BYTES
Definition: intel-ipsec-mb.h:445
+
IMB_KEY_128_BYTES
@ IMB_KEY_128_BYTES
Definition: intel-ipsec-mb.h:443
+
aes_keyexp_256_enc_avx
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx(const void *key, void *enc_exp_keys)
+
aes_gcm_dec_128_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
free_mb_mgr
IMB_DLL_EXPORT void free_mb_mgr(IMB_MGR *ptr)
Frees memory allocated previously by alloc_mb_mgr()
+
get_completed_job_sse
IMB_DLL_EXPORT IMB_JOB * get_completed_job_sse(IMB_MGR *state)
Get next completed job.
+
aes_keyexp_128_enc_sse
IMB_DLL_EXPORT void aes_keyexp_128_enc_sse(const void *key, void *enc_exp_keys)
+
snow3g_f8_1_buffer_t
void(* snow3g_f8_1_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:927
+
aes_gcm_precomp_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
init_mb_mgr_sse
IMB_DLL_EXPORT void init_mb_mgr_sse(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
init_mb_mgr_auto
IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch)
Automatically initialize most performant Multi-buffer manager based on CPU features.
+
snow3g_f8_iv_gen
IMB_DLL_EXPORT int snow3g_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of SNOW3G F8 Initialization Vector.
+
sha256_one_block_avx
IMB_DLL_EXPORT void sha256_one_block_avx(const void *data, void *digest)
+
sha224_sse
IMB_DLL_EXPORT void sha224_sse(const void *data, const uint64_t length, void *digest)
+
aes_gcm_pre_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
zuc_eea3_n_buffer_t
void(* zuc_eea3_n_buffer_t)(const void *const *, const void *const *, const void *const *, void **, const uint32_t *, const uint32_t)
Definition: intel-ipsec-mb.h:860
+
aes_gmac_init_t
void(* aes_gmac_init_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
Definition: intel-ipsec-mb.h:832
+
aes_keyexp_128_sse
IMB_DLL_EXPORT void aes_keyexp_128_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
get_next_job_avx
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx(IMB_MGR *state)
Get next available job.
+
aes_gcm_enc_256_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
submit_job_nocheck_sse
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_sse(IMB_MGR *state)
Submit job for processing without validating.
+
sha1_sse
IMB_DLL_EXPORT void sha1_sse(const void *data, const uint64_t length, void *digest)
+
aes_keyexp_256_avx512
IMB_DLL_EXPORT void aes_keyexp_256_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
kasumi_f8_iv_gen
IMB_DLL_EXPORT int kasumi_f8_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of KASUMI F8 Initialization Vector.
+
sha384_avx2
IMB_DLL_EXPORT void sha384_avx2(const void *data, const uint64_t length, void *digest)
+
sha512_sse
IMB_DLL_EXPORT void sha512_sse(const void *data, const uint64_t length, void *digest)
+
sha384_one_block_avx512
IMB_DLL_EXPORT void sha384_one_block_avx512(const void *data, void *digest)
+
imb_set_pointers_mb_mgr
IMB_DLL_EXPORT IMB_MGR * imb_set_pointers_mb_mgr(void *ptr, const uint64_t flags, const unsigned reset_mgr)
Initializes IMB_MGR pointers to out-of-order managers with use of externally allocated memory.
+
aes_keyexp_192_avx
IMB_DLL_EXPORT void aes_keyexp_192_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
IMB_CIPHER_MODE
IMB_CIPHER_MODE
Definition: intel-ipsec-mb.h:355
+
IMB_CIPHER_DES
@ IMB_CIPHER_DES
Definition: intel-ipsec-mb.h:362
+
IMB_CIPHER_NULL
@ IMB_CIPHER_NULL
Definition: intel-ipsec-mb.h:358
+
IMB_CIPHER_CBC
@ IMB_CIPHER_CBC
Definition: intel-ipsec-mb.h:356
+
IMB_CIPHER_GCM
@ IMB_CIPHER_GCM
Definition: intel-ipsec-mb.h:360
+
IMB_CIPHER_ECB
@ IMB_CIPHER_ECB
Definition: intel-ipsec-mb.h:367
+
IMB_CIPHER_ZUC_EEA3
@ IMB_CIPHER_ZUC_EEA3
Definition: intel-ipsec-mb.h:369
+
IMB_CIPHER_CHACHA20
@ IMB_CIPHER_CHACHA20
Definition: intel-ipsec-mb.h:373
+
IMB_CIPHER_KASUMI_UEA1_BITLEN
@ IMB_CIPHER_KASUMI_UEA1_BITLEN
Definition: intel-ipsec-mb.h:371
+
IMB_CIPHER_CHACHA20_POLY1305_SGL
@ IMB_CIPHER_CHACHA20_POLY1305_SGL
Definition: intel-ipsec-mb.h:375
+
IMB_CIPHER_CUSTOM
@ IMB_CIPHER_CUSTOM
Definition: intel-ipsec-mb.h:361
+
IMB_CIPHER_GCM_SGL
@ IMB_CIPHER_GCM_SGL
Definition: intel-ipsec-mb.h:378
+
IMB_CIPHER_CNTR
@ IMB_CIPHER_CNTR
Definition: intel-ipsec-mb.h:357
+
IMB_CIPHER_DES3
@ IMB_CIPHER_DES3
Definition: intel-ipsec-mb.h:365
+
IMB_CIPHER_CNTR_BITLEN
@ IMB_CIPHER_CNTR_BITLEN
Definition: intel-ipsec-mb.h:368
+
IMB_CIPHER_CBCS_1_9
@ IMB_CIPHER_CBCS_1_9
Definition: intel-ipsec-mb.h:372
+
IMB_CIPHER_NUM
@ IMB_CIPHER_NUM
Definition: intel-ipsec-mb.h:379
+
IMB_CIPHER_CCM
@ IMB_CIPHER_CCM
Definition: intel-ipsec-mb.h:364
+
IMB_CIPHER_SNOW3G_UEA2_BITLEN
@ IMB_CIPHER_SNOW3G_UEA2_BITLEN
Definition: intel-ipsec-mb.h:370
+
IMB_CIPHER_PON_AES_CNTR
@ IMB_CIPHER_PON_AES_CNTR
Definition: intel-ipsec-mb.h:366
+
IMB_CIPHER_CHACHA20_POLY1305
@ IMB_CIPHER_CHACHA20_POLY1305
Definition: intel-ipsec-mb.h:374
+
IMB_CIPHER_SNOW_V_AEAD
@ IMB_CIPHER_SNOW_V_AEAD
Definition: intel-ipsec-mb.h:377
+
IMB_CIPHER_DOCSIS_SEC_BPI
@ IMB_CIPHER_DOCSIS_SEC_BPI
Definition: intel-ipsec-mb.h:359
+
IMB_CIPHER_DOCSIS_DES
@ IMB_CIPHER_DOCSIS_DES
Definition: intel-ipsec-mb.h:363
+
IMB_CIPHER_SNOW_V
@ IMB_CIPHER_SNOW_V
Definition: intel-ipsec-mb.h:376
+
aes_gcm_pre_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_128_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
aes_gcm_init_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
snow3g_f8_n_buffer_multikey_t
void(* snow3g_f8_n_buffer_multikey_t)(const snow3g_key_schedule_t *const [], const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
Definition: intel-ipsec-mb.h:973
+
aes_gcm_enc_128_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
get_next_job_avx512
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx512(IMB_MGR *state)
Get next available job.
+
submit_job_t
IMB_JOB *(* submit_job_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:781
+
chacha_poly_enc_dec_update_t
void(* chacha_poly_enc_dec_update_t)(const void *, struct chacha20_poly1305_context_data *, void *, const void *, const uint64_t)
Definition: intel-ipsec-mb.h:845
+
aes_gcm_dec_128_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
imb_get_version_str
IMB_DLL_EXPORT const char * imb_get_version_str(void)
Get library version in string format.
+
aes_gcm_enc_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
aes_keyexp_192_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx2(const void *key, void *enc_exp_keys)
+
aes_gcm_enc_256_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
aes_gcm_precomp_128_sse
IMB_DLL_EXPORT void aes_gcm_precomp_128_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
aes_gcm_enc_192_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
zuc_eea3_iv_gen
IMB_DLL_EXPORT int zuc_eea3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of ZUC-EEA3 Initialization Vector.
+
get_next_job_avx2
IMB_DLL_EXPORT IMB_JOB * get_next_job_avx2(IMB_MGR *state)
Get next available job.
+
IMB_CHAIN_ORDER
IMB_CHAIN_ORDER
Definition: intel-ipsec-mb.h:437
+
IMB_ORDER_HASH_CIPHER
@ IMB_ORDER_HASH_CIPHER
Definition: intel-ipsec-mb.h:439
+
IMB_ORDER_CIPHER_HASH
@ IMB_ORDER_CIPHER_HASH
Definition: intel-ipsec-mb.h:438
+
chacha_poly_init_t
void(* chacha_poly_init_t)(const void *, struct chacha20_poly1305_context_data *, const void *, const void *, const uint64_t)
Definition: intel-ipsec-mb.h:842
+
sha256_sse
IMB_DLL_EXPORT void sha256_sse(const void *data, const uint64_t length, void *digest)
+
aes_cmac_subkey_gen_sse
IMB_DLL_EXPORT void aes_cmac_subkey_gen_sse(const void *key_exp, void *key1, void *key2)
+
snow3g_f8_2_buffer_t
void(* snow3g_f8_2_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:935
+
submit_hash_burst_t
uint32_t(* submit_hash_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_HASH_ALG hash)
Definition: intel-ipsec-mb.h:793
+
aes_gcm_enc_192_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
aes_keyexp_128_avx2
IMB_DLL_EXPORT void aes_keyexp_128_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_keyexp_256_sse
IMB_DLL_EXPORT void aes_keyexp_256_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_enc_192_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
kasumi_f8_n_buffer_t
void(* kasumi_f8_n_buffer_t)(const kasumi_key_sched_t *, const uint64_t *, const void *const *, void **, const uint32_t *, const uint32_t)
Definition: intel-ipsec-mb.h:901
+
queue_size_avx512
IMB_DLL_EXPORT uint32_t queue_size_avx512(IMB_MGR *state)
Get number of jobs queued to be processed.
+
sha224_avx512
IMB_DLL_EXPORT void sha224_avx512(const void *data, const uint64_t length, void *digest)
+
aes_gcm_enc_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Encryption.
+
sha1_one_block_avx
IMB_DLL_EXPORT void sha1_one_block_avx(const void *data, void *digest)
+
aes_keyexp_192_enc_avx
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx(const void *key, void *enc_exp_keys)
+
kasumi_f8_3_buffer_t
void(* kasumi_f8_3_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:886
+
aes_gcm_enc_192_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-192 encryption message.
+
crc32_fn_t
uint32_t(* crc32_fn_t)(const void *, const uint64_t)
Definition: intel-ipsec-mb.h:991
+
aes_gcm_dec_192_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
sha224_one_block_avx2
IMB_DLL_EXPORT void sha224_one_block_avx2(const void *data, void *digest)
+
IMB_GCM_BLOCK_LEN
#define IMB_GCM_BLOCK_LEN
Definition: intel-ipsec-mb.h:660
+
aes_cfb_128_one_avx
IMB_DLL_EXPORT void aes_cfb_128_one_avx(void *out, const void *in, const void *iv, const void *keys, uint64_t len)
AES-CFB-128 Encrypt/Decrypt up to one block.
+
get_next_job_t
IMB_JOB *(* get_next_job_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:780
+
submit_job_nocheck_avx512
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx512(IMB_MGR *state)
Submit job for processing without validating.
+
aes_keyexp_256_avx
IMB_DLL_EXPORT void aes_keyexp_256_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_init_192_sse
IMB_DLL_EXPORT void aes_gcm_init_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
aes_gcm_dec_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
aes_gcm_pre_128_sse
IMB_DLL_EXPORT void aes_gcm_pre_128_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-128 key data.
+
sha512_one_block_avx512
IMB_DLL_EXPORT void sha512_one_block_avx512(const void *data, void *digest)
+
aes_gcm_enc_192_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-192 encryption message.
+
des_keysched_t
int(* des_keysched_t)(uint64_t *, const void *)
Definition: intel-ipsec-mb.h:802
+
keyexp_t
void(* keyexp_t)(const void *, void *, void *)
Definition: intel-ipsec-mb.h:797
+
snow3g_f8_8_buffer_t
void(* snow3g_f8_8_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:948
+
aes_gcm_dec_192_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
imb_get_strerror
IMB_DLL_EXPORT const char * imb_get_strerror(int errnum)
API to get description for errnum.
+
aes_cmac_subkey_gen_avx
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx(const void *key_exp, void *key1, void *key2)
+
aes_gcm_init_var_iv_t
void(* aes_gcm_init_var_iv_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t, const uint8_t *, const uint64_t)
Definition: intel-ipsec-mb.h:819
+
aes_keyexp_128_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx512(const void *key, void *enc_exp_keys)
+
aes_cmac_subkey_gen_avx2
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx2(const void *key_exp, void *key1, void *key2)
+
get_completed_job_avx512
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx512(IMB_MGR *state)
Get next completed job.
+
sha224_one_block_sse
IMB_DLL_EXPORT void sha224_one_block_sse(const void *data, void *digest)
+
snow3g_f8_n_buffer_t
void(* snow3g_f8_n_buffer_t)(const snow3g_key_schedule_t *, const void *const [], const void *const [], void *[], const uint32_t[], const uint32_t)
Definition: intel-ipsec-mb.h:966
+
IMB_SGL_STATE
IMB_SGL_STATE
Definition: intel-ipsec-mb.h:448
+
IMB_SGL_UPDATE
@ IMB_SGL_UPDATE
Definition: intel-ipsec-mb.h:450
+
IMB_SGL_ALL
@ IMB_SGL_ALL
Definition: intel-ipsec-mb.h:452
+
IMB_SGL_COMPLETE
@ IMB_SGL_COMPLETE
Definition: intel-ipsec-mb.h:451
+
IMB_SGL_INIT
@ IMB_SGL_INIT
Definition: intel-ipsec-mb.h:449
+
aes_gcm_pre_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
aes_gcm_precomp_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
aes_cfb_128_one_avx2
IMB_DLL_EXPORT void aes_cfb_128_one_avx2(void *out, const void *in, const void *iv, const void *keys, uint64_t len)
AES-CFB-128 Encrypt/Decrypt up to one block.
+
init_mb_mgr_avx2
IMB_DLL_EXPORT void init_mb_mgr_avx2(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
aes_gcm_enc_dec_t
void(* aes_gcm_enc_dec_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, uint64_t, const uint8_t *, uint8_t const *, uint64_t, uint8_t *, uint64_t)
Definition: intel-ipsec-mb.h:805
+
imb_get_feature_flags
IMB_DLL_EXPORT uint64_t imb_get_feature_flags(void)
Retrieves the bitmask with the features supported by the library, without having to allocate/initiali...
+
aes_keyexp_192_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_192_enc_avx512(const void *key, void *enc_exp_keys)
+
submit_job_nocheck_avx2
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx2(IMB_MGR *state)
Submit job for processing without validating.
+
aes_gcm_precomp_192_sse
IMB_DLL_EXPORT void aes_gcm_precomp_192_sse(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
aes_gcm_dec_256_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
aes_cmac_subkey_gen_avx512
IMB_DLL_EXPORT void aes_cmac_subkey_gen_avx512(const void *key_exp, void *key1, void *key2)
+
IMB_GCM_KEY_SETS
#define IMB_GCM_KEY_SETS
Definition: intel-ipsec-mb.h:719
+
aes_keyexp_256_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx2(const void *key, void *enc_exp_keys)
+
sha1_one_block_avx512
IMB_DLL_EXPORT void sha1_one_block_avx512(const void *data, void *digest)
+
aes_gcm_init_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
submit_job_avx512
IMB_DLL_EXPORT IMB_JOB * submit_job_avx512(IMB_MGR *state)
Submit job for processing after validating.
+
aes_keyexp_192_sse
IMB_DLL_EXPORT void aes_keyexp_192_sse(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
sha224_one_block_avx
IMB_DLL_EXPORT void sha224_one_block_avx(const void *data, void *digest)
+
submit_job_sse
IMB_DLL_EXPORT IMB_JOB * submit_job_sse(IMB_MGR *state)
Submit job for processing after validating.
+
aes_gcm_dec_256_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
aes_gcm_dec_128_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-128 encryption message.
+
sha256_avx
IMB_DLL_EXPORT void sha256_avx(const void *data, const uint64_t length, void *digest)
+
imb_get_errno
IMB_DLL_EXPORT int imb_get_errno(IMB_MGR *mb_mgr)
API to get error status.
+
aes_gcm_init_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
md5_one_block_avx
IMB_DLL_EXPORT void md5_one_block_avx(const void *data, void *digest)
+
aes_gcm_enc_128_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
sha384_avx
IMB_DLL_EXPORT void sha384_avx(const void *data, const uint64_t length, void *digest)
+
aes_cfb_128_one_sse
IMB_DLL_EXPORT void aes_cfb_128_one_sse(void *out, const void *in, const void *iv, const void *keys, uint64_t len)
AES-CFB-128 Encrypt/Decrypt up to one block.
+
snow3g_f8_1_buffer_bit_t
void(* snow3g_f8_1_buffer_bit_t)(const snow3g_key_schedule_t *, const void *, const void *, void *, const uint32_t, const uint32_t)
Definition: intel-ipsec-mb.h:931
+
kasumi_init_f8_key_sched_t
int(* kasumi_init_f8_key_sched_t)(const void *, kasumi_key_sched_t *)
Definition: intel-ipsec-mb.h:912
+
sha512_one_block_avx
IMB_DLL_EXPORT void sha512_one_block_avx(const void *data, void *digest)
+
zuc_eea3_4_buffer_t
void(* zuc_eea3_4_buffer_t)(const void *const *, const void *const *, const void *const *, void **, const uint32_t *)
Definition: intel-ipsec-mb.h:856
+
aes_gcm_enc_256_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
sha512_avx512
IMB_DLL_EXPORT void sha512_avx512(const void *data, const uint64_t length, void *digest)
+
queue_size_avx
IMB_DLL_EXPORT uint32_t queue_size_avx(IMB_MGR *state)
Get number of jobs queued to be processed.
+
queue_size_sse
IMB_DLL_EXPORT uint32_t queue_size_sse(IMB_MGR *state)
Get number of jobs queued to be processed.
+
kasumi_f8_1_buffer_bit_t
void(* kasumi_f8_1_buffer_bit_t)(const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t, const uint32_t)
Definition: intel-ipsec-mb.h:876
+
kasumi_key_sched_t
struct kasumi_key_sched_s kasumi_key_sched_t
+
IMB_HASH_ALG
IMB_HASH_ALG
Definition: intel-ipsec-mb.h:387
+
IMB_AUTH_AES_CMAC_256
@ IMB_AUTH_AES_CMAC_256
Definition: intel-ipsec-mb.h:414
+
IMB_AUTH_AES_XCBC
@ IMB_AUTH_AES_XCBC
Definition: intel-ipsec-mb.h:393
+
IMB_AUTH_ZUC_EIA3_BITLEN
@ IMB_AUTH_ZUC_EIA3_BITLEN
Definition: intel-ipsec-mb.h:407
+
IMB_AUTH_AES_GMAC_128
@ IMB_AUTH_AES_GMAC_128
Definition: intel-ipsec-mb.h:411
+
IMB_AUTH_CRC16_X25
@ IMB_AUTH_CRC16_X25
Definition: intel-ipsec-mb.h:426
+
IMB_AUTH_NULL
@ IMB_AUTH_NULL
Definition: intel-ipsec-mb.h:395
+
IMB_AUTH_PON_CRC_BIP
@ IMB_AUTH_PON_CRC_BIP
Definition: intel-ipsec-mb.h:406
+
IMB_AUTH_SHA_224
@ IMB_AUTH_SHA_224
Definition: intel-ipsec-mb.h:401
+
IMB_AUTH_CRC24_LTE_A
@ IMB_AUTH_CRC24_LTE_A
Definition: intel-ipsec-mb.h:424
+
IMB_AUTH_CRC32_ETHERNET_FCS
@ IMB_AUTH_CRC32_ETHERNET_FCS
Definition: intel-ipsec-mb.h:421
+
IMB_AUTH_KASUMI_UIA1
@ IMB_AUTH_KASUMI_UIA1
Definition: intel-ipsec-mb.h:410
+
IMB_AUTH_HMAC_SHA_1
@ IMB_AUTH_HMAC_SHA_1
Definition: intel-ipsec-mb.h:388
+
IMB_AUTH_SHA_256
@ IMB_AUTH_SHA_256
Definition: intel-ipsec-mb.h:402
+
IMB_AUTH_SNOW3G_UIA2_BITLEN
@ IMB_AUTH_SNOW3G_UIA2_BITLEN
Definition: intel-ipsec-mb.h:409
+
IMB_AUTH_SNOW_V_AEAD
@ IMB_AUTH_SNOW_V_AEAD
Definition: intel-ipsec-mb.h:419
+
IMB_AUTH_CRC32_SCTP
@ IMB_AUTH_CRC32_SCTP
Definition: intel-ipsec-mb.h:422
+
IMB_AUTH_NUM
@ IMB_AUTH_NUM
Definition: intel-ipsec-mb.h:434
+
IMB_AUTH_HMAC_SHA_256
@ IMB_AUTH_HMAC_SHA_256
Definition: intel-ipsec-mb.h:390
+
IMB_AUTH_GHASH
@ IMB_AUTH_GHASH
Definition: intel-ipsec-mb.h:433
+
IMB_AUTH_CUSTOM
@ IMB_AUTH_CUSTOM
Definition: intel-ipsec-mb.h:397
+
IMB_AUTH_CHACHA20_POLY1305_SGL
@ IMB_AUTH_CHACHA20_POLY1305_SGL
Definition: intel-ipsec-mb.h:417
+
IMB_AUTH_CRC24_LTE_B
@ IMB_AUTH_CRC24_LTE_B
Definition: intel-ipsec-mb.h:425
+
IMB_AUTH_ZUC256_EIA3_BITLEN
@ IMB_AUTH_ZUC256_EIA3_BITLEN
Definition: intel-ipsec-mb.h:418
+
IMB_AUTH_AES_CCM
@ IMB_AUTH_AES_CCM
Definition: intel-ipsec-mb.h:398
+
IMB_AUTH_CRC6_IUUP_HEADER
@ IMB_AUTH_CRC6_IUUP_HEADER
Definition: intel-ipsec-mb.h:432
+
IMB_AUTH_POLY1305
@ IMB_AUTH_POLY1305
Definition: intel-ipsec-mb.h:415
+
IMB_AUTH_MD5
@ IMB_AUTH_MD5
Definition: intel-ipsec-mb.h:394
+
IMB_AUTH_SHA_512
@ IMB_AUTH_SHA_512
Definition: intel-ipsec-mb.h:404
+
IMB_AUTH_AES_GMAC
@ IMB_AUTH_AES_GMAC
Definition: intel-ipsec-mb.h:396
+
IMB_AUTH_CRC16_FP_DATA
@ IMB_AUTH_CRC16_FP_DATA
Definition: intel-ipsec-mb.h:427
+
IMB_AUTH_AES_CMAC_BITLEN
@ IMB_AUTH_AES_CMAC_BITLEN
Definition: intel-ipsec-mb.h:405
+
IMB_AUTH_CRC11_FP_HEADER
@ IMB_AUTH_CRC11_FP_HEADER
Definition: intel-ipsec-mb.h:428
+
IMB_AUTH_CHACHA20_POLY1305
@ IMB_AUTH_CHACHA20_POLY1305
Definition: intel-ipsec-mb.h:416
+
IMB_AUTH_CRC7_FP_HEADER
@ IMB_AUTH_CRC7_FP_HEADER
Definition: intel-ipsec-mb.h:431
+
IMB_AUTH_CRC10_IUUP_DATA
@ IMB_AUTH_CRC10_IUUP_DATA
Definition: intel-ipsec-mb.h:429
+
IMB_AUTH_HMAC_SHA_224
@ IMB_AUTH_HMAC_SHA_224
Definition: intel-ipsec-mb.h:389
+
IMB_AUTH_CRC32_WIMAX_OFDMA_DATA
@ IMB_AUTH_CRC32_WIMAX_OFDMA_DATA
Definition: intel-ipsec-mb.h:423
+
IMB_AUTH_HMAC_SHA_512
@ IMB_AUTH_HMAC_SHA_512
Definition: intel-ipsec-mb.h:392
+
IMB_AUTH_GCM_SGL
@ IMB_AUTH_GCM_SGL
Definition: intel-ipsec-mb.h:420
+
IMB_AUTH_SHA_1
@ IMB_AUTH_SHA_1
Definition: intel-ipsec-mb.h:400
+
IMB_AUTH_AES_GMAC_192
@ IMB_AUTH_AES_GMAC_192
Definition: intel-ipsec-mb.h:412
+
IMB_AUTH_DOCSIS_CRC32
@ IMB_AUTH_DOCSIS_CRC32
Definition: intel-ipsec-mb.h:408
+
IMB_AUTH_CRC8_WIMAX_OFDMA_HCS
@ IMB_AUTH_CRC8_WIMAX_OFDMA_HCS
Definition: intel-ipsec-mb.h:430
+
IMB_AUTH_AES_GMAC_256
@ IMB_AUTH_AES_GMAC_256
Definition: intel-ipsec-mb.h:413
+
IMB_AUTH_HMAC_SHA_384
@ IMB_AUTH_HMAC_SHA_384
Definition: intel-ipsec-mb.h:391
+
IMB_AUTH_AES_CMAC
@ IMB_AUTH_AES_CMAC
Definition: intel-ipsec-mb.h:399
+
IMB_AUTH_SHA_384
@ IMB_AUTH_SHA_384
Definition: intel-ipsec-mb.h:403
+
aes_keyexp_128_enc_avx2
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx2(const void *key, void *enc_exp_keys)
+
aes_gcm_enc_256_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
submit_job_avx2
IMB_DLL_EXPORT IMB_JOB * submit_job_avx2(IMB_MGR *state)
Submit job for processing after validating.
+
kasumi_f8_1_buffer_t
void(* kasumi_f8_1_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:873
+
queue_size_avx2
IMB_DLL_EXPORT uint32_t queue_size_avx2(IMB_MGR *state)
Get number of jobs queued to be processed.
+
kasumi_f9_1_buffer_t
void(* kasumi_f9_1_buffer_t)(const kasumi_key_sched_t *, const void *, const uint32_t, void *)
Definition: intel-ipsec-mb.h:909
+
kasumi_init_f9_key_sched_t
int(* kasumi_init_f9_key_sched_t)(const void *, kasumi_key_sched_t *)
Definition: intel-ipsec-mb.h:914
+
md5_one_block_avx2
IMB_DLL_EXPORT void md5_one_block_avx2(const void *data, void *digest)
+
aes_gcm_enc_dec_finalize_t
void(* aes_gcm_enc_dec_finalize_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint64_t)
Definition: intel-ipsec-mb.h:826
+
aes_gcm_dec_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
aes_gcm_init_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
aes_gcm_dec_128_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
sha384_avx512
IMB_DLL_EXPORT void sha384_avx512(const void *data, const uint64_t length, void *digest)
+
snow3g_f9_iv_gen
IMB_DLL_EXPORT int snow3g_f9_iv_gen(const uint32_t count, const uint32_t fresh, const uint8_t dir, void *iv_ptr)
Generation of SNOW3G F9 Initialization Vector.
+
sha1_avx
IMB_DLL_EXPORT void sha1_avx(const void *data, const uint64_t length, void *digest)
+
submit_job_avx
IMB_DLL_EXPORT IMB_JOB * submit_job_avx(IMB_MGR *state)
Submit job for processing after validating.
+
zuc_eia3_1_buffer_t
void(* zuc_eia3_1_buffer_t)(const void *, const void *, const void *, const uint32_t, uint32_t *)
Definition: intel-ipsec-mb.h:864
+
flush_job_sse
IMB_DLL_EXPORT IMB_JOB * flush_job_sse(IMB_MGR *state)
Force processing until next job in queue is completed.
+
IMB_DLL_EXPORT
#define IMB_DLL_EXPORT
Definition: intel-ipsec-mb.h:86
+
queue_size_t
uint32_t(* queue_size_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:784
+
init_mb_mgr_avx512
IMB_DLL_EXPORT void init_mb_mgr_avx512(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
des_key_schedule
IMB_DLL_EXPORT int des_key_schedule(uint64_t *ks, const void *key)
DES key schedule set up.
+
aes_gcm_init_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_init_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-256 Encryption.
+
sha1_one_block_sse
IMB_DLL_EXPORT void sha1_one_block_sse(const void *data, void *digest)
+
aes_gcm_enc_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_192_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
aes_gcm_dec_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
aes_keyexp_128_avx512
IMB_DLL_EXPORT void aes_keyexp_128_avx512(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
get_completed_job_avx
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx(IMB_MGR *state)
Get next completed job.
+
md5_one_block_sse
IMB_DLL_EXPORT void md5_one_block_sse(const void *data, void *digest)
+
aes_keyexp_256_enc_avx512
IMB_DLL_EXPORT void aes_keyexp_256_enc_avx512(const void *key, void *enc_exp_keys)
+
IMB_ERR
IMB_ERR
Definition: intel-ipsec-mb.h:178
+
IMB_ERR_NULL_JOB
@ IMB_ERR_NULL_JOB
Definition: intel-ipsec-mb.h:226
+
IMB_ERR_JOB_NULL_SGL_CTX
@ IMB_ERR_JOB_NULL_SGL_CTX
Definition: intel-ipsec-mb.h:198
+
IMB_ERR_AAD_LEN
@ IMB_ERR_AAD_LEN
Definition: intel-ipsec-mb.h:213
+
IMB_ERR_NULL_CTX
@ IMB_ERR_NULL_CTX
Definition: intel-ipsec-mb.h:216
+
IMB_ERR_JOB_NULL_IV
@ IMB_ERR_JOB_NULL_IV
Definition: intel-ipsec-mb.h:184
+
IMB_ERR_JOB_NULL_HMAC_IPAD
@ IMB_ERR_JOB_NULL_HMAC_IPAD
Definition: intel-ipsec-mb.h:219
+
IMB_ERR_NULL_DST
@ IMB_ERR_NULL_DST
Definition: intel-ipsec-mb.h:202
+
IMB_ERR_JOB_AUTH_TAG_LEN
@ IMB_ERR_JOB_AUTH_TAG_LEN
Definition: intel-ipsec-mb.h:191
+
IMB_ERR_NULL_AUTH
@ IMB_ERR_NULL_AUTH
Definition: intel-ipsec-mb.h:206
+
IMB_ERR_JOB_SRC_OFFSET
@ IMB_ERR_JOB_SRC_OFFSET
Definition: intel-ipsec-mb.h:193
+
IMB_ERR_SRC_OFFSET
@ IMB_ERR_SRC_OFFSET
Definition: intel-ipsec-mb.h:214
+
IMB_ERR_JOB_NULL_KEY
@ IMB_ERR_JOB_NULL_KEY
Definition: intel-ipsec-mb.h:183
+
IMB_ERR_NULL_SRC
@ IMB_ERR_NULL_SRC
Definition: intel-ipsec-mb.h:201
+
IMB_ERR_JOB_AAD_LEN
@ IMB_ERR_JOB_AAD_LEN
Definition: intel-ipsec-mb.h:192
+
IMB_ERR_JOB_IV_LEN
@ IMB_ERR_JOB_IV_LEN
Definition: intel-ipsec-mb.h:189
+
IMB_ERR_NULL_EXP_KEY
@ IMB_ERR_NULL_EXP_KEY
Definition: intel-ipsec-mb.h:204
+
IMB_ERR_JOB_NULL_AAD
@ IMB_ERR_JOB_NULL_AAD
Definition: intel-ipsec-mb.h:186
+
IMB_ERR_MIN
@ IMB_ERR_MIN
Definition: intel-ipsec-mb.h:179
+
IMB_ERR_NO_AESNI_EMU
@ IMB_ERR_NO_AESNI_EMU
Definition: intel-ipsec-mb.h:217
+
IMB_ERR_JOB_NULL_SRC
@ IMB_ERR_JOB_NULL_SRC
Definition: intel-ipsec-mb.h:181
+
IMB_ERR_JOB_NULL_AUTH
@ IMB_ERR_JOB_NULL_AUTH
Definition: intel-ipsec-mb.h:185
+
IMB_ERR_JOB_NULL_DST
@ IMB_ERR_JOB_NULL_DST
Definition: intel-ipsec-mb.h:182
+
IMB_ERR_CIPH_MODE
@ IMB_ERR_CIPH_MODE
Definition: intel-ipsec-mb.h:195
+
IMB_ERR_NULL_KEY
@ IMB_ERR_NULL_KEY
Definition: intel-ipsec-mb.h:203
+
IMB_ERR_JOB_NULL_XCBC_K1_EXP
@ IMB_ERR_JOB_NULL_XCBC_K1_EXP
Definition: intel-ipsec-mb.h:220
+
IMB_ERR_JOB_AUTH_LEN
@ IMB_ERR_JOB_AUTH_LEN
Definition: intel-ipsec-mb.h:188
+
IMB_ERR_AUTH_TAG_LEN
@ IMB_ERR_AUTH_TAG_LEN
Definition: intel-ipsec-mb.h:212
+
IMB_ERR_JOB_NULL_GHASH_INIT_TAG
@ IMB_ERR_JOB_NULL_GHASH_INIT_TAG
Definition: intel-ipsec-mb.h:224
+
IMB_ERR_HASH_ALGO
@ IMB_ERR_HASH_ALGO
Definition: intel-ipsec-mb.h:196
+
IMB_ERR_KEY_LEN
@ IMB_ERR_KEY_LEN
Definition: intel-ipsec-mb.h:211
+
IMB_ERR_JOB_NULL_AUTH_KEY
@ IMB_ERR_JOB_NULL_AUTH_KEY
Definition: intel-ipsec-mb.h:197
+
IMB_ERR_JOB_NULL_NEXT_IV
@ IMB_ERR_JOB_NULL_NEXT_IV
Definition: intel-ipsec-mb.h:199
+
IMB_ERR_NULL_MBMGR
@ IMB_ERR_NULL_MBMGR
Definition: intel-ipsec-mb.h:180
+
IMB_ERR_NULL_AUTH_KEY
@ IMB_ERR_NULL_AUTH_KEY
Definition: intel-ipsec-mb.h:215
+
IMB_ERR_IV_LEN
@ IMB_ERR_IV_LEN
Definition: intel-ipsec-mb.h:210
+
IMB_ERR_CIPH_LEN
@ IMB_ERR_CIPH_LEN
Definition: intel-ipsec-mb.h:208
+
IMB_ERR_MISSING_CPUFLAGS_INIT_MGR
@ IMB_ERR_MISSING_CPUFLAGS_INIT_MGR
Definition: intel-ipsec-mb.h:225
+
IMB_ERR_MAX
@ IMB_ERR_MAX
Definition: intel-ipsec-mb.h:228
+
IMB_ERR_JOB_CHAIN_ORDER
@ IMB_ERR_JOB_CHAIN_ORDER
Definition: intel-ipsec-mb.h:194
+
IMB_ERR_JOB_NULL_HMAC_OPAD
@ IMB_ERR_JOB_NULL_HMAC_OPAD
Definition: intel-ipsec-mb.h:218
+
IMB_ERR_JOB_CIPH_DIR
@ IMB_ERR_JOB_CIPH_DIR
Definition: intel-ipsec-mb.h:223
+
IMB_ERR_JOB_PON_PLI
@ IMB_ERR_JOB_PON_PLI
Definition: intel-ipsec-mb.h:200
+
IMB_ERR_JOB_NULL_XCBC_K2
@ IMB_ERR_JOB_NULL_XCBC_K2
Definition: intel-ipsec-mb.h:221
+
IMB_ERR_NULL_AAD
@ IMB_ERR_NULL_AAD
Definition: intel-ipsec-mb.h:207
+
IMB_ERR_JOB_KEY_LEN
@ IMB_ERR_JOB_KEY_LEN
Definition: intel-ipsec-mb.h:190
+
IMB_ERR_JOB_CIPH_LEN
@ IMB_ERR_JOB_CIPH_LEN
Definition: intel-ipsec-mb.h:187
+
IMB_ERR_JOB_NULL_XCBC_K3
@ IMB_ERR_JOB_NULL_XCBC_K3
Definition: intel-ipsec-mb.h:222
+
IMB_ERR_NULL_IV
@ IMB_ERR_NULL_IV
Definition: intel-ipsec-mb.h:205
+
IMB_ERR_AUTH_LEN
@ IMB_ERR_AUTH_LEN
Definition: intel-ipsec-mb.h:209
+
aes_gcm_enc_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
hec_32_t
uint32_t(* hec_32_t)(const uint8_t *)
Definition: intel-ipsec-mb.h:988
+
sha384_one_block_sse
IMB_DLL_EXPORT void sha384_one_block_sse(const void *data, void *digest)
+
sha512_one_block_sse
IMB_DLL_EXPORT void sha512_one_block_sse(const void *data, void *digest)
+
aes_gcm_dec_192_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-192 encryption message.
+
hec_64_t
uint64_t(* hec_64_t)(const uint8_t *)
Definition: intel-ipsec-mb.h:989
+
aes_gcm_pre_256_avx_gen2
IMB_DLL_EXPORT void aes_gcm_pre_256_avx_gen2(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-256 key data.
+
cmac_subkey_gen_t
void(* cmac_subkey_gen_t)(const void *, void *, void *)
Definition: intel-ipsec-mb.h:798
+
aes_gcm_dec_192_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_gcm_enc_128_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
hash_one_block_t
void(* hash_one_block_t)(const void *, void *)
Definition: intel-ipsec-mb.h:799
+
aes_gcm_precomp_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_192_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-192 HashKey constants.
+
IMB_MAX_JOBS
#define IMB_MAX_JOBS
Definition: intel-ipsec-mb.h:1041
+
aes_gcm_dec_256_update_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Decrypt a block of a AES-GCM-256 encryption message.
+
aes_gcm_precomp_t
void(* aes_gcm_precomp_t)(struct gcm_key_data *)
Definition: intel-ipsec-mb.h:829
+
aes_gcm_pre_192_avx_gen4
IMB_DLL_EXPORT void aes_gcm_pre_192_avx_gen4(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
aes_gcm_enc_256_update_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_update_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
alloc_mb_mgr
IMB_DLL_EXPORT IMB_MGR * alloc_mb_mgr(uint64_t flags)
Allocates memory for multi-buffer manager instance.
+
aes_gmac_update_t
void(* aes_gmac_update_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, const uint64_t)
Definition: intel-ipsec-mb.h:835
+
aes_keyexp_192_avx2
IMB_DLL_EXPORT void aes_keyexp_192_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
xcbc_keyexp_t
void(* xcbc_keyexp_t)(const void *, void *, void *, void *)
Definition: intel-ipsec-mb.h:801
+
aes_gcm_enc_128_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
aes_cfb_128_one_avx512
IMB_DLL_EXPORT void aes_cfb_128_one_avx512(void *out, const void *in, const void *iv, const void *keys, uint64_t len)
AES-CFB-128 Encrypt/Decrypt up to one block.
+
flush_job_avx
IMB_DLL_EXPORT IMB_JOB * flush_job_avx(IMB_MGR *state)
Force processing until next job in queue is completed.
+
IMB_CIPHER_DIRECTION
IMB_CIPHER_DIRECTION
Definition: intel-ipsec-mb.h:382
+
IMB_DIR_DECRYPT
@ IMB_DIR_DECRYPT
Definition: intel-ipsec-mb.h:384
+
IMB_DIR_ENCRYPT
@ IMB_DIR_ENCRYPT
Definition: intel-ipsec-mb.h:383
+
aes_gcm_precomp_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_256_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-256 HashKey constants.
+
aes_gcm_pre_192_sse
IMB_DLL_EXPORT void aes_gcm_pre_192_sse(const void *key, struct gcm_key_data *key_data)
Pre-processes AES-GCM-192 key data.
+
sha1_avx2
IMB_DLL_EXPORT void sha1_avx2(const void *data, const uint64_t length, void *digest)
+
aes_gcm_dec_256_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_256_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
kasumi_f9_1_buffer_user_t
void(* kasumi_f9_1_buffer_user_t)(const kasumi_key_sched_t *, const uint64_t, const void *, const uint32_t, void *, const uint32_t)
Definition: intel-ipsec-mb.h:905
+
aes_gcm_dec_128_finalize_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_128_finalize_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-128 encryption message.
+
aes_gcm_init_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_init_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-192 Encryption.
+
sha256_one_block_avx512
IMB_DLL_EXPORT void sha256_one_block_avx512(const void *data, void *digest)
+
aes_xcbc_expand_key_avx2
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx2(const void *key, void *k1_exp, void *k2, void *k3)
+
submit_cipher_burst_t
uint32_t(* submit_cipher_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size)
Definition: intel-ipsec-mb.h:787
+
aes_gcm_enc_dec_iv_t
void(* aes_gcm_enc_dec_iv_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, uint8_t const *, const uint64_t, const uint8_t *, uint8_t const *, const uint64_t, uint8_t *, const uint64_t, const uint64_t)
Definition: intel-ipsec-mb.h:810
+
flush_job_avx512
IMB_DLL_EXPORT IMB_JOB * flush_job_avx512(IMB_MGR *state)
Force processing until next job in queue is completed.
+
submit_job_nocheck_avx
IMB_DLL_EXPORT IMB_JOB * submit_job_nocheck_avx(IMB_MGR *state)
Submit job for processing without validating.
+
aes_keyexp_128_enc_avx
IMB_DLL_EXPORT void aes_keyexp_128_enc_avx(const void *key, void *enc_exp_keys)
+
md5_one_block_avx512
IMB_DLL_EXPORT void md5_one_block_avx512(const void *data, void *digest)
+
imb_get_mb_mgr_size
IMB_DLL_EXPORT size_t imb_get_mb_mgr_size(void)
Calculates necessary memory size for IMB_MGR.
+
kasumi_f9_iv_gen
IMB_DLL_EXPORT int kasumi_f9_iv_gen(const uint32_t count, const uint32_t fresh, void *iv_ptr)
Generation of KASUMI F9 Initialization Vector.
+
get_completed_job_t
IMB_JOB *(* get_completed_job_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:782
+
aes_gcm_init_128_sse
IMB_DLL_EXPORT void aes_gcm_init_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len)
Initialize a gcm_context_data structure to prepare for AES-GCM-128 Encryption.
+
aes_gcm_dec_192_sse
IMB_DLL_EXPORT void aes_gcm_dec_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
kasumi_f8_4_buffer_t
void(* kasumi_f8_4_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const uint64_t, const uint64_t, const void *, void *, const void *, void *, const void *, void *, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:893
+
IMB_ARCH
IMB_ARCH
Definition: intel-ipsec-mb.h:119
+
IMB_ARCH_AVX512
@ IMB_ARCH_AVX512
Definition: intel-ipsec-mb.h:125
+
IMB_ARCH_NONE
@ IMB_ARCH_NONE
Definition: intel-ipsec-mb.h:120
+
IMB_ARCH_SSE
@ IMB_ARCH_SSE
Definition: intel-ipsec-mb.h:122
+
IMB_ARCH_NOAESNI
@ IMB_ARCH_NOAESNI
Definition: intel-ipsec-mb.h:121
+
IMB_ARCH_AVX2
@ IMB_ARCH_AVX2
Definition: intel-ipsec-mb.h:124
+
IMB_ARCH_AVX
@ IMB_ARCH_AVX
Definition: intel-ipsec-mb.h:123
+
IMB_ARCH_NUM
@ IMB_ARCH_NUM
Definition: intel-ipsec-mb.h:126
+
aes_gcm_dec_192_avx_gen2
IMB_DLL_EXPORT void aes_gcm_dec_192_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Decryption.
+
sha224_avx2
IMB_DLL_EXPORT void sha224_avx2(const void *data, const uint64_t length, void *digest)
+
sha384_sse
IMB_DLL_EXPORT void sha384_sse(const void *data, const uint64_t length, void *digest)
+
kasumi_key_sched_size_t
size_t(* kasumi_key_sched_size_t)(void)
Definition: intel-ipsec-mb.h:916
+
snow3g_key_sched_size_t
size_t(* snow3g_key_sched_size_t)(void)
Definition: intel-ipsec-mb.h:986
+
get_completed_job_avx2
IMB_DLL_EXPORT IMB_JOB * get_completed_job_avx2(IMB_MGR *state)
Get next completed job.
+
aes_gcm_enc_256_update_avx_gen4
IMB_DLL_EXPORT void aes_gcm_enc_256_update_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-256 encryption message.
+
sha1_avx512
IMB_DLL_EXPORT void sha1_avx512(const void *data, const uint64_t length, void *digest)
+
sha512_one_block_avx2
IMB_DLL_EXPORT void sha512_one_block_avx2(const void *data, void *digest)
+
zuc_eia3_n_buffer_t
void(* zuc_eia3_n_buffer_t)(const void *const *, const void *const *, const void *const *, const uint32_t *, uint32_t **, const uint32_t)
Definition: intel-ipsec-mb.h:867
+
aes_gcm_dec_128_sse
IMB_DLL_EXPORT void aes_gcm_dec_128_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
aes_gcm_precomp_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen2(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
aes_gcm_enc_192_sse
IMB_DLL_EXPORT void aes_gcm_enc_192_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-192 Encryption.
+
aes_gcm_init_t
void(* aes_gcm_init_t)(const struct gcm_key_data *, struct gcm_context_data *, const uint8_t *, uint8_t const *, uint64_t)
Definition: intel-ipsec-mb.h:816
+
aes_gcm_dec_256_finalize_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-256 encryption message.
+
snow3g_init_key_sched_t
int(* snow3g_init_key_sched_t)(const void *, snow3g_key_schedule_t *)
Definition: intel-ipsec-mb.h:983
+
snow3g_f8_4_buffer_t
void(* snow3g_f8_4_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const void *, const void *, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:940
+
aes_gcm_dec_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_128_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Decryption.
+
flush_job_avx2
IMB_DLL_EXPORT IMB_JOB * flush_job_avx2(IMB_MGR *state)
Force processing until next job in queue is completed.
+
aes_gcm_enc_128_update_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_update_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, const uint8_t *in, uint64_t len)
Encrypt a block of a AES-GCM-128 encryption message.
+
zuc_eia3_iv_gen
IMB_DLL_EXPORT int zuc_eia3_iv_gen(const uint32_t count, const uint8_t bearer, const uint8_t dir, void *iv_ptr)
Generation of ZUC-EIA3 Initialization Vector.
+
snow3g_key_schedule_t
struct snow3g_key_schedule_s snow3g_key_schedule_t
+
sha256_one_block_sse
IMB_DLL_EXPORT void sha256_one_block_sse(const void *data, void *digest)
+
sha384_one_block_avx2
IMB_DLL_EXPORT void sha384_one_block_avx2(const void *data, void *digest)
+
flush_job_t
IMB_JOB *(* flush_job_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:783
+
snow3g_f9_1_buffer_t
void(* snow3g_f9_1_buffer_t)(const snow3g_key_schedule_t *, const void *, const void *, const uint64_t, void *)
Definition: intel-ipsec-mb.h:979
+
aes_cfb_t
void(* aes_cfb_t)(void *, const void *, const void *, const void *, uint64_t)
Definition: intel-ipsec-mb.h:803
+
sha256_avx2
IMB_DLL_EXPORT void sha256_avx2(const void *data, const uint64_t length, void *digest)
+
aes_keyexp_128_avx
IMB_DLL_EXPORT void aes_keyexp_128_avx(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
imb_clear_mem
IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size)
Force clearing/zeroing of memory.
+
aes_gcm_precomp_128_avx_gen4
IMB_DLL_EXPORT void aes_gcm_precomp_128_avx_gen4(struct gcm_key_data *key_data)
Precomputation of AES-GCM-128 HashKey constants.
+
imb_get_version
IMB_DLL_EXPORT unsigned imb_get_version(void)
Get library version in numerical format.
+
init_mb_mgr_avx
IMB_DLL_EXPORT void init_mb_mgr_avx(IMB_MGR *state)
Initialize Multi-Buffer Manager structure.
+
sha384_one_block_avx
IMB_DLL_EXPORT void sha384_one_block_avx(const void *data, void *digest)
+
aes_xcbc_expand_key_sse
IMB_DLL_EXPORT void aes_xcbc_expand_key_sse(const void *key, void *k1_exp, void *k2, void *k3)
+
init_mb_mgr_t
void(* init_mb_mgr_t)(struct IMB_MGR *)
Definition: intel-ipsec-mb.h:779
+
sha224_avx
IMB_DLL_EXPORT void sha224_avx(const void *data, const uint64_t length, void *digest)
+
sha1_one_block_avx2
IMB_DLL_EXPORT void sha1_one_block_avx2(const void *data, void *digest)
+
kasumi_f8_2_buffer_t
void(* kasumi_f8_2_buffer_t)(const kasumi_key_sched_t *, const uint64_t, const uint64_t, const void *, void *, const uint32_t, const void *, void *, const uint32_t)
Definition: intel-ipsec-mb.h:880
+
aes_keyexp_256_avx2
IMB_DLL_EXPORT void aes_keyexp_256_avx2(const void *key, void *enc_exp_keys, void *dec_exp_keys)
+
aes_gcm_enc_128_avx_gen2
IMB_DLL_EXPORT void aes_gcm_enc_128_avx_gen2(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-128 Encryption.
+
aes_gmac_finalize_t
void(* aes_gmac_finalize_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint64_t)
Definition: intel-ipsec-mb.h:838
+
aes_gcm_enc_128_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_128_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-128 encryption message.
+
aes_gcm_enc_dec_update_t
void(* aes_gcm_enc_dec_update_t)(const struct gcm_key_data *, struct gcm_context_data *, uint8_t *, const uint8_t *, uint64_t)
Definition: intel-ipsec-mb.h:823
+
aes_gcm_dec_192_finalize_avx_gen4
IMB_DLL_EXPORT void aes_gcm_dec_192_finalize_avx_gen4(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End decryption of a AES-GCM-192 encryption message.
+
aes_keyexp_256_enc_sse
IMB_DLL_EXPORT void aes_keyexp_256_enc_sse(const void *key, void *enc_exp_keys)
+
hash_fn_t
void(* hash_fn_t)(const void *, const uint64_t, void *)
Definition: intel-ipsec-mb.h:800
+
sha512_avx2
IMB_DLL_EXPORT void sha512_avx2(const void *data, const uint64_t length, void *digest)
+
aes_gcm_enc_256_finalize_sse
IMB_DLL_EXPORT void aes_gcm_enc_256_finalize_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *auth_tag, uint64_t auth_tag_len)
End encryption of a AES-GCM-256 encryption message.
+
IMB_STATUS
IMB_STATUS
Definition: intel-ipsec-mb.h:164
+
IMB_STATUS_ERROR
@ IMB_STATUS_ERROR
Definition: intel-ipsec-mb.h:172
+
IMB_STATUS_COMPLETED
@ IMB_STATUS_COMPLETED
Definition: intel-ipsec-mb.h:168
+
IMB_STATUS_INTERNAL_ERROR
@ IMB_STATUS_INTERNAL_ERROR
Definition: intel-ipsec-mb.h:171
+
IMB_STATUS_COMPLETED_CIPHER
@ IMB_STATUS_COMPLETED_CIPHER
Definition: intel-ipsec-mb.h:166
+
IMB_STATUS_COMPLETED_AUTH
@ IMB_STATUS_COMPLETED_AUTH
Definition: intel-ipsec-mb.h:167
+
IMB_STATUS_BEING_PROCESSED
@ IMB_STATUS_BEING_PROCESSED
Definition: intel-ipsec-mb.h:165
+
IMB_STATUS_INVALID_ARGS
@ IMB_STATUS_INVALID_ARGS
Definition: intel-ipsec-mb.h:170
+
aes_gcm_dec_256_sse
IMB_DLL_EXPORT void aes_gcm_dec_256_sse(const struct gcm_key_data *key_data, struct gcm_context_data *context_data, uint8_t *out, uint8_t const *in, uint64_t len, const uint8_t *iv, uint8_t const *aad, uint64_t aad_len, uint8_t *auth_tag, uint64_t auth_tag_len)
AES-GCM-256 Decryption.
+
aes_xcbc_expand_key_avx
IMB_DLL_EXPORT void aes_xcbc_expand_key_avx(const void *key, void *k1_exp, void *k2, void *k3)
+
IMB_JOB
Definition: intel-ipsec-mb.h:483
+
IMB_JOB::cipher_fields
union IMB_JOB::@11 cipher_fields
+
IMB_JOB::ctx
struct gcm_context_data * ctx
Definition: intel-ipsec-mb.h:561
+
IMB_JOB::GMAC
struct IMB_JOB::@10::_AES_GMAC_specific_fields GMAC
+
IMB_JOB::KASUMI_UIA1
struct IMB_JOB::@10::_KASUMI_UIA1_specific_fields KASUMI_UIA1
+
IMB_JOB::cipher_direction
IMB_CIPHER_DIRECTION cipher_direction
Definition: intel-ipsec-mb.h:619
+
IMB_JOB::POLY1305
struct IMB_JOB::@10::_POLY1305_specific_fields POLY1305
+
IMB_JOB::num_sgl_io_segs
uint64_t num_sgl_io_segs
Definition: intel-ipsec-mb.h:498
+
IMB_JOB::_init_tag
const void * _init_tag
Definition: intel-ipsec-mb.h:593
+
IMB_JOB::sgl_io_segs
const struct IMB_SGL_IOV * sgl_io_segs
Definition: intel-ipsec-mb.h:491
+
IMB_JOB::iv_len_in_bytes
uint64_t iv_len_in_bytes
Definition: intel-ipsec-mb.h:523
+
IMB_JOB::SNOW3G_UIA2
struct IMB_JOB::@10::_SNOW3G_UIA2_specific_fields SNOW3G_UIA2
+
IMB_JOB::_iv23
const uint8_t * _iv23
Definition: intel-ipsec-mb.h:569
+
IMB_JOB::_hashed_auth_key_xor_ipad
const uint8_t * _hashed_auth_key_xor_ipad
Definition: intel-ipsec-mb.h:531
+
IMB_JOB::XCBC
struct IMB_JOB::@10::_AES_XCBC_specific_fields XCBC
+
IMB_JOB::_key
const struct gcm_key_data * _key
Definition: intel-ipsec-mb.h:583
+
IMB_JOB::sgl_state
IMB_SGL_STATE sgl_state
Definition: intel-ipsec-mb.h:632
+
IMB_JOB::msg_len_to_hash_in_bytes
uint64_t msg_len_to_hash_in_bytes
Definition: intel-ipsec-mb.h:517
+
IMB_JOB::chain_order
IMB_CHAIN_ORDER chain_order
Definition: intel-ipsec-mb.h:621
+
IMB_JOB::msg_len_to_cipher_in_bits
uint64_t msg_len_to_cipher_in_bits
Definition: intel-ipsec-mb.h:512
+
IMB_JOB::_iv
const void * _iv
Definition: intel-ipsec-mb.h:575
+
IMB_JOB::CMAC
struct IMB_JOB::@10::_AES_CMAC_specific_fields CMAC
+
IMB_JOB::ctx
struct chacha20_poly1305_context_data * ctx
Definition: intel-ipsec-mb.h:604
+
IMB_JOB::aad
const void * aad
Definition: intel-ipsec-mb.h:547
+
IMB_JOB::user_data2
void * user_data2
Definition: intel-ipsec-mb.h:625
+
IMB_JOB::next_iv
void * next_iv
Definition: intel-ipsec-mb.h:638
+
IMB_JOB::u
union IMB_JOB::@10 u
+
IMB_JOB::cipher_start_offset_in_bits
uint64_t cipher_start_offset_in_bits
Definition: intel-ipsec-mb.h:506
+
IMB_JOB::hash_func
int(* hash_func)(struct IMB_JOB *)
Definition: intel-ipsec-mb.h:629
+
IMB_JOB::cipher_func
int(* cipher_func)(struct IMB_JOB *)
Definition: intel-ipsec-mb.h:627
+
IMB_JOB::_hashed_auth_key_xor_opad
const uint8_t * _hashed_auth_key_xor_opad
Definition: intel-ipsec-mb.h:534
+
IMB_JOB::auth_tag_output_len_in_bytes
uint64_t auth_tag_output_len_in_bytes
Definition: intel-ipsec-mb.h:525
+
IMB_JOB::_k1_expanded
const uint32_t * _k1_expanded
Definition: intel-ipsec-mb.h:539
+
IMB_JOB::cipher_mode
IMB_CIPHER_MODE cipher_mode
Definition: intel-ipsec-mb.h:618
+
IMB_JOB::_key
const void * _key
Definition: intel-ipsec-mb.h:573
+
IMB_JOB::_key
const uint8_t * _key
Definition: intel-ipsec-mb.h:565
+
IMB_JOB::CBCS
struct IMB_JOB::@11::_CBCS_specific_fields CBCS
+
IMB_JOB::_skey2
const void * _skey2
Definition: intel-ipsec-mb.h:555
+
IMB_JOB::_key_expanded
const void * _key_expanded
Definition: intel-ipsec-mb.h:552
+
IMB_JOB::iv
const uint8_t * iv
Definition: intel-ipsec-mb.h:522
+
IMB_JOB::msg_len_to_cipher_in_bytes
uint64_t msg_len_to_cipher_in_bytes
Definition: intel-ipsec-mb.h:510
+
IMB_JOB::reserved
void * reserved
Definition: intel-ipsec-mb.h:612
+
IMB_JOB::hash_alg
IMB_HASH_ALG hash_alg
Definition: intel-ipsec-mb.h:620
+
IMB_JOB::aad_len_in_bytes
uint64_t aad_len_in_bytes
Definition: intel-ipsec-mb.h:549
+
IMB_JOB::HMAC
struct IMB_JOB::@10::_HMAC_specific_fields HMAC
+
IMB_JOB::ZUC_EIA3
struct IMB_JOB::@10::_ZUC_EIA3_specific_fields ZUC_EIA3
+
IMB_JOB::_skey1
const void * _skey1
Definition: intel-ipsec-mb.h:554
+
IMB_JOB::user_data
void * user_data
Definition: intel-ipsec-mb.h:624
+
IMB_JOB::key_len_in_bytes
uint64_t key_len_in_bytes
Definition: intel-ipsec-mb.h:486
+
IMB_JOB::SNOW_V_AEAD
struct IMB_JOB::@10::_SNOW_V_AEAD_specific_fields SNOW_V_AEAD
+
IMB_JOB::dst
uint8_t * dst
Definition: intel-ipsec-mb.h:495
+
IMB_JOB::auth_tag_output
uint8_t * auth_tag_output
Definition: intel-ipsec-mb.h:524
+
IMB_JOB::enc_keys
const void * enc_keys
Definition: intel-ipsec-mb.h:484
+
IMB_JOB::GCM
struct IMB_JOB::@10::_AES_GCM_specific_fields GCM
+
IMB_JOB::CHACHA20_POLY1305
struct IMB_JOB::@10::_CHACHA20_POLY1305_specific_fields CHACHA20_POLY1305
+
IMB_JOB::CCM
struct IMB_JOB::@10::_AES_CCM_specific_fields CCM
+
IMB_JOB::status
IMB_STATUS status
Definition: intel-ipsec-mb.h:617
+
IMB_JOB::_k2
const uint8_t * _k2
Definition: intel-ipsec-mb.h:541
+
IMB_JOB::cipher_start_src_offset_in_bytes
uint64_t cipher_start_src_offset_in_bytes
Definition: intel-ipsec-mb.h:502
+
IMB_JOB::msg_len_to_hash_in_bits
uint64_t msg_len_to_hash_in_bits
Definition: intel-ipsec-mb.h:519
+
IMB_JOB::_iv
const uint8_t * _iv
Definition: intel-ipsec-mb.h:567
+
IMB_JOB::GHASH
struct IMB_JOB::@10::_GHASH_specific_fields GHASH
+
IMB_JOB::cipher_start_src_offset_in_bits
uint64_t cipher_start_src_offset_in_bits
Definition: intel-ipsec-mb.h:504
+
IMB_JOB::dec_keys
const void * dec_keys
Definition: intel-ipsec-mb.h:485
+
IMB_JOB::hash_start_src_offset_in_bytes
uint64_t hash_start_src_offset_in_bytes
Definition: intel-ipsec-mb.h:515
+
IMB_JOB::src
const uint8_t * src
Definition: intel-ipsec-mb.h:488
+
IMB_JOB::_k3
const uint8_t * _k3
Definition: intel-ipsec-mb.h:543
+
IMB_MGR
Definition: intel-ipsec-mb.h:1043
+
IMB_MGR::gmac192_finalize
aes_gmac_finalize_t gmac192_finalize
Definition: intel-ipsec-mb.h:1154
+
IMB_MGR::gcm192_init
aes_gcm_init_t gcm192_init
Definition: intel-ipsec-mb.h:1091
+
IMB_MGR::gmac128_finalize
aes_gmac_finalize_t gmac128_finalize
Definition: intel-ipsec-mb.h:1153
+
IMB_MGR::aes256_ccm_ooo
void * aes256_ccm_ooo
Definition: intel-ipsec-mb.h:1219
+
IMB_MGR::crc24_lte_a
crc32_fn_t crc24_lte_a
Definition: intel-ipsec-mb.h:1163
+
IMB_MGR::gmac256_update
aes_gmac_update_t gmac256_update
Definition: intel-ipsec-mb.h:1152
+
IMB_MGR::hec_32
hec_32_t hec_32
Definition: intel-ipsec-mb.h:1156
+
IMB_MGR::imb_errno
int imb_errno
Definition: intel-ipsec-mb.h:1051
+
IMB_MGR::aes_cmac_ooo
void * aes_cmac_ooo
Definition: intel-ipsec-mb.h:1213
+
IMB_MGR::next_job
int next_job
Definition: intel-ipsec-mb.h:1187
+
IMB_MGR::gmac128_init
aes_gmac_init_t gmac128_init
Definition: intel-ipsec-mb.h:1147
+
IMB_MGR::reserved
uint64_t reserved[5]
Definition: intel-ipsec-mb.h:1048
+
IMB_MGR::gcm192_pre
aes_gcm_pre_t gcm192_pre
Definition: intel-ipsec-mb.h:1109
+
IMB_MGR::snow3g_f8_4_buffer
snow3g_f8_4_buffer_t snow3g_f8_4_buffer
Definition: intel-ipsec-mb.h:1132
+
IMB_MGR::gcm256_precomp
aes_gcm_precomp_t gcm256_precomp
Definition: intel-ipsec-mb.h:1107
+
IMB_MGR::gcm128_pre
aes_gcm_pre_t gcm128_pre
Definition: intel-ipsec-mb.h:1108
+
IMB_MGR::submit_hash_burst
submit_hash_burst_t submit_hash_burst
Definition: intel-ipsec-mb.h:1182
+
IMB_MGR::flush_job
flush_job_t flush_job
Definition: intel-ipsec-mb.h:1063
+
IMB_MGR::snow3g_f8_2_buffer
snow3g_f8_2_buffer_t snow3g_f8_2_buffer
Definition: intel-ipsec-mb.h:1131
+
IMB_MGR::zuc_eea3_ooo
void * zuc_eea3_ooo
Definition: intel-ipsec-mb.h:1214
+
IMB_MGR::gcm128_dec_update
aes_gcm_enc_dec_update_t gcm128_dec_update
Definition: intel-ipsec-mb.h:1096
+
IMB_MGR::des3_dec_ooo
void * des3_dec_ooo
Definition: intel-ipsec-mb.h:1201
+
IMB_MGR::keyexp_192
keyexp_t keyexp_192
Definition: intel-ipsec-mb.h:1066
+
IMB_MGR::gcm128_dec
aes_gcm_enc_dec_t gcm128_dec
Definition: intel-ipsec-mb.h:1087
+
IMB_MGR::get_next_job
get_next_job_t get_next_job
Definition: intel-ipsec-mb.h:1059
+
IMB_MGR::sha_256_ooo
void * sha_256_ooo
Definition: intel-ipsec-mb.h:1225
+
IMB_MGR::snow3g_init_key_sched
snow3g_init_key_sched_t snow3g_init_key_sched
Definition: intel-ipsec-mb.h:1138
+
IMB_MGR::gcm192_enc_update
aes_gcm_enc_dec_update_t gcm192_enc_update
Definition: intel-ipsec-mb.h:1094
+
IMB_MGR::gmac256_finalize
aes_gmac_finalize_t gmac256_finalize
Definition: intel-ipsec-mb.h:1155
+
IMB_MGR::docsis_des_dec_ooo
void * docsis_des_dec_ooo
Definition: intel-ipsec-mb.h:1203
+
IMB_MGR::gmac192_init
aes_gmac_init_t gmac192_init
Definition: intel-ipsec-mb.h:1148
+
IMB_MGR::eia3_1_buffer
zuc_eia3_1_buffer_t eia3_1_buffer
Definition: intel-ipsec-mb.h:1115
+
IMB_MGR::aes_ccm_ooo
void * aes_ccm_ooo
Definition: intel-ipsec-mb.h:1212
+
IMB_MGR::hec_64
hec_64_t hec_64
Definition: intel-ipsec-mb.h:1157
+
IMB_MGR::keyexp_256
keyexp_t keyexp_256
Definition: intel-ipsec-mb.h:1067
+
IMB_MGR::gcm256_pre
aes_gcm_pre_t gcm256_pre
Definition: intel-ipsec-mb.h:1110
+
IMB_MGR::used_arch
uint32_t used_arch
Definition: intel-ipsec-mb.h:1049
+
IMB_MGR::hmac_sha_1_ooo
void * hmac_sha_1_ooo
Definition: intel-ipsec-mb.h:1205
+
IMB_MGR::crc32_sctp
crc32_fn_t crc32_sctp
Definition: intel-ipsec-mb.h:1162
+
IMB_MGR::crc11_fp_header
crc32_fn_t crc11_fp_header
Definition: intel-ipsec-mb.h:1166
+
IMB_MGR::gcm192_dec_finalize
aes_gcm_enc_dec_finalize_t gcm192_dec_finalize
Definition: intel-ipsec-mb.h:1103
+
IMB_MGR::f9_1_buffer_user
kasumi_f9_1_buffer_user_t f9_1_buffer_user
Definition: intel-ipsec-mb.h:1124
+
IMB_MGR::sha_384_ooo
void * sha_384_ooo
Definition: intel-ipsec-mb.h:1226
+
IMB_MGR::gmac192_update
aes_gmac_update_t gmac192_update
Definition: intel-ipsec-mb.h:1151
+
IMB_MGR::crc16_x25
crc32_fn_t crc16_x25
Definition: intel-ipsec-mb.h:1161
+
IMB_MGR::sha224
hash_fn_t sha224
Definition: intel-ipsec-mb.h:1078
+
IMB_MGR::md5_one_block
hash_one_block_t md5_one_block
Definition: intel-ipsec-mb.h:1076
+
IMB_MGR::des_enc_ooo
void * des_enc_ooo
Definition: intel-ipsec-mb.h:1198
+
IMB_MGR::kasumi_init_f9_key_sched
kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched
Definition: intel-ipsec-mb.h:1126
+
IMB_MGR::queue_size
queue_size_t queue_size
Definition: intel-ipsec-mb.h:1064
+
IMB_MGR::gcm256_dec_finalize
aes_gcm_enc_dec_finalize_t gcm256_dec_finalize
Definition: intel-ipsec-mb.h:1104
+
IMB_MGR::snow3g_key_sched_size
snow3g_key_sched_size_t snow3g_key_sched_size
Definition: intel-ipsec-mb.h:1139
+
IMB_MGR::crc8_wimax_ofdma_hcs
crc32_fn_t crc8_wimax_ofdma_hcs
Definition: intel-ipsec-mb.h:1171
+
IMB_MGR::zuc256_eia3_ooo
void * zuc256_eia3_ooo
Definition: intel-ipsec-mb.h:1218
+
IMB_MGR::snow3g_f8_8_buffer
snow3g_f8_8_buffer_t snow3g_f8_8_buffer
Definition: intel-ipsec-mb.h:1133
+
IMB_MGR::sha512
hash_fn_t sha512
Definition: intel-ipsec-mb.h:1081
+
IMB_MGR::hmac_sha_256_ooo
void * hmac_sha_256_ooo
Definition: intel-ipsec-mb.h:1207
+
IMB_MGR::snow3g_f8_n_buffer_multikey
snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey
Definition: intel-ipsec-mb.h:1136
+
IMB_MGR::gcm192_dec_update
aes_gcm_enc_dec_update_t gcm192_dec_update
Definition: intel-ipsec-mb.h:1097
+
IMB_MGR::chacha20_poly1305_init
chacha_poly_init_t chacha20_poly1305_init
Definition: intel-ipsec-mb.h:1173
+
IMB_MGR::eea3_n_buffer
zuc_eea3_n_buffer_t eea3_n_buffer
Definition: intel-ipsec-mb.h:1114
+
IMB_MGR::gcm128_precomp
aes_gcm_precomp_t gcm128_precomp
Definition: intel-ipsec-mb.h:1105
+
IMB_MGR::submit_burst
submit_burst_t submit_burst
Definition: intel-ipsec-mb.h:1178
+
IMB_MGR::aes256_ooo
void * aes256_ooo
Definition: intel-ipsec-mb.h:1193
+
IMB_MGR::zuc256_eea3_ooo
void * zuc256_eea3_ooo
Definition: intel-ipsec-mb.h:1217
+
IMB_MGR::end_ooo
void * end_ooo
Definition: intel-ipsec-mb.h:1228
+
IMB_MGR::sha_512_ooo
void * sha_512_ooo
Definition: intel-ipsec-mb.h:1227
+
IMB_MGR::chacha20_poly1305_dec_update
chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update
Definition: intel-ipsec-mb.h:1175
+
IMB_MGR::gcm256_enc
aes_gcm_enc_dec_t gcm256_enc
Definition: intel-ipsec-mb.h:1086
+
IMB_MGR::gcm192_init_var_iv
aes_gcm_init_var_iv_t gcm192_init_var_iv
Definition: intel-ipsec-mb.h:1144
+
IMB_MGR::keyexp_128
keyexp_t keyexp_128
Definition: intel-ipsec-mb.h:1065
+
IMB_MGR::crc10_iuup_data
crc32_fn_t crc10_iuup_data
Definition: intel-ipsec-mb.h:1168
+
IMB_MGR::submit_hash_burst_nocheck
submit_hash_burst_t submit_hash_burst_nocheck
Definition: intel-ipsec-mb.h:1183
+
IMB_MGR::sha1_one_block
hash_one_block_t sha1_one_block
Definition: intel-ipsec-mb.h:1071
+
IMB_MGR::snow3g_f8_n_buffer
snow3g_f8_n_buffer_t snow3g_f8_n_buffer
Definition: intel-ipsec-mb.h:1134
+
IMB_MGR::sha1
hash_fn_t sha1
Definition: intel-ipsec-mb.h:1077
+
IMB_MGR::crc32_ethernet_fcs
crc32_fn_t crc32_ethernet_fcs
Definition: intel-ipsec-mb.h:1160
+
IMB_MGR::hmac_md5_ooo
void * hmac_md5_ooo
Definition: intel-ipsec-mb.h:1210
+
IMB_MGR::des3_enc_ooo
void * des3_enc_ooo
Definition: intel-ipsec-mb.h:1200
+
IMB_MGR::sha512_one_block
hash_one_block_t sha512_one_block
Definition: intel-ipsec-mb.h:1075
+
IMB_MGR::gcm256_init
aes_gcm_init_t gcm256_init
Definition: intel-ipsec-mb.h:1092
+
IMB_MGR::snow3g_f8_8_buffer_multikey
snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey
Definition: intel-ipsec-mb.h:1135
+
IMB_MGR::submit_cipher_burst
submit_cipher_burst_t submit_cipher_burst
Definition: intel-ipsec-mb.h:1180
+
IMB_MGR::sha256
hash_fn_t sha256
Definition: intel-ipsec-mb.h:1079
+
IMB_MGR::gcm128_enc_finalize
aes_gcm_enc_dec_finalize_t gcm128_enc_finalize
Definition: intel-ipsec-mb.h:1099
+
IMB_MGR::gmac128_update
aes_gmac_update_t gmac128_update
Definition: intel-ipsec-mb.h:1150
+
IMB_MGR::zuc_eia3_ooo
void * zuc_eia3_ooo
Definition: intel-ipsec-mb.h:1215
+
IMB_MGR::gcm192_enc_finalize
aes_gcm_enc_dec_finalize_t gcm192_enc_finalize
Definition: intel-ipsec-mb.h:1100
+
IMB_MGR::f8_3_buffer
kasumi_f8_3_buffer_t f8_3_buffer
Definition: intel-ipsec-mb.h:1120
+
IMB_MGR::sha384_one_block
hash_one_block_t sha384_one_block
Definition: intel-ipsec-mb.h:1074
+
IMB_MGR::docsis_des_enc_ooo
void * docsis_des_enc_ooo
Definition: intel-ipsec-mb.h:1202
+
IMB_MGR::docsis128_sec_ooo
void * docsis128_sec_ooo
Definition: intel-ipsec-mb.h:1194
+
IMB_MGR::gcm256_enc_finalize
aes_gcm_enc_dec_finalize_t gcm256_enc_finalize
Definition: intel-ipsec-mb.h:1101
+
IMB_MGR::aes128_ooo
void * aes128_ooo
Definition: intel-ipsec-mb.h:1191
+
IMB_MGR::get_completed_job
get_completed_job_t get_completed_job
Definition: intel-ipsec-mb.h:1062
+
IMB_MGR::gcm256_dec_update
aes_gcm_enc_dec_update_t gcm256_dec_update
Definition: intel-ipsec-mb.h:1098
+
IMB_MGR::snow3g_f9_1_buffer
snow3g_f9_1_buffer_t snow3g_f9_1_buffer
Definition: intel-ipsec-mb.h:1137
+
IMB_MGR::aes128_cbcs_ooo
void * aes128_cbcs_ooo
Definition: intel-ipsec-mb.h:1216
+
IMB_MGR::gcm128_enc
aes_gcm_enc_dec_t gcm128_enc
Definition: intel-ipsec-mb.h:1084
+
IMB_MGR::sha384
hash_fn_t sha384
Definition: intel-ipsec-mb.h:1080
+
IMB_MGR::sha_224_ooo
void * sha_224_ooo
Definition: intel-ipsec-mb.h:1224
+
IMB_MGR::cmac_subkey_gen_128
cmac_subkey_gen_t cmac_subkey_gen_128
Definition: intel-ipsec-mb.h:1068
+
IMB_MGR::crc6_iuup_header
crc32_fn_t crc6_iuup_header
Definition: intel-ipsec-mb.h:1169
+
IMB_MGR::snow3g_uia2_ooo
void * snow3g_uia2_ooo
Definition: intel-ipsec-mb.h:1222
+
IMB_MGR::gcm192_enc
aes_gcm_enc_dec_t gcm192_enc
Definition: intel-ipsec-mb.h:1085
+
IMB_MGR::submit_job_nocheck
submit_job_t submit_job_nocheck
Definition: intel-ipsec-mb.h:1061
+
IMB_MGR::f8_1_buffer_bit
kasumi_f8_1_buffer_bit_t f8_1_buffer_bit
Definition: intel-ipsec-mb.h:1118
+
IMB_MGR::sha224_one_block
hash_one_block_t sha224_one_block
Definition: intel-ipsec-mb.h:1072
+
IMB_MGR::des_key_sched
des_keysched_t des_key_sched
Definition: intel-ipsec-mb.h:1070
+
IMB_MGR::snow3g_uea2_ooo
void * snow3g_uea2_ooo
Definition: intel-ipsec-mb.h:1221
+
IMB_MGR::submit_job
submit_job_t submit_job
Definition: intel-ipsec-mb.h:1060
+
IMB_MGR::chacha20_poly1305_finalize
chacha_poly_finalize_t chacha20_poly1305_finalize
Definition: intel-ipsec-mb.h:1176
+
IMB_MGR::chacha20_poly1305_enc_update
chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update
Definition: intel-ipsec-mb.h:1174
+
IMB_MGR::kasumi_init_f8_key_sched
kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched
Definition: intel-ipsec-mb.h:1125
+
IMB_MGR::jobs
IMB_JOB jobs[IMB_MAX_JOBS]
Definition: intel-ipsec-mb.h:1188
+
IMB_MGR::eia3_n_buffer
zuc_eia3_n_buffer_t eia3_n_buffer
Definition: intel-ipsec-mb.h:1142
+
IMB_MGR::snow3g_f8_1_buffer
snow3g_f8_1_buffer_t snow3g_f8_1_buffer
Definition: intel-ipsec-mb.h:1130
+
IMB_MGR::hmac_sha_224_ooo
void * hmac_sha_224_ooo
Definition: intel-ipsec-mb.h:1206
+
IMB_MGR::kasumi_key_sched_size
kasumi_key_sched_size_t kasumi_key_sched_size
Definition: intel-ipsec-mb.h:1127
+
IMB_MGR::aes128_cfb_one
aes_cfb_t aes128_cfb_one
Definition: intel-ipsec-mb.h:1082
+
IMB_MGR::cmac_subkey_gen_256
cmac_subkey_gen_t cmac_subkey_gen_256
Definition: intel-ipsec-mb.h:1158
+
IMB_MGR::crc24_lte_b
crc32_fn_t crc24_lte_b
Definition: intel-ipsec-mb.h:1164
+
IMB_MGR::gcm192_precomp
aes_gcm_precomp_t gcm192_precomp
Definition: intel-ipsec-mb.h:1106
+
IMB_MGR::gcm256_enc_update
aes_gcm_enc_dec_update_t gcm256_enc_update
Definition: intel-ipsec-mb.h:1095
+
IMB_MGR::eea3_4_buffer
zuc_eea3_4_buffer_t eea3_4_buffer
Definition: intel-ipsec-mb.h:1113
+
IMB_MGR::earliest_job
int earliest_job
Definition: intel-ipsec-mb.h:1186
+
IMB_MGR::f8_2_buffer
kasumi_f8_2_buffer_t f8_2_buffer
Definition: intel-ipsec-mb.h:1119
+
IMB_MGR::crc7_fp_header
crc32_fn_t crc7_fp_header
Definition: intel-ipsec-mb.h:1167
+
IMB_MGR::gmac256_init
aes_gmac_init_t gmac256_init
Definition: intel-ipsec-mb.h:1149
+
IMB_MGR::submit_cipher_burst_nocheck
submit_cipher_burst_t submit_cipher_burst_nocheck
Definition: intel-ipsec-mb.h:1181
+
IMB_MGR::docsis256_sec_ooo
void * docsis256_sec_ooo
Definition: intel-ipsec-mb.h:1196
+
IMB_MGR::aes256_cmac_ooo
void * aes256_cmac_ooo
Definition: intel-ipsec-mb.h:1220
+
IMB_MGR::sha_1_ooo
void * sha_1_ooo
Definition: intel-ipsec-mb.h:1223
+
IMB_MGR::docsis128_crc32_sec_ooo
void * docsis128_crc32_sec_ooo
Definition: intel-ipsec-mb.h:1195
+
IMB_MGR::gcm256_dec
aes_gcm_enc_dec_t gcm256_dec
Definition: intel-ipsec-mb.h:1089
+
IMB_MGR::crc32_wimax_ofdma_data
crc32_fn_t crc32_wimax_ofdma_data
Definition: intel-ipsec-mb.h:1170
+
IMB_MGR::submit_burst_nocheck
submit_burst_t submit_burst_nocheck
Definition: intel-ipsec-mb.h:1179
+
IMB_MGR::gcm128_enc_update
aes_gcm_enc_dec_update_t gcm128_enc_update
Definition: intel-ipsec-mb.h:1093
+
IMB_MGR::aes_xcbc_ooo
void * aes_xcbc_ooo
Definition: intel-ipsec-mb.h:1211
+
IMB_MGR::crc16_fp_data
crc32_fn_t crc16_fp_data
Definition: intel-ipsec-mb.h:1165
+
IMB_MGR::snow3g_f8_1_buffer_bit
snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit
Definition: intel-ipsec-mb.h:1129
+
IMB_MGR::gcm128_init
aes_gcm_init_t gcm128_init
Definition: intel-ipsec-mb.h:1090
+
IMB_MGR::sha256_one_block
hash_one_block_t sha256_one_block
Definition: intel-ipsec-mb.h:1073
+
IMB_MGR::gcm192_dec
aes_gcm_enc_dec_t gcm192_dec
Definition: intel-ipsec-mb.h:1088
+
IMB_MGR::f8_n_buffer
kasumi_f8_n_buffer_t f8_n_buffer
Definition: intel-ipsec-mb.h:1122
+
IMB_MGR::flags
uint64_t flags
Definition: intel-ipsec-mb.h:1045
+
IMB_MGR::xcbc_keyexp
xcbc_keyexp_t xcbc_keyexp
Definition: intel-ipsec-mb.h:1069
+
IMB_MGR::gcm128_init_var_iv
aes_gcm_init_var_iv_t gcm128_init_var_iv
Definition: intel-ipsec-mb.h:1143
+
IMB_MGR::eea3_1_buffer
zuc_eea3_1_buffer_t eea3_1_buffer
Definition: intel-ipsec-mb.h:1112
+
IMB_MGR::des_dec_ooo
void * des_dec_ooo
Definition: intel-ipsec-mb.h:1199
+
IMB_MGR::f8_4_buffer
kasumi_f8_4_buffer_t f8_4_buffer
Definition: intel-ipsec-mb.h:1121
+
IMB_MGR::hmac_sha_512_ooo
void * hmac_sha_512_ooo
Definition: intel-ipsec-mb.h:1209
+
IMB_MGR::docsis256_crc32_sec_ooo
void * docsis256_crc32_sec_ooo
Definition: intel-ipsec-mb.h:1197
+
IMB_MGR::ghash_pre
aes_gcm_pre_t ghash_pre
Definition: intel-ipsec-mb.h:1159
+
IMB_MGR::f8_1_buffer
kasumi_f8_1_buffer_t f8_1_buffer
Definition: intel-ipsec-mb.h:1117
+
IMB_MGR::features
uint64_t features
Definition: intel-ipsec-mb.h:1046
+
IMB_MGR::gcm128_dec_finalize
aes_gcm_enc_dec_finalize_t gcm128_dec_finalize
Definition: intel-ipsec-mb.h:1102
+
IMB_MGR::gcm256_init_var_iv
aes_gcm_init_var_iv_t gcm256_init_var_iv
Definition: intel-ipsec-mb.h:1145
+
IMB_MGR::f9_1_buffer
kasumi_f9_1_buffer_t f9_1_buffer
Definition: intel-ipsec-mb.h:1123
+
IMB_MGR::ghash
ghash_t ghash
Definition: intel-ipsec-mb.h:1141
+
IMB_MGR::hmac_sha_384_ooo
void * hmac_sha_384_ooo
Definition: intel-ipsec-mb.h:1208
+
IMB_MGR::aes192_ooo
void * aes192_ooo
Definition: intel-ipsec-mb.h:1192
+
IMB_SGL_IOV
Definition: intel-ipsec-mb.h:458
+
IMB_SGL_IOV::len
uint64_t len
Definition: intel-ipsec-mb.h:461
+
IMB_SGL_IOV::out
void * out
Definition: intel-ipsec-mb.h:460
+
IMB_SGL_IOV::in
const void * in
Definition: intel-ipsec-mb.h:459
+
chacha20_poly1305_context_data
holds Chacha20-Poly1305 operation context
Definition: intel-ipsec-mb.h:681
+
chacha20_poly1305_context_data::remain_ct_bytes
uint64_t remain_ct_bytes
Definition: intel-ipsec-mb.h:691
+
chacha20_poly1305_context_data::aad_len
uint64_t aad_len
Definition: intel-ipsec-mb.h:683
+
chacha20_poly1305_context_data::remain_ks_bytes
uint64_t remain_ks_bytes
Definition: intel-ipsec-mb.h:689
+
chacha20_poly1305_context_data::poly_scratch
uint8_t poly_scratch[16]
Definition: intel-ipsec-mb.h:687
+
chacha20_poly1305_context_data::poly_key
uint8_t poly_key[32]
Definition: intel-ipsec-mb.h:686
+
chacha20_poly1305_context_data::IV
uint8_t IV[12]
Definition: intel-ipsec-mb.h:694
+
chacha20_poly1305_context_data::last_block_count
uint64_t last_block_count
Definition: intel-ipsec-mb.h:688
+
chacha20_poly1305_context_data::hash_len
uint64_t hash_len
Definition: intel-ipsec-mb.h:684
+
chacha20_poly1305_context_data::hash
uint64_t hash[3]
Definition: intel-ipsec-mb.h:682
+
chacha20_poly1305_context_data::last_ks
uint8_t last_ks[64]
Definition: intel-ipsec-mb.h:685
+
gcm_context_data
holds GCM operation context
Definition: intel-ipsec-mb.h:667
+
gcm_context_data::in_length
uint64_t in_length
Definition: intel-ipsec-mb.h:670
+
gcm_context_data::current_counter
uint8_t current_counter[IMB_GCM_BLOCK_LEN]
Definition: intel-ipsec-mb.h:673
+
gcm_context_data::partial_block_enc_key
uint8_t partial_block_enc_key[IMB_GCM_BLOCK_LEN]
Definition: intel-ipsec-mb.h:671
+
gcm_context_data::orig_IV
uint8_t orig_IV[IMB_GCM_BLOCK_LEN]
Definition: intel-ipsec-mb.h:672
+
gcm_context_data::partial_block_length
uint64_t partial_block_length
Definition: intel-ipsec-mb.h:674
+
gcm_context_data::aad_hash
uint8_t aad_hash[IMB_GCM_BLOCK_LEN]
Definition: intel-ipsec-mb.h:668
+
gcm_context_data::aad_length
uint64_t aad_length
Definition: intel-ipsec-mb.h:669
+
gcm_key_data
holds intermediate key data needed to improve performance
Definition: intel-ipsec-mb.h:729
+
gcm_key_data::avx2_avx512
struct gcm_key_data::@12::@14 avx2_avx512
+
gcm_key_data::shifted_hkey_k
uint8_t shifted_hkey_k[IMB_GCM_ENC_KEY_LEN *8]
Definition: intel-ipsec-mb.h:748
+
gcm_key_data::shifted_hkey
uint8_t shifted_hkey[IMB_GCM_ENC_KEY_LEN *8]
Definition: intel-ipsec-mb.h:739
+
gcm_key_data::vaes_avx512
struct gcm_key_data::@12::@15 vaes_avx512
+
gcm_key_data::sse_avx
struct gcm_key_data::@12::@13 sse_avx
+
gcm_key_data::expanded_keys
uint8_t expanded_keys[IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
Definition: intel-ipsec-mb.h:730
+
gcm_key_data::ghash_keys
union gcm_key_data::@12 ghash_keys
+
imb_uint128_t
Definition: intel-ipsec-mb.h:40
+
imb_uint128_t::low
uint64_t low
Definition: intel-ipsec-mb.h:41
+
imb_uint128_t::high
uint64_t high
Definition: intel-ipsec-mb.h:42
+
kasumi_key_sched_s
Definition: intel-ipsec-mb.h:653
+
kasumi_key_sched_s::msk16
uint16_t msk16[KASUMI_KEY_SCHEDULE_SIZE]
Definition: intel-ipsec-mb.h:656
+
kasumi_key_sched_s::sk16
uint16_t sk16[KASUMI_KEY_SCHEDULE_SIZE]
Definition: intel-ipsec-mb.h:655
+
snow3g_key_schedule_s
Definition: intel-ipsec-mb.h:922
+
snow3g_key_schedule_s::k
uint32_t k[4]
Definition: intel-ipsec-mb.h:924
+
+
+ + + + diff --git a/docs/jquery.js b/docs/jquery.js new file mode 100644 index 00000000..c9ed3d99 --- /dev/null +++ b/docs/jquery.js @@ -0,0 +1,35 @@ +/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0a;a++)for(i in o[a])n=o[a][i],o[a].hasOwnProperty(i)&&void 0!==n&&(e[i]=t.isPlainObject(n)?t.isPlainObject(e[i])?t.widget.extend({},e[i],n):t.widget.extend({},n):n);return e},t.widget.bridge=function(e,i){var n=i.prototype.widgetFullName||e;t.fn[e]=function(o){var a="string"==typeof o,r=s.call(arguments,1),h=this;return a?this.length||"instance"!==o?this.each(function(){var i,s=t.data(this,n);return"instance"===o?(h=s,!1):s?t.isFunction(s[o])&&"_"!==o.charAt(0)?(i=s[o].apply(s,r),i!==s&&void 0!==i?(h=i&&i.jquery?h.pushStack(i.get()):i,!1):void 0):t.error("no such method '"+o+"' for "+e+" widget instance"):t.error("cannot call methods on "+e+" prior to initialization; "+"attempted to call method '"+o+"'")}):h=void 0:(r.length&&(o=t.widget.extend.apply(null,[o].concat(r))),this.each(function(){var e=t.data(this,n);e?(e.option(o||{}),e._init&&e._init()):t.data(this,n,new i(o,this))})),h}},t.Widget=function(){},t.Widget._childConstructors=[],t.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",defaultElement:"
",options:{classes:{},disabled:!1,create:null},_createWidget:function(e,s){s=t(s||this.defaultElement||this)[0],this.element=t(s),this.uuid=i++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=t(),this.hoverable=t(),this.focusable=t(),this.classesElementLookup={},s!==this&&(t.data(s,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===s&&this.destroy()}}),this.document=t(s.style?s.ownerDocument:s.document||s),this.window=t(this.document[0].defaultView||this.document[0].parentWindow)),this.options=t.widget.extend({},this.options,this._getCreateOptions(),e),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:t.noop,_create:t.noop,_init:t.noop,destroy:function(){var e=this;this._destroy(),t.each(this.classesElementLookup,function(t,i){e._removeClass(i,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:t.noop,widget:function(){return this.element},option:function(e,i){var s,n,o,a=e;if(0===arguments.length)return t.widget.extend({},this.options);if("string"==typeof e)if(a={},s=e.split("."),e=s.shift(),s.length){for(n=a[e]=t.widget.extend({},this.options[e]),o=0;s.length-1>o;o++)n[s[o]]=n[s[o]]||{},n=n[s[o]];if(e=s.pop(),1===arguments.length)return void 0===n[e]?null:n[e];n[e]=i}else{if(1===arguments.length)return void 0===this.options[e]?null:this.options[e];a[e]=i}return this._setOptions(a),this},_setOptions:function(t){var e;for(e in t)this._setOption(e,t[e]);return this},_setOption:function(t,e){return"classes"===t&&this._setOptionClasses(e),this.options[t]=e,"disabled"===t&&this._setOptionDisabled(e),this},_setOptionClasses:function(e){var i,s,n;for(i in e)n=this.classesElementLookup[i],e[i]!==this.options.classes[i]&&n&&n.length&&(s=t(n.get()),this._removeClass(n,i),s.addClass(this._classes({element:s,keys:i,classes:e,add:!0})))},_setOptionDisabled:function(t){this._toggleClass(this.widget(),this.widgetFullName+"-disabled",null,!!t),t&&(this._removeClass(this.hoverable,null,"ui-state-hover"),this._removeClass(this.focusable,null,"ui-state-focus"))},enable:function(){return this._setOptions({disabled:!1})},disable:function(){return this._setOptions({disabled:!0})},_classes:function(e){function i(i,o){var a,r;for(r=0;i.length>r;r++)a=n.classesElementLookup[i[r]]||t(),a=e.add?t(t.unique(a.get().concat(e.element.get()))):t(a.not(e.element).get()),n.classesElementLookup[i[r]]=a,s.push(i[r]),o&&e.classes[i[r]]&&s.push(e.classes[i[r]])}var s=[],n=this;return e=t.extend({element:this.element,classes:this.options.classes||{}},e),this._on(e.element,{remove:"_untrackClassesElement"}),e.keys&&i(e.keys.match(/\S+/g)||[],!0),e.extra&&i(e.extra.match(/\S+/g)||[]),s.join(" ")},_untrackClassesElement:function(e){var i=this;t.each(i.classesElementLookup,function(s,n){-1!==t.inArray(e.target,n)&&(i.classesElementLookup[s]=t(n.not(e.target).get()))})},_removeClass:function(t,e,i){return this._toggleClass(t,e,i,!1)},_addClass:function(t,e,i){return this._toggleClass(t,e,i,!0)},_toggleClass:function(t,e,i,s){s="boolean"==typeof s?s:i;var n="string"==typeof t||null===t,o={extra:n?e:i,keys:n?t:e,element:n?this.element:t,add:s};return o.element.toggleClass(this._classes(o),s),this},_on:function(e,i,s){var n,o=this;"boolean"!=typeof e&&(s=i,i=e,e=!1),s?(i=n=t(i),this.bindings=this.bindings.add(i)):(s=i,i=this.element,n=this.widget()),t.each(s,function(s,a){function r(){return e||o.options.disabled!==!0&&!t(this).hasClass("ui-state-disabled")?("string"==typeof a?o[a]:a).apply(o,arguments):void 0}"string"!=typeof a&&(r.guid=a.guid=a.guid||r.guid||t.guid++);var h=s.match(/^([\w:-]*)\s*(.*)$/),l=h[1]+o.eventNamespace,c=h[2];c?n.on(l,c,r):i.on(l,r)})},_off:function(e,i){i=(i||"").split(" ").join(this.eventNamespace+" ")+this.eventNamespace,e.off(i).off(i),this.bindings=t(this.bindings.not(e).get()),this.focusable=t(this.focusable.not(e).get()),this.hoverable=t(this.hoverable.not(e).get())},_delay:function(t,e){function i(){return("string"==typeof t?s[t]:t).apply(s,arguments)}var s=this;return setTimeout(i,e||0)},_hoverable:function(e){this.hoverable=this.hoverable.add(e),this._on(e,{mouseenter:function(e){this._addClass(t(e.currentTarget),null,"ui-state-hover")},mouseleave:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-hover")}})},_focusable:function(e){this.focusable=this.focusable.add(e),this._on(e,{focusin:function(e){this._addClass(t(e.currentTarget),null,"ui-state-focus")},focusout:function(e){this._removeClass(t(e.currentTarget),null,"ui-state-focus")}})},_trigger:function(e,i,s){var n,o,a=this.options[e];if(s=s||{},i=t.Event(i),i.type=(e===this.widgetEventPrefix?e:this.widgetEventPrefix+e).toLowerCase(),i.target=this.element[0],o=i.originalEvent)for(n in o)n in i||(i[n]=o[n]);return this.element.trigger(i,s),!(t.isFunction(a)&&a.apply(this.element[0],[i].concat(s))===!1||i.isDefaultPrevented())}},t.each({show:"fadeIn",hide:"fadeOut"},function(e,i){t.Widget.prototype["_"+e]=function(s,n,o){"string"==typeof n&&(n={effect:n});var a,r=n?n===!0||"number"==typeof n?i:n.effect||i:e;n=n||{},"number"==typeof n&&(n={duration:n}),a=!t.isEmptyObject(n),n.complete=o,n.delay&&s.delay(n.delay),a&&t.effects&&t.effects.effect[r]?s[e](n):r!==e&&s[r]?s[r](n.duration,n.easing,o):s.queue(function(i){t(this)[e](),o&&o.call(s[0]),i()})}}),t.widget,function(){function e(t,e,i){return[parseFloat(t[0])*(u.test(t[0])?e/100:1),parseFloat(t[1])*(u.test(t[1])?i/100:1)]}function i(e,i){return parseInt(t.css(e,i),10)||0}function s(e){var i=e[0];return 9===i.nodeType?{width:e.width(),height:e.height(),offset:{top:0,left:0}}:t.isWindow(i)?{width:e.width(),height:e.height(),offset:{top:e.scrollTop(),left:e.scrollLeft()}}:i.preventDefault?{width:0,height:0,offset:{top:i.pageY,left:i.pageX}}:{width:e.outerWidth(),height:e.outerHeight(),offset:e.offset()}}var n,o=Math.max,a=Math.abs,r=/left|center|right/,h=/top|center|bottom/,l=/[\+\-]\d+(\.[\d]+)?%?/,c=/^\w+/,u=/%$/,d=t.fn.position;t.position={scrollbarWidth:function(){if(void 0!==n)return n;var e,i,s=t("
"),o=s.children()[0];return t("body").append(s),e=o.offsetWidth,s.css("overflow","scroll"),i=o.offsetWidth,e===i&&(i=s[0].clientWidth),s.remove(),n=e-i},getScrollInfo:function(e){var i=e.isWindow||e.isDocument?"":e.element.css("overflow-x"),s=e.isWindow||e.isDocument?"":e.element.css("overflow-y"),n="scroll"===i||"auto"===i&&e.widthi?"left":e>0?"right":"center",vertical:0>r?"top":s>0?"bottom":"middle"};l>p&&p>a(e+i)&&(u.horizontal="center"),c>f&&f>a(s+r)&&(u.vertical="middle"),u.important=o(a(e),a(i))>o(a(s),a(r))?"horizontal":"vertical",n.using.call(this,t,u)}),h.offset(t.extend(D,{using:r}))})},t.ui.position={fit:{left:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollLeft:s.offset.left,a=s.width,r=t.left-e.collisionPosition.marginLeft,h=n-r,l=r+e.collisionWidth-a-n;e.collisionWidth>a?h>0&&0>=l?(i=t.left+h+e.collisionWidth-a-n,t.left+=h-i):t.left=l>0&&0>=h?n:h>l?n+a-e.collisionWidth:n:h>0?t.left+=h:l>0?t.left-=l:t.left=o(t.left-r,t.left)},top:function(t,e){var i,s=e.within,n=s.isWindow?s.scrollTop:s.offset.top,a=e.within.height,r=t.top-e.collisionPosition.marginTop,h=n-r,l=r+e.collisionHeight-a-n;e.collisionHeight>a?h>0&&0>=l?(i=t.top+h+e.collisionHeight-a-n,t.top+=h-i):t.top=l>0&&0>=h?n:h>l?n+a-e.collisionHeight:n:h>0?t.top+=h:l>0?t.top-=l:t.top=o(t.top-r,t.top)}},flip:{left:function(t,e){var i,s,n=e.within,o=n.offset.left+n.scrollLeft,r=n.width,h=n.isWindow?n.scrollLeft:n.offset.left,l=t.left-e.collisionPosition.marginLeft,c=l-h,u=l+e.collisionWidth-r-h,d="left"===e.my[0]?-e.elemWidth:"right"===e.my[0]?e.elemWidth:0,p="left"===e.at[0]?e.targetWidth:"right"===e.at[0]?-e.targetWidth:0,f=-2*e.offset[0];0>c?(i=t.left+d+p+f+e.collisionWidth-r-o,(0>i||a(c)>i)&&(t.left+=d+p+f)):u>0&&(s=t.left-e.collisionPosition.marginLeft+d+p+f-h,(s>0||u>a(s))&&(t.left+=d+p+f))},top:function(t,e){var i,s,n=e.within,o=n.offset.top+n.scrollTop,r=n.height,h=n.isWindow?n.scrollTop:n.offset.top,l=t.top-e.collisionPosition.marginTop,c=l-h,u=l+e.collisionHeight-r-h,d="top"===e.my[1],p=d?-e.elemHeight:"bottom"===e.my[1]?e.elemHeight:0,f="top"===e.at[1]?e.targetHeight:"bottom"===e.at[1]?-e.targetHeight:0,m=-2*e.offset[1];0>c?(s=t.top+p+f+m+e.collisionHeight-r-o,(0>s||a(c)>s)&&(t.top+=p+f+m)):u>0&&(i=t.top-e.collisionPosition.marginTop+p+f+m-h,(i>0||u>a(i))&&(t.top+=p+f+m))}},flipfit:{left:function(){t.ui.position.flip.left.apply(this,arguments),t.ui.position.fit.left.apply(this,arguments)},top:function(){t.ui.position.flip.top.apply(this,arguments),t.ui.position.fit.top.apply(this,arguments)}}}}(),t.ui.position,t.extend(t.expr[":"],{data:t.expr.createPseudo?t.expr.createPseudo(function(e){return function(i){return!!t.data(i,e)}}):function(e,i,s){return!!t.data(e,s[3])}}),t.fn.extend({disableSelection:function(){var t="onselectstart"in document.createElement("div")?"selectstart":"mousedown";return function(){return this.on(t+".ui-disableSelection",function(t){t.preventDefault()})}}(),enableSelection:function(){return this.off(".ui-disableSelection")}}),t.ui.focusable=function(i,s){var n,o,a,r,h,l=i.nodeName.toLowerCase();return"area"===l?(n=i.parentNode,o=n.name,i.href&&o&&"map"===n.nodeName.toLowerCase()?(a=t("img[usemap='#"+o+"']"),a.length>0&&a.is(":visible")):!1):(/^(input|select|textarea|button|object)$/.test(l)?(r=!i.disabled,r&&(h=t(i).closest("fieldset")[0],h&&(r=!h.disabled))):r="a"===l?i.href||s:s,r&&t(i).is(":visible")&&e(t(i)))},t.extend(t.expr[":"],{focusable:function(e){return t.ui.focusable(e,null!=t.attr(e,"tabindex"))}}),t.ui.focusable,t.fn.form=function(){return"string"==typeof this[0].form?this.closest("form"):t(this[0].form)},t.ui.formResetMixin={_formResetHandler:function(){var e=t(this);setTimeout(function(){var i=e.data("ui-form-reset-instances");t.each(i,function(){this.refresh()})})},_bindFormResetHandler:function(){if(this.form=this.element.form(),this.form.length){var t=this.form.data("ui-form-reset-instances")||[];t.length||this.form.on("reset.ui-form-reset",this._formResetHandler),t.push(this),this.form.data("ui-form-reset-instances",t)}},_unbindFormResetHandler:function(){if(this.form.length){var e=this.form.data("ui-form-reset-instances");e.splice(t.inArray(this,e),1),e.length?this.form.data("ui-form-reset-instances",e):this.form.removeData("ui-form-reset-instances").off("reset.ui-form-reset")}}},"1.7"===t.fn.jquery.substring(0,3)&&(t.each(["Width","Height"],function(e,i){function s(e,i,s,o){return t.each(n,function(){i-=parseFloat(t.css(e,"padding"+this))||0,s&&(i-=parseFloat(t.css(e,"border"+this+"Width"))||0),o&&(i-=parseFloat(t.css(e,"margin"+this))||0)}),i}var n="Width"===i?["Left","Right"]:["Top","Bottom"],o=i.toLowerCase(),a={innerWidth:t.fn.innerWidth,innerHeight:t.fn.innerHeight,outerWidth:t.fn.outerWidth,outerHeight:t.fn.outerHeight};t.fn["inner"+i]=function(e){return void 0===e?a["inner"+i].call(this):this.each(function(){t(this).css(o,s(this,e)+"px")})},t.fn["outer"+i]=function(e,n){return"number"!=typeof e?a["outer"+i].call(this,e):this.each(function(){t(this).css(o,s(this,e,!0,n)+"px")})}}),t.fn.addBack=function(t){return this.add(null==t?this.prevObject:this.prevObject.filter(t))}),t.ui.keyCode={BACKSPACE:8,COMMA:188,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,LEFT:37,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SPACE:32,TAB:9,UP:38},t.ui.escapeSelector=function(){var t=/([!"#$%&'()*+,./:;<=>?@[\]^`{|}~])/g;return function(e){return e.replace(t,"\\$1")}}(),t.fn.labels=function(){var e,i,s,n,o;return this[0].labels&&this[0].labels.length?this.pushStack(this[0].labels):(n=this.eq(0).parents("label"),s=this.attr("id"),s&&(e=this.eq(0).parents().last(),o=e.add(e.length?e.siblings():this.siblings()),i="label[for='"+t.ui.escapeSelector(s)+"']",n=n.add(o.find(i).addBack(i))),this.pushStack(n))},t.fn.scrollParent=function(e){var i=this.css("position"),s="absolute"===i,n=e?/(auto|scroll|hidden)/:/(auto|scroll)/,o=this.parents().filter(function(){var e=t(this);return s&&"static"===e.css("position")?!1:n.test(e.css("overflow")+e.css("overflow-y")+e.css("overflow-x"))}).eq(0);return"fixed"!==i&&o.length?o:t(this[0].ownerDocument||document)},t.extend(t.expr[":"],{tabbable:function(e){var i=t.attr(e,"tabindex"),s=null!=i;return(!s||i>=0)&&t.ui.focusable(e,s)}}),t.fn.extend({uniqueId:function(){var t=0;return function(){return this.each(function(){this.id||(this.id="ui-id-"+ ++t)})}}(),removeUniqueId:function(){return this.each(function(){/^ui-id-\d+$/.test(this.id)&&t(this).removeAttr("id")})}}),t.ui.ie=!!/msie [\w.]+/.exec(navigator.userAgent.toLowerCase());var n=!1;t(document).on("mouseup",function(){n=!1}),t.widget("ui.mouse",{version:"1.12.1",options:{cancel:"input, textarea, button, select, option",distance:1,delay:0},_mouseInit:function(){var e=this;this.element.on("mousedown."+this.widgetName,function(t){return e._mouseDown(t)}).on("click."+this.widgetName,function(i){return!0===t.data(i.target,e.widgetName+".preventClickEvent")?(t.removeData(i.target,e.widgetName+".preventClickEvent"),i.stopImmediatePropagation(),!1):void 0}),this.started=!1},_mouseDestroy:function(){this.element.off("."+this.widgetName),this._mouseMoveDelegate&&this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate)},_mouseDown:function(e){if(!n){this._mouseMoved=!1,this._mouseStarted&&this._mouseUp(e),this._mouseDownEvent=e;var i=this,s=1===e.which,o="string"==typeof this.options.cancel&&e.target.nodeName?t(e.target).closest(this.options.cancel).length:!1;return s&&!o&&this._mouseCapture(e)?(this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){i.mouseDelayMet=!0},this.options.delay)),this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(e)!==!1,!this._mouseStarted)?(e.preventDefault(),!0):(!0===t.data(e.target,this.widgetName+".preventClickEvent")&&t.removeData(e.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(t){return i._mouseMove(t)},this._mouseUpDelegate=function(t){return i._mouseUp(t)},this.document.on("mousemove."+this.widgetName,this._mouseMoveDelegate).on("mouseup."+this.widgetName,this._mouseUpDelegate),e.preventDefault(),n=!0,!0)):!0}},_mouseMove:function(e){if(this._mouseMoved){if(t.ui.ie&&(!document.documentMode||9>document.documentMode)&&!e.button)return this._mouseUp(e);if(!e.which)if(e.originalEvent.altKey||e.originalEvent.ctrlKey||e.originalEvent.metaKey||e.originalEvent.shiftKey)this.ignoreMissingWhich=!0;else if(!this.ignoreMissingWhich)return this._mouseUp(e)}return(e.which||e.button)&&(this._mouseMoved=!0),this._mouseStarted?(this._mouseDrag(e),e.preventDefault()):(this._mouseDistanceMet(e)&&this._mouseDelayMet(e)&&(this._mouseStarted=this._mouseStart(this._mouseDownEvent,e)!==!1,this._mouseStarted?this._mouseDrag(e):this._mouseUp(e)),!this._mouseStarted)},_mouseUp:function(e){this.document.off("mousemove."+this.widgetName,this._mouseMoveDelegate).off("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,e.target===this._mouseDownEvent.target&&t.data(e.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(e)),this._mouseDelayTimer&&(clearTimeout(this._mouseDelayTimer),delete this._mouseDelayTimer),this.ignoreMissingWhich=!1,n=!1,e.preventDefault()},_mouseDistanceMet:function(t){return Math.max(Math.abs(this._mouseDownEvent.pageX-t.pageX),Math.abs(this._mouseDownEvent.pageY-t.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return!0}}),t.ui.plugin={add:function(e,i,s){var n,o=t.ui[e].prototype;for(n in s)o.plugins[n]=o.plugins[n]||[],o.plugins[n].push([i,s[n]])},call:function(t,e,i,s){var n,o=t.plugins[e];if(o&&(s||t.element[0].parentNode&&11!==t.element[0].parentNode.nodeType))for(n=0;o.length>n;n++)t.options[o[n][0]]&&o[n][1].apply(t.element,i)}},t.widget("ui.resizable",t.ui.mouse,{version:"1.12.1",widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,classes:{"ui-resizable-se":"ui-icon ui-icon-gripsmall-diagonal-se"},containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:90,resize:null,start:null,stop:null},_num:function(t){return parseFloat(t)||0},_isNumber:function(t){return!isNaN(parseFloat(t))},_hasScroll:function(e,i){if("hidden"===t(e).css("overflow"))return!1;var s=i&&"left"===i?"scrollLeft":"scrollTop",n=!1;return e[s]>0?!0:(e[s]=1,n=e[s]>0,e[s]=0,n)},_create:function(){var e,i=this.options,s=this;this._addClass("ui-resizable"),t.extend(this,{_aspectRatio:!!i.aspectRatio,aspectRatio:i.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:i.helper||i.ghost||i.animate?i.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/^(canvas|textarea|input|select|button|img)$/i)&&(this.element.wrap(t("
").css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("ui-resizable",this.element.resizable("instance")),this.elementIsWrapper=!0,e={marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom"),marginLeft:this.originalElement.css("marginLeft")},this.element.css(e),this.originalElement.css("margin",0),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css(e),this._proportionallyResize()),this._setupHandles(),i.autoHide&&t(this.element).on("mouseenter",function(){i.disabled||(s._removeClass("ui-resizable-autohide"),s._handles.show())}).on("mouseleave",function(){i.disabled||s.resizing||(s._addClass("ui-resizable-autohide"),s._handles.hide())}),this._mouseInit()},_destroy:function(){this._mouseDestroy();var e,i=function(e){t(e).removeData("resizable").removeData("ui-resizable").off(".resizable").find(".ui-resizable-handle").remove()};return this.elementIsWrapper&&(i(this.element),e=this.element,this.originalElement.css({position:e.css("position"),width:e.outerWidth(),height:e.outerHeight(),top:e.css("top"),left:e.css("left")}).insertAfter(e),e.remove()),this.originalElement.css("resize",this.originalResizeStyle),i(this.originalElement),this},_setOption:function(t,e){switch(this._super(t,e),t){case"handles":this._removeHandles(),this._setupHandles();break;default:}},_setupHandles:function(){var e,i,s,n,o,a=this.options,r=this;if(this.handles=a.handles||(t(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se"),this._handles=t(),this.handles.constructor===String)for("all"===this.handles&&(this.handles="n,e,s,w,se,sw,ne,nw"),s=this.handles.split(","),this.handles={},i=0;s.length>i;i++)e=t.trim(s[i]),n="ui-resizable-"+e,o=t("
"),this._addClass(o,"ui-resizable-handle "+n),o.css({zIndex:a.zIndex}),this.handles[e]=".ui-resizable-"+e,this.element.append(o);this._renderAxis=function(e){var i,s,n,o;e=e||this.element;for(i in this.handles)this.handles[i].constructor===String?this.handles[i]=this.element.children(this.handles[i]).first().show():(this.handles[i].jquery||this.handles[i].nodeType)&&(this.handles[i]=t(this.handles[i]),this._on(this.handles[i],{mousedown:r._mouseDown})),this.elementIsWrapper&&this.originalElement[0].nodeName.match(/^(textarea|input|select|button)$/i)&&(s=t(this.handles[i],this.element),o=/sw|ne|nw|se|n|s/.test(i)?s.outerHeight():s.outerWidth(),n=["padding",/ne|nw|n/.test(i)?"Top":/se|sw|s/.test(i)?"Bottom":/^e$/.test(i)?"Right":"Left"].join(""),e.css(n,o),this._proportionallyResize()),this._handles=this._handles.add(this.handles[i])},this._renderAxis(this.element),this._handles=this._handles.add(this.element.find(".ui-resizable-handle")),this._handles.disableSelection(),this._handles.on("mouseover",function(){r.resizing||(this.className&&(o=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i)),r.axis=o&&o[1]?o[1]:"se")}),a.autoHide&&(this._handles.hide(),this._addClass("ui-resizable-autohide"))},_removeHandles:function(){this._handles.remove()},_mouseCapture:function(e){var i,s,n=!1;for(i in this.handles)s=t(this.handles[i])[0],(s===e.target||t.contains(s,e.target))&&(n=!0);return!this.options.disabled&&n},_mouseStart:function(e){var i,s,n,o=this.options,a=this.element;return this.resizing=!0,this._renderProxy(),i=this._num(this.helper.css("left")),s=this._num(this.helper.css("top")),o.containment&&(i+=t(o.containment).scrollLeft()||0,s+=t(o.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:i,top:s},this.size=this._helper?{width:this.helper.width(),height:this.helper.height()}:{width:a.width(),height:a.height()},this.originalSize=this._helper?{width:a.outerWidth(),height:a.outerHeight()}:{width:a.width(),height:a.height()},this.sizeDiff={width:a.outerWidth()-a.width(),height:a.outerHeight()-a.height()},this.originalPosition={left:i,top:s},this.originalMousePosition={left:e.pageX,top:e.pageY},this.aspectRatio="number"==typeof o.aspectRatio?o.aspectRatio:this.originalSize.width/this.originalSize.height||1,n=t(".ui-resizable-"+this.axis).css("cursor"),t("body").css("cursor","auto"===n?this.axis+"-resize":n),this._addClass("ui-resizable-resizing"),this._propagate("start",e),!0},_mouseDrag:function(e){var i,s,n=this.originalMousePosition,o=this.axis,a=e.pageX-n.left||0,r=e.pageY-n.top||0,h=this._change[o];return this._updatePrevProperties(),h?(i=h.apply(this,[e,a,r]),this._updateVirtualBoundaries(e.shiftKey),(this._aspectRatio||e.shiftKey)&&(i=this._updateRatio(i,e)),i=this._respectSize(i,e),this._updateCache(i),this._propagate("resize",e),s=this._applyChanges(),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),t.isEmptyObject(s)||(this._updatePrevProperties(),this._trigger("resize",e,this.ui()),this._applyChanges()),!1):!1},_mouseStop:function(e){this.resizing=!1;var i,s,n,o,a,r,h,l=this.options,c=this;return this._helper&&(i=this._proportionallyResizeElements,s=i.length&&/textarea/i.test(i[0].nodeName),n=s&&this._hasScroll(i[0],"left")?0:c.sizeDiff.height,o=s?0:c.sizeDiff.width,a={width:c.helper.width()-o,height:c.helper.height()-n},r=parseFloat(c.element.css("left"))+(c.position.left-c.originalPosition.left)||null,h=parseFloat(c.element.css("top"))+(c.position.top-c.originalPosition.top)||null,l.animate||this.element.css(t.extend(a,{top:h,left:r})),c.helper.height(c.size.height),c.helper.width(c.size.width),this._helper&&!l.animate&&this._proportionallyResize()),t("body").css("cursor","auto"),this._removeClass("ui-resizable-resizing"),this._propagate("stop",e),this._helper&&this.helper.remove(),!1},_updatePrevProperties:function(){this.prevPosition={top:this.position.top,left:this.position.left},this.prevSize={width:this.size.width,height:this.size.height}},_applyChanges:function(){var t={};return this.position.top!==this.prevPosition.top&&(t.top=this.position.top+"px"),this.position.left!==this.prevPosition.left&&(t.left=this.position.left+"px"),this.size.width!==this.prevSize.width&&(t.width=this.size.width+"px"),this.size.height!==this.prevSize.height&&(t.height=this.size.height+"px"),this.helper.css(t),t},_updateVirtualBoundaries:function(t){var e,i,s,n,o,a=this.options;o={minWidth:this._isNumber(a.minWidth)?a.minWidth:0,maxWidth:this._isNumber(a.maxWidth)?a.maxWidth:1/0,minHeight:this._isNumber(a.minHeight)?a.minHeight:0,maxHeight:this._isNumber(a.maxHeight)?a.maxHeight:1/0},(this._aspectRatio||t)&&(e=o.minHeight*this.aspectRatio,s=o.minWidth/this.aspectRatio,i=o.maxHeight*this.aspectRatio,n=o.maxWidth/this.aspectRatio,e>o.minWidth&&(o.minWidth=e),s>o.minHeight&&(o.minHeight=s),o.maxWidth>i&&(o.maxWidth=i),o.maxHeight>n&&(o.maxHeight=n)),this._vBoundaries=o},_updateCache:function(t){this.offset=this.helper.offset(),this._isNumber(t.left)&&(this.position.left=t.left),this._isNumber(t.top)&&(this.position.top=t.top),this._isNumber(t.height)&&(this.size.height=t.height),this._isNumber(t.width)&&(this.size.width=t.width)},_updateRatio:function(t){var e=this.position,i=this.size,s=this.axis;return this._isNumber(t.height)?t.width=t.height*this.aspectRatio:this._isNumber(t.width)&&(t.height=t.width/this.aspectRatio),"sw"===s&&(t.left=e.left+(i.width-t.width),t.top=null),"nw"===s&&(t.top=e.top+(i.height-t.height),t.left=e.left+(i.width-t.width)),t},_respectSize:function(t){var e=this._vBoundaries,i=this.axis,s=this._isNumber(t.width)&&e.maxWidth&&e.maxWidtht.width,a=this._isNumber(t.height)&&e.minHeight&&e.minHeight>t.height,r=this.originalPosition.left+this.originalSize.width,h=this.originalPosition.top+this.originalSize.height,l=/sw|nw|w/.test(i),c=/nw|ne|n/.test(i);return o&&(t.width=e.minWidth),a&&(t.height=e.minHeight),s&&(t.width=e.maxWidth),n&&(t.height=e.maxHeight),o&&l&&(t.left=r-e.minWidth),s&&l&&(t.left=r-e.maxWidth),a&&c&&(t.top=h-e.minHeight),n&&c&&(t.top=h-e.maxHeight),t.width||t.height||t.left||!t.top?t.width||t.height||t.top||!t.left||(t.left=null):t.top=null,t},_getPaddingPlusBorderDimensions:function(t){for(var e=0,i=[],s=[t.css("borderTopWidth"),t.css("borderRightWidth"),t.css("borderBottomWidth"),t.css("borderLeftWidth")],n=[t.css("paddingTop"),t.css("paddingRight"),t.css("paddingBottom"),t.css("paddingLeft")];4>e;e++)i[e]=parseFloat(s[e])||0,i[e]+=parseFloat(n[e])||0;return{height:i[0]+i[2],width:i[1]+i[3]}},_proportionallyResize:function(){if(this._proportionallyResizeElements.length)for(var t,e=0,i=this.helper||this.element;this._proportionallyResizeElements.length>e;e++)t=this._proportionallyResizeElements[e],this.outerDimensions||(this.outerDimensions=this._getPaddingPlusBorderDimensions(t)),t.css({height:i.height()-this.outerDimensions.height||0,width:i.width()-this.outerDimensions.width||0})},_renderProxy:function(){var e=this.element,i=this.options;this.elementOffset=e.offset(),this._helper?(this.helper=this.helper||t("
"),this._addClass(this.helper,this._helper),this.helper.css({width:this.element.outerWidth(),height:this.element.outerHeight(),position:"absolute",left:this.elementOffset.left+"px",top:this.elementOffset.top+"px",zIndex:++i.zIndex}),this.helper.appendTo("body").disableSelection()):this.helper=this.element +},_change:{e:function(t,e){return{width:this.originalSize.width+e}},w:function(t,e){var i=this.originalSize,s=this.originalPosition;return{left:s.left+e,width:i.width-e}},n:function(t,e,i){var s=this.originalSize,n=this.originalPosition;return{top:n.top+i,height:s.height-i}},s:function(t,e,i){return{height:this.originalSize.height+i}},se:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},sw:function(e,i,s){return t.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[e,i,s]))},ne:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[e,i,s]))},nw:function(e,i,s){return t.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[e,i,s]))}},_propagate:function(e,i){t.ui.plugin.call(this,e,[i,this.ui()]),"resize"!==e&&this._trigger(e,i,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),t.ui.plugin.add("resizable","animate",{stop:function(e){var i=t(this).resizable("instance"),s=i.options,n=i._proportionallyResizeElements,o=n.length&&/textarea/i.test(n[0].nodeName),a=o&&i._hasScroll(n[0],"left")?0:i.sizeDiff.height,r=o?0:i.sizeDiff.width,h={width:i.size.width-r,height:i.size.height-a},l=parseFloat(i.element.css("left"))+(i.position.left-i.originalPosition.left)||null,c=parseFloat(i.element.css("top"))+(i.position.top-i.originalPosition.top)||null;i.element.animate(t.extend(h,c&&l?{top:c,left:l}:{}),{duration:s.animateDuration,easing:s.animateEasing,step:function(){var s={width:parseFloat(i.element.css("width")),height:parseFloat(i.element.css("height")),top:parseFloat(i.element.css("top")),left:parseFloat(i.element.css("left"))};n&&n.length&&t(n[0]).css({width:s.width,height:s.height}),i._updateCache(s),i._propagate("resize",e)}})}}),t.ui.plugin.add("resizable","containment",{start:function(){var e,i,s,n,o,a,r,h=t(this).resizable("instance"),l=h.options,c=h.element,u=l.containment,d=u instanceof t?u.get(0):/parent/.test(u)?c.parent().get(0):u;d&&(h.containerElement=t(d),/document/.test(u)||u===document?(h.containerOffset={left:0,top:0},h.containerPosition={left:0,top:0},h.parentData={element:t(document),left:0,top:0,width:t(document).width(),height:t(document).height()||document.body.parentNode.scrollHeight}):(e=t(d),i=[],t(["Top","Right","Left","Bottom"]).each(function(t,s){i[t]=h._num(e.css("padding"+s))}),h.containerOffset=e.offset(),h.containerPosition=e.position(),h.containerSize={height:e.innerHeight()-i[3],width:e.innerWidth()-i[1]},s=h.containerOffset,n=h.containerSize.height,o=h.containerSize.width,a=h._hasScroll(d,"left")?d.scrollWidth:o,r=h._hasScroll(d)?d.scrollHeight:n,h.parentData={element:d,left:s.left,top:s.top,width:a,height:r}))},resize:function(e){var i,s,n,o,a=t(this).resizable("instance"),r=a.options,h=a.containerOffset,l=a.position,c=a._aspectRatio||e.shiftKey,u={top:0,left:0},d=a.containerElement,p=!0;d[0]!==document&&/static/.test(d.css("position"))&&(u=h),l.left<(a._helper?h.left:0)&&(a.size.width=a.size.width+(a._helper?a.position.left-h.left:a.position.left-u.left),c&&(a.size.height=a.size.width/a.aspectRatio,p=!1),a.position.left=r.helper?h.left:0),l.top<(a._helper?h.top:0)&&(a.size.height=a.size.height+(a._helper?a.position.top-h.top:a.position.top),c&&(a.size.width=a.size.height*a.aspectRatio,p=!1),a.position.top=a._helper?h.top:0),n=a.containerElement.get(0)===a.element.parent().get(0),o=/relative|absolute/.test(a.containerElement.css("position")),n&&o?(a.offset.left=a.parentData.left+a.position.left,a.offset.top=a.parentData.top+a.position.top):(a.offset.left=a.element.offset().left,a.offset.top=a.element.offset().top),i=Math.abs(a.sizeDiff.width+(a._helper?a.offset.left-u.left:a.offset.left-h.left)),s=Math.abs(a.sizeDiff.height+(a._helper?a.offset.top-u.top:a.offset.top-h.top)),i+a.size.width>=a.parentData.width&&(a.size.width=a.parentData.width-i,c&&(a.size.height=a.size.width/a.aspectRatio,p=!1)),s+a.size.height>=a.parentData.height&&(a.size.height=a.parentData.height-s,c&&(a.size.width=a.size.height*a.aspectRatio,p=!1)),p||(a.position.left=a.prevPosition.left,a.position.top=a.prevPosition.top,a.size.width=a.prevSize.width,a.size.height=a.prevSize.height)},stop:function(){var e=t(this).resizable("instance"),i=e.options,s=e.containerOffset,n=e.containerPosition,o=e.containerElement,a=t(e.helper),r=a.offset(),h=a.outerWidth()-e.sizeDiff.width,l=a.outerHeight()-e.sizeDiff.height;e._helper&&!i.animate&&/relative/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l}),e._helper&&!i.animate&&/static/.test(o.css("position"))&&t(this).css({left:r.left-n.left-s.left,width:h,height:l})}}),t.ui.plugin.add("resizable","alsoResize",{start:function(){var e=t(this).resizable("instance"),i=e.options;t(i.alsoResize).each(function(){var e=t(this);e.data("ui-resizable-alsoresize",{width:parseFloat(e.width()),height:parseFloat(e.height()),left:parseFloat(e.css("left")),top:parseFloat(e.css("top"))})})},resize:function(e,i){var s=t(this).resizable("instance"),n=s.options,o=s.originalSize,a=s.originalPosition,r={height:s.size.height-o.height||0,width:s.size.width-o.width||0,top:s.position.top-a.top||0,left:s.position.left-a.left||0};t(n.alsoResize).each(function(){var e=t(this),s=t(this).data("ui-resizable-alsoresize"),n={},o=e.parents(i.originalElement[0]).length?["width","height"]:["width","height","top","left"];t.each(o,function(t,e){var i=(s[e]||0)+(r[e]||0);i&&i>=0&&(n[e]=i||null)}),e.css(n)})},stop:function(){t(this).removeData("ui-resizable-alsoresize")}}),t.ui.plugin.add("resizable","ghost",{start:function(){var e=t(this).resizable("instance"),i=e.size;e.ghost=e.originalElement.clone(),e.ghost.css({opacity:.25,display:"block",position:"relative",height:i.height,width:i.width,margin:0,left:0,top:0}),e._addClass(e.ghost,"ui-resizable-ghost"),t.uiBackCompat!==!1&&"string"==typeof e.options.ghost&&e.ghost.addClass(this.options.ghost),e.ghost.appendTo(e.helper)},resize:function(){var e=t(this).resizable("instance");e.ghost&&e.ghost.css({position:"relative",height:e.size.height,width:e.size.width})},stop:function(){var e=t(this).resizable("instance");e.ghost&&e.helper&&e.helper.get(0).removeChild(e.ghost.get(0))}}),t.ui.plugin.add("resizable","grid",{resize:function(){var e,i=t(this).resizable("instance"),s=i.options,n=i.size,o=i.originalSize,a=i.originalPosition,r=i.axis,h="number"==typeof s.grid?[s.grid,s.grid]:s.grid,l=h[0]||1,c=h[1]||1,u=Math.round((n.width-o.width)/l)*l,d=Math.round((n.height-o.height)/c)*c,p=o.width+u,f=o.height+d,m=s.maxWidth&&p>s.maxWidth,g=s.maxHeight&&f>s.maxHeight,_=s.minWidth&&s.minWidth>p,v=s.minHeight&&s.minHeight>f;s.grid=h,_&&(p+=l),v&&(f+=c),m&&(p-=l),g&&(f-=c),/^(se|s|e)$/.test(r)?(i.size.width=p,i.size.height=f):/^(ne)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.top=a.top-d):/^(sw)$/.test(r)?(i.size.width=p,i.size.height=f,i.position.left=a.left-u):((0>=f-c||0>=p-l)&&(e=i._getPaddingPlusBorderDimensions(this)),f-c>0?(i.size.height=f,i.position.top=a.top-d):(f=c-e.height,i.size.height=f,i.position.top=a.top+o.height-f),p-l>0?(i.size.width=p,i.position.left=a.left-u):(p=l-e.width,i.size.width=p,i.position.left=a.left+o.width-p))}}),t.ui.resizable});/** + * Copyright (c) 2007 Ariel Flesler - aflesler ○ gmail • com | https://github.com/flesler + * Licensed under MIT + * @author Ariel Flesler + * @version 2.1.2 + */ +;(function(f){"use strict";"function"===typeof define&&define.amd?define(["jquery"],f):"undefined"!==typeof module&&module.exports?module.exports=f(require("jquery")):f(jQuery)})(function($){"use strict";function n(a){return!a.nodeName||-1!==$.inArray(a.nodeName.toLowerCase(),["iframe","#document","html","body"])}function h(a){return $.isFunction(a)||$.isPlainObject(a)?a:{top:a,left:a}}var p=$.scrollTo=function(a,d,b){return $(window).scrollTo(a,d,b)};p.defaults={axis:"xy",duration:0,limit:!0};$.fn.scrollTo=function(a,d,b){"object"=== typeof d&&(b=d,d=0);"function"===typeof b&&(b={onAfter:b});"max"===a&&(a=9E9);b=$.extend({},p.defaults,b);d=d||b.duration;var u=b.queue&&1=f[g]?0:Math.min(f[g],n));!a&&1-1){targetElements.on(evt+EVENT_NAMESPACE,function elementToggle(event){$.powerTip.toggle(this,event)})}else{targetElements.on(evt+EVENT_NAMESPACE,function elementOpen(event){$.powerTip.show(this,event)})}});$.each(options.closeEvents,function(idx,evt){if($.inArray(evt,options.openEvents)<0){targetElements.on(evt+EVENT_NAMESPACE,function elementClose(event){$.powerTip.hide(this,!isMouseEvent(event))})}});targetElements.on("keydown"+EVENT_NAMESPACE,function elementKeyDown(event){if(event.keyCode===27){$.powerTip.hide(this,true)}})}return targetElements};$.fn.powerTip.defaults={fadeInTime:200,fadeOutTime:100,followMouse:false,popupId:"powerTip",popupClass:null,intentSensitivity:7,intentPollInterval:100,closeDelay:100,placement:"n",smartPlacement:false,offset:10,mouseOnToPopup:false,manual:false,openEvents:["mouseenter","focus"],closeEvents:["mouseleave","blur"]};$.fn.powerTip.smartPlacementLists={n:["n","ne","nw","s"],e:["e","ne","se","w","nw","sw","n","s","e"],s:["s","se","sw","n"],w:["w","nw","sw","e","ne","se","n","s","w"],nw:["nw","w","sw","n","s","se","nw"],ne:["ne","e","se","n","s","sw","ne"],sw:["sw","w","nw","s","n","ne","sw"],se:["se","e","ne","s","n","nw","se"],"nw-alt":["nw-alt","n","ne-alt","sw-alt","s","se-alt","w","e"],"ne-alt":["ne-alt","n","nw-alt","se-alt","s","sw-alt","e","w"],"sw-alt":["sw-alt","s","se-alt","nw-alt","n","ne-alt","w","e"],"se-alt":["se-alt","s","sw-alt","ne-alt","n","nw-alt","e","w"]};$.powerTip={show:function apiShowTip(element,event){if(isMouseEvent(event)){trackMouse(event);session.previousX=event.pageX;session.previousY=event.pageY;$(element).data(DATA_DISPLAYCONTROLLER).show()}else{$(element).first().data(DATA_DISPLAYCONTROLLER).show(true,true)}return element},reposition:function apiResetPosition(element){$(element).first().data(DATA_DISPLAYCONTROLLER).resetPosition();return element},hide:function apiCloseTip(element,immediate){var displayController;immediate=element?immediate:true;if(element){displayController=$(element).first().data(DATA_DISPLAYCONTROLLER)}else if(session.activeHover){displayController=session.activeHover.data(DATA_DISPLAYCONTROLLER)}if(displayController){displayController.hide(immediate)}return element},toggle:function apiToggle(element,event){if(session.activeHover&&session.activeHover.is(element)){$.powerTip.hide(element,!isMouseEvent(event))}else{$.powerTip.show(element,event)}return element}};$.powerTip.showTip=$.powerTip.show;$.powerTip.closeTip=$.powerTip.hide;function CSSCoordinates(){var me=this;me.top="auto";me.left="auto";me.right="auto";me.bottom="auto";me.set=function(property,value){if($.isNumeric(value)){me[property]=Math.round(value)}}}function DisplayController(element,options,tipController){var hoverTimer=null,myCloseDelay=null;function openTooltip(immediate,forceOpen){cancelTimer();if(!element.data(DATA_HASACTIVEHOVER)){if(!immediate){session.tipOpenImminent=true;hoverTimer=setTimeout(function intentDelay(){hoverTimer=null;checkForIntent()},options.intentPollInterval)}else{if(forceOpen){element.data(DATA_FORCEDOPEN,true)}closeAnyDelayed();tipController.showTip(element)}}else{cancelClose()}}function closeTooltip(disableDelay){if(myCloseDelay){myCloseDelay=session.closeDelayTimeout=clearTimeout(myCloseDelay);session.delayInProgress=false}cancelTimer();session.tipOpenImminent=false;if(element.data(DATA_HASACTIVEHOVER)){element.data(DATA_FORCEDOPEN,false);if(!disableDelay){session.delayInProgress=true;session.closeDelayTimeout=setTimeout(function closeDelay(){session.closeDelayTimeout=null;tipController.hideTip(element);session.delayInProgress=false;myCloseDelay=null},options.closeDelay);myCloseDelay=session.closeDelayTimeout}else{tipController.hideTip(element)}}}function checkForIntent(){var xDifference=Math.abs(session.previousX-session.currentX),yDifference=Math.abs(session.previousY-session.currentY),totalDifference=xDifference+yDifference;if(totalDifference",{id:options.popupId});if($body.length===0){$body=$("body")}$body.append(tipElement);session.tooltips=session.tooltips?session.tooltips.add(tipElement):tipElement}if(options.followMouse){if(!tipElement.data(DATA_HASMOUSEMOVE)){$document.on("mousemove"+EVENT_NAMESPACE,positionTipOnCursor);$window.on("scroll"+EVENT_NAMESPACE,positionTipOnCursor);tipElement.data(DATA_HASMOUSEMOVE,true)}}function beginShowTip(element){element.data(DATA_HASACTIVEHOVER,true);tipElement.queue(function queueTipInit(next){showTip(element);next()})}function showTip(element){var tipContent;if(!element.data(DATA_HASACTIVEHOVER)){return}if(session.isTipOpen){if(!session.isClosing){hideTip(session.activeHover)}tipElement.delay(100).queue(function queueTipAgain(next){showTip(element);next()});return}element.trigger("powerTipPreRender");tipContent=getTooltipContent(element);if(tipContent){tipElement.empty().append(tipContent)}else{return}element.trigger("powerTipRender");session.activeHover=element;session.isTipOpen=true;tipElement.data(DATA_MOUSEONTOTIP,options.mouseOnToPopup);tipElement.addClass(options.popupClass);if(!options.followMouse||element.data(DATA_FORCEDOPEN)){positionTipOnElement(element);session.isFixedTipOpen=true}else{positionTipOnCursor()}if(!element.data(DATA_FORCEDOPEN)&&!options.followMouse){$document.on("click"+EVENT_NAMESPACE,function documentClick(event){var target=event.target;if(target!==element[0]){if(options.mouseOnToPopup){if(target!==tipElement[0]&&!$.contains(tipElement[0],target)){$.powerTip.hide()}}else{$.powerTip.hide()}}})}if(options.mouseOnToPopup&&!options.manual){tipElement.on("mouseenter"+EVENT_NAMESPACE,function tipMouseEnter(){if(session.activeHover){session.activeHover.data(DATA_DISPLAYCONTROLLER).cancel()}});tipElement.on("mouseleave"+EVENT_NAMESPACE,function tipMouseLeave(){if(session.activeHover){session.activeHover.data(DATA_DISPLAYCONTROLLER).hide()}})}tipElement.fadeIn(options.fadeInTime,function fadeInCallback(){if(!session.desyncTimeout){session.desyncTimeout=setInterval(closeDesyncedTip,500)}element.trigger("powerTipOpen")})}function hideTip(element){session.isClosing=true;session.isTipOpen=false;session.desyncTimeout=clearInterval(session.desyncTimeout);element.data(DATA_HASACTIVEHOVER,false);element.data(DATA_FORCEDOPEN,false);$document.off("click"+EVENT_NAMESPACE);tipElement.off(EVENT_NAMESPACE);tipElement.fadeOut(options.fadeOutTime,function fadeOutCallback(){var coords=new CSSCoordinates;session.activeHover=null;session.isClosing=false;session.isFixedTipOpen=false;tipElement.removeClass();coords.set("top",session.currentY+options.offset);coords.set("left",session.currentX+options.offset);tipElement.css(coords);element.trigger("powerTipClose")})}function positionTipOnCursor(){var tipWidth,tipHeight,coords,collisions,collisionCount;if(!session.isFixedTipOpen&&(session.isTipOpen||session.tipOpenImminent&&tipElement.data(DATA_HASMOUSEMOVE))){tipWidth=tipElement.outerWidth();tipHeight=tipElement.outerHeight();coords=new CSSCoordinates;coords.set("top",session.currentY+options.offset);coords.set("left",session.currentX+options.offset);collisions=getViewportCollisions(coords,tipWidth,tipHeight);if(collisions!==Collision.none){collisionCount=countFlags(collisions);if(collisionCount===1){if(collisions===Collision.right){coords.set("left",session.scrollLeft+session.windowWidth-tipWidth)}else if(collisions===Collision.bottom){coords.set("top",session.scrollTop+session.windowHeight-tipHeight)}}else{coords.set("left",session.currentX-tipWidth-options.offset);coords.set("top",session.currentY-tipHeight-options.offset)}}tipElement.css(coords)}}function positionTipOnElement(element){var priorityList,finalPlacement;if(options.smartPlacement||options.followMouse&&element.data(DATA_FORCEDOPEN)){priorityList=$.fn.powerTip.smartPlacementLists[options.placement];$.each(priorityList,function(idx,pos){var collisions=getViewportCollisions(placeTooltip(element,pos),tipElement.outerWidth(),tipElement.outerHeight());finalPlacement=pos;return collisions!==Collision.none})}else{placeTooltip(element,options.placement);finalPlacement=options.placement}tipElement.removeClass("w nw sw e ne se n s w se-alt sw-alt ne-alt nw-alt");tipElement.addClass(finalPlacement)}function placeTooltip(element,placement){var iterationCount=0,tipWidth,tipHeight,coords=new CSSCoordinates;coords.set("top",0);coords.set("left",0);tipElement.css(coords);do{tipWidth=tipElement.outerWidth();tipHeight=tipElement.outerHeight();coords=placementCalculator.compute(element,placement,tipWidth,tipHeight,options.offset);tipElement.css(coords)}while(++iterationCount<=5&&(tipWidth!==tipElement.outerWidth()||tipHeight!==tipElement.outerHeight()));return coords}function closeDesyncedTip(){var isDesynced=false,hasDesyncableCloseEvent=$.grep(["mouseleave","mouseout","blur","focusout"],function(eventType){return $.inArray(eventType,options.closeEvents)!==-1}).length>0;if(session.isTipOpen&&!session.isClosing&&!session.delayInProgress&&hasDesyncableCloseEvent){if(session.activeHover.data(DATA_HASACTIVEHOVER)===false||session.activeHover.is(":disabled")){isDesynced=true}else if(!isMouseOver(session.activeHover)&&!session.activeHover.is(":focus")&&!session.activeHover.data(DATA_FORCEDOPEN)){if(tipElement.data(DATA_MOUSEONTOTIP)){if(!isMouseOver(tipElement)){isDesynced=true}}else{isDesynced=true}}if(isDesynced){hideTip(session.activeHover)}}}this.showTip=beginShowTip;this.hideTip=hideTip;this.resetPosition=positionTipOnElement}function isSvgElement(element){return Boolean(window.SVGElement&&element[0]instanceof SVGElement)}function isMouseEvent(event){return Boolean(event&&$.inArray(event.type,MOUSE_EVENTS)>-1&&typeof event.pageX==="number")}function initTracking(){if(!session.mouseTrackingActive){session.mouseTrackingActive=true;getViewportDimensions();$(getViewportDimensions);$document.on("mousemove"+EVENT_NAMESPACE,trackMouse);$window.on("resize"+EVENT_NAMESPACE,trackResize);$window.on("scroll"+EVENT_NAMESPACE,trackScroll)}}function getViewportDimensions(){session.scrollLeft=$window.scrollLeft();session.scrollTop=$window.scrollTop();session.windowWidth=$window.width();session.windowHeight=$window.height()}function trackResize(){session.windowWidth=$window.width();session.windowHeight=$window.height()}function trackScroll(){var x=$window.scrollLeft(),y=$window.scrollTop();if(x!==session.scrollLeft){session.currentX+=x-session.scrollLeft;session.scrollLeft=x}if(y!==session.scrollTop){session.currentY+=y-session.scrollTop;session.scrollTop=y}}function trackMouse(event){session.currentX=event.pageX;session.currentY=event.pageY}function isMouseOver(element){var elementPosition=element.offset(),elementBox=element[0].getBoundingClientRect(),elementWidth=elementBox.right-elementBox.left,elementHeight=elementBox.bottom-elementBox.top;return session.currentX>=elementPosition.left&&session.currentX<=elementPosition.left+elementWidth&&session.currentY>=elementPosition.top&&session.currentY<=elementPosition.top+elementHeight}function getTooltipContent(element){var tipText=element.data(DATA_POWERTIP),tipObject=element.data(DATA_POWERTIPJQ),tipTarget=element.data(DATA_POWERTIPTARGET),targetElement,content;if(tipText){if($.isFunction(tipText)){tipText=tipText.call(element[0])}content=tipText}else if(tipObject){if($.isFunction(tipObject)){tipObject=tipObject.call(element[0])}if(tipObject.length>0){content=tipObject.clone(true,true)}}else if(tipTarget){targetElement=$("#"+tipTarget);if(targetElement.length>0){content=targetElement.html()}}return content}function getViewportCollisions(coords,elementWidth,elementHeight){var viewportTop=session.scrollTop,viewportLeft=session.scrollLeft,viewportBottom=viewportTop+session.windowHeight,viewportRight=viewportLeft+session.windowWidth,collisions=Collision.none;if(coords.topviewportBottom||Math.abs(coords.bottom-session.windowHeight)>viewportBottom){collisions|=Collision.bottom}if(coords.leftviewportRight){collisions|=Collision.left}if(coords.left+elementWidth>viewportRight||coords.right1)){a.preventDefault();var c=a.originalEvent.changedTouches[0],d=document.createEvent("MouseEvents");d.initMouseEvent(b,!0,!0,window,1,c.screenX,c.screenY,c.clientX,c.clientY,!1,!1,!1,!1,0,null),a.target.dispatchEvent(d)}}if(a.support.touch="ontouchend"in document,a.support.touch){var e,b=a.ui.mouse.prototype,c=b._mouseInit,d=b._mouseDestroy;b._touchStart=function(a){var b=this;!e&&b._mouseCapture(a.originalEvent.changedTouches[0])&&(e=!0,b._touchMoved=!1,f(a,"mouseover"),f(a,"mousemove"),f(a,"mousedown"))},b._touchMove=function(a){e&&(this._touchMoved=!0,f(a,"mousemove"))},b._touchEnd=function(a){e&&(f(a,"mouseup"),f(a,"mouseout"),this._touchMoved||f(a,"click"),e=!1)},b._mouseInit=function(){var b=this;b.element.bind({touchstart:a.proxy(b,"_touchStart"),touchmove:a.proxy(b,"_touchMove"),touchend:a.proxy(b,"_touchEnd")}),c.call(b)},b._mouseDestroy=function(){var b=this;b.element.unbind({touchstart:a.proxy(b,"_touchStart"),touchmove:a.proxy(b,"_touchMove"),touchend:a.proxy(b,"_touchEnd")}),d.call(b)}}}(jQuery);/*! SmartMenus jQuery Plugin - v1.1.0 - September 17, 2017 + * http://www.smartmenus.org/ + * Copyright Vasil Dinkov, Vadikom Web Ltd. http://vadikom.com; Licensed MIT */(function(t){"function"==typeof define&&define.amd?define(["jquery"],t):"object"==typeof module&&"object"==typeof module.exports?module.exports=t(require("jquery")):t(jQuery)})(function($){function initMouseDetection(t){var e=".smartmenus_mouse";if(mouseDetectionEnabled||t)mouseDetectionEnabled&&t&&($(document).off(e),mouseDetectionEnabled=!1);else{var i=!0,s=null,o={mousemove:function(t){var e={x:t.pageX,y:t.pageY,timeStamp:(new Date).getTime()};if(s){var o=Math.abs(s.x-e.x),a=Math.abs(s.y-e.y);if((o>0||a>0)&&2>=o&&2>=a&&300>=e.timeStamp-s.timeStamp&&(mouse=!0,i)){var n=$(t.target).closest("a");n.is("a")&&$.each(menuTrees,function(){return $.contains(this.$root[0],n[0])?(this.itemEnter({currentTarget:n[0]}),!1):void 0}),i=!1}}s=e}};o[touchEvents?"touchstart":"pointerover pointermove pointerout MSPointerOver MSPointerMove MSPointerOut"]=function(t){isTouchEvent(t.originalEvent)&&(mouse=!1)},$(document).on(getEventsNS(o,e)),mouseDetectionEnabled=!0}}function isTouchEvent(t){return!/^(4|mouse)$/.test(t.pointerType)}function getEventsNS(t,e){e||(e="");var i={};for(var s in t)i[s.split(" ").join(e+" ")+e]=t[s];return i}var menuTrees=[],mouse=!1,touchEvents="ontouchstart"in window,mouseDetectionEnabled=!1,requestAnimationFrame=window.requestAnimationFrame||function(t){return setTimeout(t,1e3/60)},cancelAnimationFrame=window.cancelAnimationFrame||function(t){clearTimeout(t)},canAnimate=!!$.fn.animate;return $.SmartMenus=function(t,e){this.$root=$(t),this.opts=e,this.rootId="",this.accessIdPrefix="",this.$subArrow=null,this.activatedItems=[],this.visibleSubMenus=[],this.showTimeout=0,this.hideTimeout=0,this.scrollTimeout=0,this.clickActivated=!1,this.focusActivated=!1,this.zIndexInc=0,this.idInc=0,this.$firstLink=null,this.$firstSub=null,this.disabled=!1,this.$disableOverlay=null,this.$touchScrollingSub=null,this.cssTransforms3d="perspective"in t.style||"webkitPerspective"in t.style,this.wasCollapsible=!1,this.init()},$.extend($.SmartMenus,{hideAll:function(){$.each(menuTrees,function(){this.menuHideAll()})},destroy:function(){for(;menuTrees.length;)menuTrees[0].destroy();initMouseDetection(!0)},prototype:{init:function(t){var e=this;if(!t){menuTrees.push(this),this.rootId=((new Date).getTime()+Math.random()+"").replace(/\D/g,""),this.accessIdPrefix="sm-"+this.rootId+"-",this.$root.hasClass("sm-rtl")&&(this.opts.rightToLeftSubMenus=!0);var i=".smartmenus";this.$root.data("smartmenus",this).attr("data-smartmenus-id",this.rootId).dataSM("level",1).on(getEventsNS({"mouseover focusin":$.proxy(this.rootOver,this),"mouseout focusout":$.proxy(this.rootOut,this),keydown:$.proxy(this.rootKeyDown,this)},i)).on(getEventsNS({mouseenter:$.proxy(this.itemEnter,this),mouseleave:$.proxy(this.itemLeave,this),mousedown:$.proxy(this.itemDown,this),focus:$.proxy(this.itemFocus,this),blur:$.proxy(this.itemBlur,this),click:$.proxy(this.itemClick,this)},i),"a"),i+=this.rootId,this.opts.hideOnClick&&$(document).on(getEventsNS({touchstart:$.proxy(this.docTouchStart,this),touchmove:$.proxy(this.docTouchMove,this),touchend:$.proxy(this.docTouchEnd,this),click:$.proxy(this.docClick,this)},i)),$(window).on(getEventsNS({"resize orientationchange":$.proxy(this.winResize,this)},i)),this.opts.subIndicators&&(this.$subArrow=$("").addClass("sub-arrow"),this.opts.subIndicatorsText&&this.$subArrow.html(this.opts.subIndicatorsText)),initMouseDetection()}if(this.$firstSub=this.$root.find("ul").each(function(){e.menuInit($(this))}).eq(0),this.$firstLink=this.$root.find("a").eq(0),this.opts.markCurrentItem){var s=/(index|default)\.[^#\?\/]*/i,o=/#.*/,a=window.location.href.replace(s,""),n=a.replace(o,"");this.$root.find("a").each(function(){var t=this.href.replace(s,""),i=$(this);(t==a||t==n)&&(i.addClass("current"),e.opts.markCurrentTree&&i.parentsUntil("[data-smartmenus-id]","ul").each(function(){$(this).dataSM("parent-a").addClass("current")}))})}this.wasCollapsible=this.isCollapsible()},destroy:function(t){if(!t){var e=".smartmenus";this.$root.removeData("smartmenus").removeAttr("data-smartmenus-id").removeDataSM("level").off(e),e+=this.rootId,$(document).off(e),$(window).off(e),this.opts.subIndicators&&(this.$subArrow=null)}this.menuHideAll();var i=this;this.$root.find("ul").each(function(){var t=$(this);t.dataSM("scroll-arrows")&&t.dataSM("scroll-arrows").remove(),t.dataSM("shown-before")&&((i.opts.subMenusMinWidth||i.opts.subMenusMaxWidth)&&t.css({width:"",minWidth:"",maxWidth:""}).removeClass("sm-nowrap"),t.dataSM("scroll-arrows")&&t.dataSM("scroll-arrows").remove(),t.css({zIndex:"",top:"",left:"",marginLeft:"",marginTop:"",display:""})),0==(t.attr("id")||"").indexOf(i.accessIdPrefix)&&t.removeAttr("id")}).removeDataSM("in-mega").removeDataSM("shown-before").removeDataSM("scroll-arrows").removeDataSM("parent-a").removeDataSM("level").removeDataSM("beforefirstshowfired").removeAttr("role").removeAttr("aria-hidden").removeAttr("aria-labelledby").removeAttr("aria-expanded"),this.$root.find("a.has-submenu").each(function(){var t=$(this);0==t.attr("id").indexOf(i.accessIdPrefix)&&t.removeAttr("id")}).removeClass("has-submenu").removeDataSM("sub").removeAttr("aria-haspopup").removeAttr("aria-controls").removeAttr("aria-expanded").closest("li").removeDataSM("sub"),this.opts.subIndicators&&this.$root.find("span.sub-arrow").remove(),this.opts.markCurrentItem&&this.$root.find("a.current").removeClass("current"),t||(this.$root=null,this.$firstLink=null,this.$firstSub=null,this.$disableOverlay&&(this.$disableOverlay.remove(),this.$disableOverlay=null),menuTrees.splice($.inArray(this,menuTrees),1))},disable:function(t){if(!this.disabled){if(this.menuHideAll(),!t&&!this.opts.isPopup&&this.$root.is(":visible")){var e=this.$root.offset();this.$disableOverlay=$('
').css({position:"absolute",top:e.top,left:e.left,width:this.$root.outerWidth(),height:this.$root.outerHeight(),zIndex:this.getStartZIndex(!0),opacity:0}).appendTo(document.body)}this.disabled=!0}},docClick:function(t){return this.$touchScrollingSub?(this.$touchScrollingSub=null,void 0):((this.visibleSubMenus.length&&!$.contains(this.$root[0],t.target)||$(t.target).closest("a").length)&&this.menuHideAll(),void 0)},docTouchEnd:function(){if(this.lastTouch){if(!(!this.visibleSubMenus.length||void 0!==this.lastTouch.x2&&this.lastTouch.x1!=this.lastTouch.x2||void 0!==this.lastTouch.y2&&this.lastTouch.y1!=this.lastTouch.y2||this.lastTouch.target&&$.contains(this.$root[0],this.lastTouch.target))){this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0);var t=this;this.hideTimeout=setTimeout(function(){t.menuHideAll()},350)}this.lastTouch=null}},docTouchMove:function(t){if(this.lastTouch){var e=t.originalEvent.touches[0];this.lastTouch.x2=e.pageX,this.lastTouch.y2=e.pageY}},docTouchStart:function(t){var e=t.originalEvent.touches[0];this.lastTouch={x1:e.pageX,y1:e.pageY,target:e.target}},enable:function(){this.disabled&&(this.$disableOverlay&&(this.$disableOverlay.remove(),this.$disableOverlay=null),this.disabled=!1)},getClosestMenu:function(t){for(var e=$(t).closest("ul");e.dataSM("in-mega");)e=e.parent().closest("ul");return e[0]||null},getHeight:function(t){return this.getOffset(t,!0)},getOffset:function(t,e){var i;"none"==t.css("display")&&(i={position:t[0].style.position,visibility:t[0].style.visibility},t.css({position:"absolute",visibility:"hidden"}).show());var s=t[0].getBoundingClientRect&&t[0].getBoundingClientRect(),o=s&&(e?s.height||s.bottom-s.top:s.width||s.right-s.left);return o||0===o||(o=e?t[0].offsetHeight:t[0].offsetWidth),i&&t.hide().css(i),o},getStartZIndex:function(t){var e=parseInt(this[t?"$root":"$firstSub"].css("z-index"));return!t&&isNaN(e)&&(e=parseInt(this.$root.css("z-index"))),isNaN(e)?1:e},getTouchPoint:function(t){return t.touches&&t.touches[0]||t.changedTouches&&t.changedTouches[0]||t},getViewport:function(t){var e=t?"Height":"Width",i=document.documentElement["client"+e],s=window["inner"+e];return s&&(i=Math.min(i,s)),i},getViewportHeight:function(){return this.getViewport(!0)},getViewportWidth:function(){return this.getViewport()},getWidth:function(t){return this.getOffset(t)},handleEvents:function(){return!this.disabled&&this.isCSSOn()},handleItemEvents:function(t){return this.handleEvents()&&!this.isLinkInMegaMenu(t)},isCollapsible:function(){return"static"==this.$firstSub.css("position")},isCSSOn:function(){return"inline"!=this.$firstLink.css("display")},isFixed:function(){var t="fixed"==this.$root.css("position");return t||this.$root.parentsUntil("body").each(function(){return"fixed"==$(this).css("position")?(t=!0,!1):void 0}),t},isLinkInMegaMenu:function(t){return $(this.getClosestMenu(t[0])).hasClass("mega-menu")},isTouchMode:function(){return!mouse||this.opts.noMouseOver||this.isCollapsible()},itemActivate:function(t,e){var i=t.closest("ul"),s=i.dataSM("level");if(s>1&&(!this.activatedItems[s-2]||this.activatedItems[s-2][0]!=i.dataSM("parent-a")[0])){var o=this;$(i.parentsUntil("[data-smartmenus-id]","ul").get().reverse()).add(i).each(function(){o.itemActivate($(this).dataSM("parent-a"))})}if((!this.isCollapsible()||e)&&this.menuHideSubMenus(this.activatedItems[s-1]&&this.activatedItems[s-1][0]==t[0]?s:s-1),this.activatedItems[s-1]=t,this.$root.triggerHandler("activate.smapi",t[0])!==!1){var a=t.dataSM("sub");a&&(this.isTouchMode()||!this.opts.showOnClick||this.clickActivated)&&this.menuShow(a)}},itemBlur:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&this.$root.triggerHandler("blur.smapi",e[0])},itemClick:function(t){var e=$(t.currentTarget);if(this.handleItemEvents(e)){if(this.$touchScrollingSub&&this.$touchScrollingSub[0]==e.closest("ul")[0])return this.$touchScrollingSub=null,t.stopPropagation(),!1;if(this.$root.triggerHandler("click.smapi",e[0])===!1)return!1;var i=$(t.target).is(".sub-arrow"),s=e.dataSM("sub"),o=s?2==s.dataSM("level"):!1,a=this.isCollapsible(),n=/toggle$/.test(this.opts.collapsibleBehavior),r=/link$/.test(this.opts.collapsibleBehavior),h=/^accordion/.test(this.opts.collapsibleBehavior);if(s&&!s.is(":visible")){if((!r||!a||i)&&(this.opts.showOnClick&&o&&(this.clickActivated=!0),this.itemActivate(e,h),s.is(":visible")))return this.focusActivated=!0,!1}else if(a&&(n||i))return this.itemActivate(e,h),this.menuHide(s),n&&(this.focusActivated=!1),!1;return this.opts.showOnClick&&o||e.hasClass("disabled")||this.$root.triggerHandler("select.smapi",e[0])===!1?!1:void 0}},itemDown:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&e.dataSM("mousedown",!0)},itemEnter:function(t){var e=$(t.currentTarget);if(this.handleItemEvents(e)){if(!this.isTouchMode()){this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0);var i=this;this.showTimeout=setTimeout(function(){i.itemActivate(e)},this.opts.showOnClick&&1==e.closest("ul").dataSM("level")?1:this.opts.showTimeout)}this.$root.triggerHandler("mouseenter.smapi",e[0])}},itemFocus:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&(!this.focusActivated||this.isTouchMode()&&e.dataSM("mousedown")||this.activatedItems.length&&this.activatedItems[this.activatedItems.length-1][0]==e[0]||this.itemActivate(e,!0),this.$root.triggerHandler("focus.smapi",e[0]))},itemLeave:function(t){var e=$(t.currentTarget);this.handleItemEvents(e)&&(this.isTouchMode()||(e[0].blur(),this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0)),e.removeDataSM("mousedown"),this.$root.triggerHandler("mouseleave.smapi",e[0]))},menuHide:function(t){if(this.$root.triggerHandler("beforehide.smapi",t[0])!==!1&&(canAnimate&&t.stop(!0,!0),"none"!=t.css("display"))){var e=function(){t.css("z-index","")};this.isCollapsible()?canAnimate&&this.opts.collapsibleHideFunction?this.opts.collapsibleHideFunction.call(this,t,e):t.hide(this.opts.collapsibleHideDuration,e):canAnimate&&this.opts.hideFunction?this.opts.hideFunction.call(this,t,e):t.hide(this.opts.hideDuration,e),t.dataSM("scroll")&&(this.menuScrollStop(t),t.css({"touch-action":"","-ms-touch-action":"","-webkit-transform":"",transform:""}).off(".smartmenus_scroll").removeDataSM("scroll").dataSM("scroll-arrows").hide()),t.dataSM("parent-a").removeClass("highlighted").attr("aria-expanded","false"),t.attr({"aria-expanded":"false","aria-hidden":"true"});var i=t.dataSM("level");this.activatedItems.splice(i-1,1),this.visibleSubMenus.splice($.inArray(t,this.visibleSubMenus),1),this.$root.triggerHandler("hide.smapi",t[0])}},menuHideAll:function(){this.showTimeout&&(clearTimeout(this.showTimeout),this.showTimeout=0);for(var t=this.opts.isPopup?1:0,e=this.visibleSubMenus.length-1;e>=t;e--)this.menuHide(this.visibleSubMenus[e]);this.opts.isPopup&&(canAnimate&&this.$root.stop(!0,!0),this.$root.is(":visible")&&(canAnimate&&this.opts.hideFunction?this.opts.hideFunction.call(this,this.$root):this.$root.hide(this.opts.hideDuration))),this.activatedItems=[],this.visibleSubMenus=[],this.clickActivated=!1,this.focusActivated=!1,this.zIndexInc=0,this.$root.triggerHandler("hideAll.smapi")},menuHideSubMenus:function(t){for(var e=this.activatedItems.length-1;e>=t;e--){var i=this.activatedItems[e].dataSM("sub");i&&this.menuHide(i)}},menuInit:function(t){if(!t.dataSM("in-mega")){t.hasClass("mega-menu")&&t.find("ul").dataSM("in-mega",!0);for(var e=2,i=t[0];(i=i.parentNode.parentNode)!=this.$root[0];)e++;var s=t.prevAll("a").eq(-1);s.length||(s=t.prevAll().find("a").eq(-1)),s.addClass("has-submenu").dataSM("sub",t),t.dataSM("parent-a",s).dataSM("level",e).parent().dataSM("sub",t);var o=s.attr("id")||this.accessIdPrefix+ ++this.idInc,a=t.attr("id")||this.accessIdPrefix+ ++this.idInc;s.attr({id:o,"aria-haspopup":"true","aria-controls":a,"aria-expanded":"false"}),t.attr({id:a,role:"group","aria-hidden":"true","aria-labelledby":o,"aria-expanded":"false"}),this.opts.subIndicators&&s[this.opts.subIndicatorsPos](this.$subArrow.clone())}},menuPosition:function(t){var e,i,s=t.dataSM("parent-a"),o=s.closest("li"),a=o.parent(),n=t.dataSM("level"),r=this.getWidth(t),h=this.getHeight(t),u=s.offset(),l=u.left,c=u.top,d=this.getWidth(s),m=this.getHeight(s),p=$(window),f=p.scrollLeft(),v=p.scrollTop(),b=this.getViewportWidth(),S=this.getViewportHeight(),g=a.parent().is("[data-sm-horizontal-sub]")||2==n&&!a.hasClass("sm-vertical"),M=this.opts.rightToLeftSubMenus&&!o.is("[data-sm-reverse]")||!this.opts.rightToLeftSubMenus&&o.is("[data-sm-reverse]"),w=2==n?this.opts.mainMenuSubOffsetX:this.opts.subMenusSubOffsetX,T=2==n?this.opts.mainMenuSubOffsetY:this.opts.subMenusSubOffsetY;if(g?(e=M?d-r-w:w,i=this.opts.bottomToTopSubMenus?-h-T:m+T):(e=M?w-r:d-w,i=this.opts.bottomToTopSubMenus?m-T-h:T),this.opts.keepInViewport){var y=l+e,I=c+i;if(M&&f>y?e=g?f-y+e:d-w:!M&&y+r>f+b&&(e=g?f+b-r-y+e:w-r),g||(S>h&&I+h>v+S?i+=v+S-h-I:(h>=S||v>I)&&(i+=v-I)),g&&(I+h>v+S+.49||v>I)||!g&&h>S+.49){var x=this;t.dataSM("scroll-arrows")||t.dataSM("scroll-arrows",$([$('')[0],$('')[0]]).on({mouseenter:function(){t.dataSM("scroll").up=$(this).hasClass("scroll-up"),x.menuScroll(t)},mouseleave:function(e){x.menuScrollStop(t),x.menuScrollOut(t,e)},"mousewheel DOMMouseScroll":function(t){t.preventDefault()}}).insertAfter(t));var A=".smartmenus_scroll";if(t.dataSM("scroll",{y:this.cssTransforms3d?0:i-m,step:1,itemH:m,subH:h,arrowDownH:this.getHeight(t.dataSM("scroll-arrows").eq(1))}).on(getEventsNS({mouseover:function(e){x.menuScrollOver(t,e)},mouseout:function(e){x.menuScrollOut(t,e)},"mousewheel DOMMouseScroll":function(e){x.menuScrollMousewheel(t,e)}},A)).dataSM("scroll-arrows").css({top:"auto",left:"0",marginLeft:e+(parseInt(t.css("border-left-width"))||0),width:r-(parseInt(t.css("border-left-width"))||0)-(parseInt(t.css("border-right-width"))||0),zIndex:t.css("z-index")}).eq(g&&this.opts.bottomToTopSubMenus?0:1).show(),this.isFixed()){var C={};C[touchEvents?"touchstart touchmove touchend":"pointerdown pointermove pointerup MSPointerDown MSPointerMove MSPointerUp"]=function(e){x.menuScrollTouch(t,e)},t.css({"touch-action":"none","-ms-touch-action":"none"}).on(getEventsNS(C,A))}}}t.css({top:"auto",left:"0",marginLeft:e,marginTop:i-m})},menuScroll:function(t,e,i){var s,o=t.dataSM("scroll"),a=t.dataSM("scroll-arrows"),n=o.up?o.upEnd:o.downEnd;if(!e&&o.momentum){if(o.momentum*=.92,s=o.momentum,.5>s)return this.menuScrollStop(t),void 0}else s=i||(e||!this.opts.scrollAccelerate?this.opts.scrollStep:Math.floor(o.step));var r=t.dataSM("level");if(this.activatedItems[r-1]&&this.activatedItems[r-1].dataSM("sub")&&this.activatedItems[r-1].dataSM("sub").is(":visible")&&this.menuHideSubMenus(r-1),o.y=o.up&&o.y>=n||!o.up&&n>=o.y?o.y:Math.abs(n-o.y)>s?o.y+(o.up?s:-s):n,t.css(this.cssTransforms3d?{"-webkit-transform":"translate3d(0, "+o.y+"px, 0)",transform:"translate3d(0, "+o.y+"px, 0)"}:{marginTop:o.y}),mouse&&(o.up&&o.y>o.downEnd||!o.up&&o.y0;t.dataSM("scroll-arrows").eq(i?0:1).is(":visible")&&(t.dataSM("scroll").up=i,this.menuScroll(t,!0))}e.preventDefault()},menuScrollOut:function(t,e){mouse&&(/^scroll-(up|down)/.test((e.relatedTarget||"").className)||(t[0]==e.relatedTarget||$.contains(t[0],e.relatedTarget))&&this.getClosestMenu(e.relatedTarget)==t[0]||t.dataSM("scroll-arrows").css("visibility","hidden"))},menuScrollOver:function(t,e){if(mouse&&!/^scroll-(up|down)/.test(e.target.className)&&this.getClosestMenu(e.target)==t[0]){this.menuScrollRefreshData(t);var i=t.dataSM("scroll"),s=$(window).scrollTop()-t.dataSM("parent-a").offset().top-i.itemH;t.dataSM("scroll-arrows").eq(0).css("margin-top",s).end().eq(1).css("margin-top",s+this.getViewportHeight()-i.arrowDownH).end().css("visibility","visible")}},menuScrollRefreshData:function(t){var e=t.dataSM("scroll"),i=$(window).scrollTop()-t.dataSM("parent-a").offset().top-e.itemH;this.cssTransforms3d&&(i=-(parseFloat(t.css("margin-top"))-i)),$.extend(e,{upEnd:i,downEnd:i+this.getViewportHeight()-e.subH})},menuScrollStop:function(t){return this.scrollTimeout?(cancelAnimationFrame(this.scrollTimeout),this.scrollTimeout=0,t.dataSM("scroll").step=1,!0):void 0},menuScrollTouch:function(t,e){if(e=e.originalEvent,isTouchEvent(e)){var i=this.getTouchPoint(e);if(this.getClosestMenu(i.target)==t[0]){var s=t.dataSM("scroll");if(/(start|down)$/i.test(e.type))this.menuScrollStop(t)?(e.preventDefault(),this.$touchScrollingSub=t):this.$touchScrollingSub=null,this.menuScrollRefreshData(t),$.extend(s,{touchStartY:i.pageY,touchStartTime:e.timeStamp});else if(/move$/i.test(e.type)){var o=void 0!==s.touchY?s.touchY:s.touchStartY;if(void 0!==o&&o!=i.pageY){this.$touchScrollingSub=t;var a=i.pageY>o;void 0!==s.up&&s.up!=a&&$.extend(s,{touchStartY:i.pageY,touchStartTime:e.timeStamp}),$.extend(s,{up:a,touchY:i.pageY}),this.menuScroll(t,!0,Math.abs(i.pageY-o))}e.preventDefault()}else void 0!==s.touchY&&((s.momentum=15*Math.pow(Math.abs(i.pageY-s.touchStartY)/(e.timeStamp-s.touchStartTime),2))&&(this.menuScrollStop(t),this.menuScroll(t),e.preventDefault()),delete s.touchY)}}},menuShow:function(t){if((t.dataSM("beforefirstshowfired")||(t.dataSM("beforefirstshowfired",!0),this.$root.triggerHandler("beforefirstshow.smapi",t[0])!==!1))&&this.$root.triggerHandler("beforeshow.smapi",t[0])!==!1&&(t.dataSM("shown-before",!0),canAnimate&&t.stop(!0,!0),!t.is(":visible"))){var e=t.dataSM("parent-a"),i=this.isCollapsible();if((this.opts.keepHighlighted||i)&&e.addClass("highlighted"),i)t.removeClass("sm-nowrap").css({zIndex:"",width:"auto",minWidth:"",maxWidth:"",top:"",left:"",marginLeft:"",marginTop:""});else{if(t.css("z-index",this.zIndexInc=(this.zIndexInc||this.getStartZIndex())+1),(this.opts.subMenusMinWidth||this.opts.subMenusMaxWidth)&&(t.css({width:"auto",minWidth:"",maxWidth:""}).addClass("sm-nowrap"),this.opts.subMenusMinWidth&&t.css("min-width",this.opts.subMenusMinWidth),this.opts.subMenusMaxWidth)){var s=this.getWidth(t);t.css("max-width",this.opts.subMenusMaxWidth),s>this.getWidth(t)&&t.removeClass("sm-nowrap").css("width",this.opts.subMenusMaxWidth)}this.menuPosition(t)}var o=function(){t.css("overflow","")};i?canAnimate&&this.opts.collapsibleShowFunction?this.opts.collapsibleShowFunction.call(this,t,o):t.show(this.opts.collapsibleShowDuration,o):canAnimate&&this.opts.showFunction?this.opts.showFunction.call(this,t,o):t.show(this.opts.showDuration,o),e.attr("aria-expanded","true"),t.attr({"aria-expanded":"true","aria-hidden":"false"}),this.visibleSubMenus.push(t),this.$root.triggerHandler("show.smapi",t[0])}},popupHide:function(t){this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0);var e=this;this.hideTimeout=setTimeout(function(){e.menuHideAll()},t?1:this.opts.hideTimeout)},popupShow:function(t,e){if(!this.opts.isPopup)return alert('SmartMenus jQuery Error:\n\nIf you want to show this menu via the "popupShow" method, set the isPopup:true option.'),void 0;if(this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0),this.$root.dataSM("shown-before",!0),canAnimate&&this.$root.stop(!0,!0),!this.$root.is(":visible")){this.$root.css({left:t,top:e});var i=this,s=function(){i.$root.css("overflow","")};canAnimate&&this.opts.showFunction?this.opts.showFunction.call(this,this.$root,s):this.$root.show(this.opts.showDuration,s),this.visibleSubMenus[0]=this.$root}},refresh:function(){this.destroy(!0),this.init(!0)},rootKeyDown:function(t){if(this.handleEvents())switch(t.keyCode){case 27:var e=this.activatedItems[0];if(e){this.menuHideAll(),e[0].focus();var i=e.dataSM("sub");i&&this.menuHide(i)}break;case 32:var s=$(t.target);if(s.is("a")&&this.handleItemEvents(s)){var i=s.dataSM("sub");i&&!i.is(":visible")&&(this.itemClick({currentTarget:t.target}),t.preventDefault())}}},rootOut:function(t){if(this.handleEvents()&&!this.isTouchMode()&&t.target!=this.$root[0]&&(this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0),!this.opts.showOnClick||!this.opts.hideOnClick)){var e=this;this.hideTimeout=setTimeout(function(){e.menuHideAll()},this.opts.hideTimeout)}},rootOver:function(t){this.handleEvents()&&!this.isTouchMode()&&t.target!=this.$root[0]&&this.hideTimeout&&(clearTimeout(this.hideTimeout),this.hideTimeout=0)},winResize:function(t){if(this.handleEvents()){if(!("onorientationchange"in window)||"orientationchange"==t.type){var e=this.isCollapsible();this.wasCollapsible&&e||(this.activatedItems.length&&this.activatedItems[this.activatedItems.length-1][0].blur(),this.menuHideAll()),this.wasCollapsible=e}}else if(this.$disableOverlay){var i=this.$root.offset();this.$disableOverlay.css({top:i.top,left:i.left,width:this.$root.outerWidth(),height:this.$root.outerHeight()})}}}}),$.fn.dataSM=function(t,e){return e?this.data(t+"_smartmenus",e):this.data(t+"_smartmenus")},$.fn.removeDataSM=function(t){return this.removeData(t+"_smartmenus")},$.fn.smartmenus=function(options){if("string"==typeof options){var args=arguments,method=options;return Array.prototype.shift.call(args),this.each(function(){var t=$(this).data("smartmenus");t&&t[method]&&t[method].apply(t,args)})}return this.each(function(){var dataOpts=$(this).data("sm-options")||null;if(dataOpts)try{dataOpts=eval("("+dataOpts+")")}catch(e){dataOpts=null,alert('ERROR\n\nSmartMenus jQuery init:\nInvalid "data-sm-options" attribute value syntax.')}new $.SmartMenus(this,$.extend({},$.fn.smartmenus.defaults,options,dataOpts))})},$.fn.smartmenus.defaults={isPopup:!1,mainMenuSubOffsetX:0,mainMenuSubOffsetY:0,subMenusSubOffsetX:0,subMenusSubOffsetY:0,subMenusMinWidth:"10em",subMenusMaxWidth:"20em",subIndicators:!0,subIndicatorsPos:"append",subIndicatorsText:"",scrollStep:30,scrollAccelerate:!0,showTimeout:250,hideTimeout:500,showDuration:0,showFunction:null,hideDuration:0,hideFunction:function(t,e){t.fadeOut(200,e)},collapsibleShowDuration:0,collapsibleShowFunction:function(t,e){t.slideDown(200,e)},collapsibleHideDuration:0,collapsibleHideFunction:function(t,e){t.slideUp(200,e)},showOnClick:!1,hideOnClick:!0,noMouseOver:!1,keepInViewport:!0,keepHighlighted:!0,markCurrentItem:!1,markCurrentTree:!0,rightToLeftSubMenus:!1,bottomToTopSubMenus:!1,collapsibleBehavior:"default"},$}); \ No newline at end of file diff --git a/docs/md__home_eurquhar_ipsec-mb_README.html b/docs/md__home_eurquhar_ipsec-mb_README.html new file mode 100644 index 00000000..6d2aeedd --- /dev/null +++ b/docs/md__home_eurquhar_ipsec-mb_README.html @@ -0,0 +1,566 @@ + + + + + + + +intel-ipsec-mb: README + + + + + + + + + +
+
+ + + + + + +
+
intel-ipsec-mb +
+
+
+ + + + + + + + +
+
+ + +
+ +
+ +
+
+
+
README
+
+
+

Coverity Status Linux Build Shared gcc Linux Build Static gcc Linux Build Shared clang Linux Build Static clang Linux Build Shared clang AESNI emulation FreeBSD Build Shared clang FreeBSD Build Shared gcc

+

+Intel(R) Multi-Buffer Crypto for IPsec Library

+

The library provides software crypto acceleration primarily targeting packet processing applications. It can be used for application such as: IPsec, TLS, Wireless (RAN), Cable or MPEG DRM.

+

The library is used as software crypto provider in DPDK, Intel(R) QAT Engine and FD.io.

+

Using crypto interfaces from the above frameworks gives freedom to change providers without subsequent application modifications. The library can also be used directly through its native API.

+

Key differentiating features:

+
    +
  • operation chaining (encryption and authentication)
    • +
  • advanced cryptographic pipelining
      +
    • job manager with scheduling and dispatching functions
      • +
    • API hides underlying implementation details from an application
      • +
    +
    • +
  • multi-buffer and function stitching innovations
    • +
  • low level implementations using latest instruction extensions
    • +
+

+Contents

+
    +
  1. Overview
    2. +
  2. Processor Extensions
    3. +
  3. Recommendations
    4. +
  4. Package Content
    5. +
  5. Compilation
    6. +
  6. Security Considerations & Options for Increased Security
    7. +
  7. Installation
    8. +
  8. Backwards compatibility
    9. +
  9. Disclaimer (ZUC, KASUMI, SNOW3G)
    10. +
  10. Legal Disclaimer
    11. +
+

+1\. Overview

+

Intel Multi-Buffer Crypto for IPsec Library is highly-optimized software implementations of the core cryptographic processing for IPsec, which provides industry-leading performance on a range of Intel(R) Processors.

+

For information on how the library works, see the Intel White Paper: "Fast Multi-buffer IPsec Implementations on Intel Architecture Processors". Jim Guilford, Sean Gulley, et. al.

+

The easiest way to find it is to search the Internet for the title and Intel White Paper.

+

Table 1. List of supported cipher algorithms and their implementations.

+----------------------------------------------------------------------+
+
| | Implementation |
+
| Encryption +-----------------------------------------------------|
+
| | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(5)|
+
|----------------+--------+--------+--------+--------+--------+--------|
+
| AES128-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES192-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES256-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES128-CCM | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES256-CCM | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES128-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES192-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES256-CBC | N | Y(1) | Y(3) | N | N | Y(6) |
+
| AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 |
+
| AES128-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| AES192-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| AES256-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 |
+
| NULL | Y | N | N | N | N | N |
+
| AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) |
+
| AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) |
+
| DES-DOCSIS | Y | N | N | N | Y x16 | N |
+
| 3DES | Y | N | N | N | Y x16 | N |
+
| DES | Y | N | N | N | Y x16 | N |
+
| KASUMI-F8 | Y | N | N | N | N | N |
+
| ZUC-EEA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| ZUC-EEA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| SNOW3G-UEA2 | N | Y x4 | Y | Y | Y x16 | Y x16 |
+
| AES128-CBCS(9) | N | Y(1) | Y(3) | N | N | Y(6) |
+
| Chacha20 | N | Y | Y | Y | Y | N |
+
| Chacha20 AEAD | N | Y | Y | Y | Y | N |
+
| SNOW-V | N | Y | Y | N | N | N |
+
| SNOW-V AEAD | N | Y | Y | N | N | N |
+
| PON-CRC-BIP | N | Y by8 | Y by8 | N | N | Y |
+
+----------------------------------------------------------------------+
+

Notes:
+ (1,2) - By default, decryption is by4 and encryption is x4.
+ On CPU's supporting GFNI, decryption is by8 and encryption is x8.
+ (3,4) - decryption is by8 and encryption is x8
+ (5) - AVX512 plus VAES, VPCLMULQDQ and GFNI extensions
+ (6) - decryption is by16 and encryption is x16
+ (7) - same as AES128-CBC for AVX, combines cipher and CRC32
+ (8) - decryption is by16 and encryption is x16
+ (9) - currently 1:9 crypt:skip pattern supported
+ (10) - by default, decryption and encryption are AVX by8.
+ On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16.
+

+

Legend:
+ byY - single buffer Y blocks at a time
+ xY - Y buffers at a time

+

As an example of how to read table 1 and 2, if one uses AVX512 interface to perform AES128-CBC encryption then there is no native AVX512 implementation for this cipher. In such case, the library uses best available implementation which is AVX for AES128-CBC.

+

Table 2. List of supported integrity algorithms and their implementations.

+-------------------------------------------------------------------------+
+
| | Implementation |
+
| Integrity +-----------------------------------------------------|
+
| | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(3)|
+
|-------------------+--------+--------+--------+--------+--------+--------|
+
| AES-XCBC-96 | N | Y x4 | Y x8 | N | N | Y x16 |
+
| HMAC-MD5-96 | Y(1) | Y x4x2 | Y x4x2 | Y x8x2 | N | N |
+
| HMAC-SHA1-96 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-224_112 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N |
+
| SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N |
+
| AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| NULL | Y | N | N | N | N | N |
+
| AES128-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES256-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES128-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| AES256-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 |
+
| KASUMI-F9 | Y | N | N | N | N | N |
+
| ZUC-EIA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| ZUC-EIA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 |
+
| SNOW3G-UIA2(8) | N | Y by4 | Y by4 | N | Y by32 | Y by32 |
+
| DOCSIS-CRC32(4) | N | Y | Y | N | Y | Y |
+
| HEC | N | Y | Y | N | N | N |
+
| POLY1305 | Y | N | N | N | Y | Y |
+
| POLY1305 AEAD | Y | N | N | N | Y | Y |
+
| SNOW-V AEAD | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| GHASH | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 |
+
| CRC(6) | N | Y by8 | Y by8 | N | N | Y by16 |
+
| PON-CRC-BIP(7) | N | Y | Y | N | N | Y |
+
+-------------------------------------------------------------------------+
+

Notes:
+ (1) - MD5 over one block implemented in C
+ (2) - Implementation using SHANI extensions is x2
+ (3) - AVX512 plus VAES, VPCLMULQDQ, GFNI and IFMA extensions
+ (4) - used only with AES256-DOCSIS and AES128-DOCSIS ciphers
+ (5) - x8 on selected CPU's supporting GFNI
+ (6) - Supported CRC types:

    +
  • CRC32: Ethernet FCS, SCTP, WIMAX OFDMA
    +
    • +
  • CRC24: LTE A, LTE B
    +
    • +
  • CRC16: X25, FP data
    +
    • +
  • CRC11: FP header
    +
    • +
  • CRC10: IUUP data
    +
    • +
  • CRC8: WIMAX OFDMA HCS
    +
    • +
  • CRC7: FP header
    +
    • +
  • CRC6: IUUP header
    + (7) - used only with PON-AES128-CTR cipher
    + (8) - x4/x16 for init keystream generation, then by4/by32
    +
    • +
+

Legend:
+ byY- single buffer Y blocks at a time
+ xY- Y buffers at a time
+

+

Table 3. Encryption and integrity algorithm combinations

+---------------------------------------------------------------------+
+
| Encryption | Allowed Integrity Algorithms |
+
|---------------+-----------------------------------------------------|
+
| AES128-GCM | AES128-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES192-GCM | AES192-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES256-GCM | AES256-GMAC |
+
|---------------+-----------------------------------------------------|
+
| AES128-CCM | AES128-CCM |
+
|---------------+-----------------------------------------------------|
+
| AES256-CCM | AES256-CCM |
+
|---------------+-----------------------------------------------------|
+
| AES128-CBC, | AES-XCBC-96, |
+
| AES192-CBC, | HMAC-SHA1-96, HMAC-SHA2-224_112, HMAC-SHA2-256_128, |
+
| AES256-CBC, | HMAC-SHA2-384_192, HMAC-SHA2-512_256, |
+
| AES128-CTR, | AES128-CMAC-96, |
+
| AES192-CTR, | NULL, |
+
| AES256-CTR, | KASUMI-F9, |
+
| AES128-ECB, | ZUC-EIA3, ZUC-EIA3-256, |
+
| AES192-ECB, | SNOW3G-UIA3, |
+
| AES256-ECB, | POLY1305, |
+
| NULL, | AES128-GMAC, AES192-GMAC, AES256-GMAC, GHASH |
+
| AES128-DOCSIS,| |
+
| AES256-DOCSIS,| |
+
| DES-DOCSIS, | |
+
| 3DES, | |
+
| DES, | |
+
| Chacha20, | |
+
| KASUMI-F8, | |
+
| ZUC-EEA3, | |
+
| ZUC-EEA3-256, | |
+
| SNOW3G-UEA3 | |
+
| SNOW-V | |
+
|---------------+-----------------------------------------------------|
+
| AES128-DOCSIS,| DOCSIS-CRC32 |
+
| AES256-DOCSIS | |
+
|---------------+-----------------------------------------------------|
+
| PON-AES128-CTR| PON-CRC-BIP |
+
|---------------+-----------------------------------------------------|
+
| CHACHA20 AEAD | POLY1305 AEAD |
+
+---------------+-----------------------------------------------------+
+
| SNOW-V AEAD | SNOW-V AEAD (GHASH) |
+
+---------------+-----------------------------------------------------+
+

+2\. Processor Extensions

+

Table 4. Processor extensions used in the library

+-------------------------------------------------------------------------+
+
| Algorithm | Interface | Extensions |
+
|-------------------+-----------+-----------------------------------------|
+
| HMAC-SHA1-96, | AVX512 | AVX512F, AVX512BW, AVX512VL |
+
| HMAC-SHA2-224_112,| | |
+
| HMAC-SHA2-256_128,| | |
+
| HMAC-SHA2-384_192,| | |
+
| HMAC-SHA2-512_256 | | |
+
|-------------------+-----------+-----------------------------------------|
+
| DES, 3DES, | AVX512 | AVX512F, AVX512BW |
+
| DOCSIS-DES | | |
+
|-------------------+-----------+-----------------------------------------|
+
| HMAC-SHA1-96, | SSE | SHANI |
+
| HMAC-SHA2-224_112,| | - presence is autodetected and library |
+
| HMAC-SHA2-256_128,| | falls back to SSE implementation |
+
| HMAC-SHA2-384_192,| | if not present |
+
| HMAC-SHA2-512_256 | | |
+
+-------------------+-----------+-----------------------------------------+
+

+3\. Recommendations

+

Legacy or to be avoided algorithms listed in the table below are implemented in the library in order to support legacy applications. Please use corresponding alternative algorithms instead.

+--------------------------------------------------------------+
+
| # | Algorithm | Recommendation | Alternative |
+
|---+---------------------+----------------+-------------------|
+
| 1 | DES encryption | Avoid | AES encryption |
+
|---+---------------------+----------------+-------------------|
+
| 2 | 3DES encryption | Avoid | AES encryption |
+
|---+---------------------+----------------+-------------------|
+
| 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA256 |
+
|---+---------------------+----------------+-------------------|
+
| 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR |
+
|---+---------------------+----------------+-------------------|
+
| 3 | HMAC-SHA1 integrity | Avoid | HMAC-SHA256 |
+
+--------------------------------------------------------------+
+

Intel(R) Multi-Buffer Crypto for IPsec Library depends on C library and it is recommended to use its latest version.

+

Applications using the Intel(R) Multi-Buffer Crypto for IPsec Library rely on Operating System to provide process isolation. As the result, it is recommended to use latest Operating System patches and security updates.

+

+4\. Package Content

+
    +
  • test - Library test applications
    • +
  • perf - Library performance application
    • +
  • lib - Library source files
    • +
  • lib/sse - Intel(R) SSE optimized routines
    • +
  • lib/avx - Intel(R) AVX optimized routines
    • +
  • lib/avx2 - Intel(R) AVX2 optimized routines
    • +
  • lib/avx512 - Intel(R) AVX512 optimized routines
    • +
  • lib/no-aesni - Non-AESNI accelerated routines
    • +
+

Note:
+ There is just one branch used in the project. All development is done on the master branch.
+ Code taken from the tip of the master branch should not be considered fit for production.
+

+

Refer to the releases tab for stable code versions:
+ https://github.com/intel/intel-ipsec-mb/releases

+

+5\. Compilation

+

+Linux (64-bit only)

+

Required tools:
+

    +
  • GNU make
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 4.8.3 (or newer)
    +
    • +
+

Shared library:
+ > make

+

Static library:
+ > make SHARED=n

+

Clean the build:
+ > make clean
+ or
+ > make clean SHARED=n

+

Build with debugging information:
+ > make DEBUG=y

+

Build with AESNI emulation support (disabled by default):
+ > make AESNI_EMU=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > make help

+

+Windows MSVS (x64 only)

+

Required tools:
+

    +
  • Microsoft (R) Visual Studio 2015:
    +
      +
    • NMAKE: Microsoft (R) Program Maintenance Utility Version 14.00.24210.0
      +
      • +
    • CL: Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24215.1 for x64
      +
      • +
    • LIB: Microsoft (R) Library Manager Version 14.00.24215.1
      +
      • +
    • LINK: Microsoft (R) Incremental Linker Version 14.00.24215.1
      +
      • +
    • Note: Building on later versions should work but is not verified
      +
      • +
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
+

Shared library (DLL):
+ > nmake /f win_x64.mak

+

Static library:
+ > nmake /f win_x64.mak SHARED=n

+

Clean the build:
+ > nmake /f win_x64.mak clean
+ or
+ > nmake /f win_x64.mak clean SHARED=n

+

Build without safety features:
+

    +
  • SAFE_DATA clears sensitive information stored temporarily on stack, registers or internal data structures
    +
    • +
  • SAFE_PARAM adds extra checks on input parameters
    +
    • +
  • SAFE_LOOKUP uses constant-time lookups (enabled by default)
    +
    • +
  • SAFE_OPTIONS additional option to disable all safe options. Enabled by default.
    + Disable to turn off: SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP.
    +
    • +
+

> nmake /f win_x64.mak SAFE_DATA=n SAFE_PARAM=n > nmake /f win_x64.mak SAFE_OPTIONS=n

+

Build with debugging information:
+ > nmake /f win_x64.mak DEBUG=y

+

Build with AESNI emulation support (disabled by default):
+ > nmake /f win_x64.mak AESNI_EMU=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > nmake /f win_x64.mak help

+

+Windows Mingw-w64 (64-bit only)

+

Required tools:
+

    +
  • GNU mingw32-make.exe
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 10.3.0 (or newer)
    • +
+

Shared library:
+ > mingw32-make.exe

+

Static library:
+ > mingw32-make.exe SHARED=n

+

Clean the build:
+ > mingw32-make.exe clean
+ or
+ > mingw32-make.exe clean SHARED=n

+

Build with debugging information:
+ > mingw32-make.exe DEBUG=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > mingw32-make.exe help

+

+FreeBSD (64-bit only)

+

Required tools:
+

    +
  • GNU make
    +
    • +
  • NASM version 2.14 (or newer)
    +
    • +
  • gcc (GCC) 4.8.3 (or newer) / clang 5.0 (or newer)
    +
    • +
+

Shared library:
+ > gmake

+

Static library:
+ > gmake SHARED=n

+

Clean the build:
+ > gmake clean
+ or
+ > gmake clean SHARED=n

+

Build with debugging information:
+ > gmake DEBUG=y

+

Note: Building with debugging information is not advised for production use.

+

For more build options and their explanation run:
+ > gmake help

+

+6\. Security Considerations & Options for Increased Security

+

+Security Considerations

+

The security of a system that uses cryptography depends on the strength of the cryptographic algorithms as well as the strength of the keys. Cryptographic key strength is dependent on several factors, with some of the most important factors including the length of the key, the entropy of the key bits, and maintaining the secrecy of the key.

+

The selection of an appropriate algorithm and mode of operation critically affects the security of a system. Appropriate selection criteria is beyond the scope of this document and should be determined based upon usage, appropriate standards and consultation with a cryptographic expert. This library includes some algorithms, which are considered cryptographically weak and are included only for legacy and interoperability reasons. See the "Recommendations" section for more details.

+

Secure creation of key material is not a part of this library. This library assumes that cryptographic keys have been created using approved methods with an appropriate and secure entropy source. Users of this library are referred to NIST SP800-133 Revision 1, Recommendation for Cryptographic Key Generation, found at https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-133r1.pdf

+

Even with the use of strong cryptographic algorithms and robustly generated keys, software implementations of cryptographic algorithms may be attacked at the implementation through cache-timing attacks, buffer-over-reads, and other software vulnerabilities. Counter-measures against these types of attacks are possible but require additional processing cycles. Whether a particular system should provide such counter-measures depends on the threats to that system, and cannot be determined by a general library such as this one. In order to provide the most flexible implementation, this library allows certain counter-measures to be enabled or disabled at compile time. These options are listed below as the "Options for Increased Security" and are enabled through various build flags.

+

+Options for Increased Security

+

There are three build options that are used to increase safety in the code and help protect external functions from incorrect input data. The SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP options are enabled by default. Due to the potential performance impact associated to the extra code, these can be disabled by setting the parameter equal to "n" (e.g. make SAFE_LOOKUP=n).

+

No specific code has been added, and no specific validation or security tests have been performed to help protect against or check for side-channel attacks.

+

+SAFE_DATA

+

Stack and registers containing sensitive information, such as keys or IVs, are cleared upon completion of a function call.

+

+SAFE_PARAM

+

Input parameters are checked, looking generally for NULL pointers or an incorrect input length.

+

+SAFE_LOOKUP

+

Lookups which depend on sensitive information are implemented with constant time functions.

+

Algorithms where these constant time functions are used are the following:
+

    +
  • AESNI emulation
    +
    • +
  • DES: SSE, AVX and AVX2 implementations
    +
    • +
  • KASUMI: all architectures
    +
    • +
  • SNOW3G: all architectures
    +
    • +
+

If SAFE_LOOKUP is not enabled in the build (e.g. make SAFE_LOOKUP=n) then the algorithms listed above may be susceptible to timing attacks which could expose the cryptographic key.

+

+SAFE_OPTIONS

+

SAFE_OPTIONS is a parameter that can be used to disable/enable all supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). It is set to y by default and all safe options are enabled. SAFE_OPTIONS=n disables all safe options.

+

+Security API

+

Force clearing/zeroing of memory

IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size);
+

To assist in clearing sensitive application data such as keys, plaintext etc. the library provides the imb_clear_mem() API. This API zeros _'size'_ bytes of memory pointed to by _'mem'_ followed by the sfence instruction to ensure memory is cleared before the function returns.

+

+Galois Counter Mode (GCM) TAG Size

+

The library GCM and GMAC implementation provides flexibility as to tag size selection. As explained in NIST Special Publication 800-38D section 5.2.1.2 and Appendix C, using tag sizes shorter than 96 bits can be insecure. Please refer to the aforementioned sections to understand the details, trade offs and mitigations of using shorter tag sizes.

+

+7\. Installation

+

+Linux (64-bit only)

+

First compile the library and then install:
+ > make
+ > sudo make install

+

To uninstall the library run:
+ > sudo make uninstall

+

If you want to change install location then define PREFIX:
+ > sudo make install PREFIX=<path>

+

If there is no need to run ldconfig at install stage please use NOLDCONFIG=y option:
+ > sudo make install NOLDCONFIG=y

+

If library was compiled as an archive (not a default option) then install it using SHARED=n option:
+ > sudo make install SHARED=n

+

+Windows (x64 only)

+

First compile the library and then install from a command prompt in administrator mode:
+ > nmake /f win_x64.mak
+ > nmake /f win_x64.mak install

+

To uninstall the library run:
+ > nmake /f win_x64.mak uninstall

+

If you want to change install location then define PREFIX (default C:\Program Files):
+ > nmake /f win_x64.mak install PREFIX=<path>

+

If library was compiled as a static library (not a default option) then install it using SHARED=n option:
+ > nmake /f win_x64.mak install SHARED=n

+

+FreeBSD (64-bit only)

+

First compile the library and then install:
+ > gmake
+ > sudo gmake install

+

To uninstall the library run:
+ > sudo gmake uninstall

+

If you want to change install location then define PREFIX:
+ > sudo gmake install PREFIX=<path>

+

If there is no need to run ldconfig at install stage please use NOLDCONFIG=y option:
+ > sudo gmake install NOLDCONFIG=y

+

If library was compiled as an archive (not a default option) then install it using SHARED=n option:
+ > sudo gmake install SHARED=n

+

+8\. Backwards compatibility

+

In version 0.54, some symbols have been renamed to avoid too generic names (such as cipher modes or hash algorithms).

+

When building an application and linking it against the IPSec Multi Buffer library, by default the old symbols (up to v0.53) are exported, to maintain backwards compatibility.

+

Applications should move to the new API as soon as possible, as the old symbols are marked as deprecated and will be removed in a future release.

+

For applications which face symbol conflicts due to these old generic names, they should be compiled with the flag -DNO_COMPAT_IMB_API_053, which will not export the old symbols.

+

+9\. Disclaimer (ZUC, KASUMI, SNOW3G)

+

Please note that cryptographic material, such as ciphering algorithms, may be subject to national regulations. What is more, use of some algorithms in real networks and production equipment can be subject to agreement or licensing by the GSMA and/or the ETSI.

+

For more details please see:
+

+

+10\. Legal Disclaimer

+

THIS SOFTWARE IS PROVIDED BY INTEL"AS IS". NO LICENSE, EXPRESS OR
+ IMPLIED, BY ESTOPPEL OR OTHERWISE, TO ANY INTELLECTUAL PROPERTY RIGHTS
+ ARE GRANTED THROUGH USE. EXCEPT AS PROVIDED IN INTEL'S TERMS AND
+ CONDITIONS OF SALE, INTEL ASSUMES NO LIABILITY WHATSOEVER AND INTEL
+ DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR
+ USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO
+ FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT
+ OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT.
+

+
+
+
IMB_DLL_EXPORT
#define IMB_DLL_EXPORT
Definition: intel-ipsec-mb.h:86
+
imb_clear_mem
IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size)
Force clearing/zeroing of memory.
+ +
+Generated by   +doxygen + 1.8.17 +
+ + diff --git a/docs/menu.js b/docs/menu.js new file mode 100644 index 00000000..433c15b8 --- /dev/null +++ b/docs/menu.js @@ -0,0 +1,50 @@ +/* + @licstart The following is the entire license notice for the + JavaScript code in this file. + + Copyright (C) 1997-2017 by Dimitri van Heesch + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + @licend The above is the entire license notice + for the JavaScript code in this file + */ +function initMenu(relPath,searchEnabled,serverSide,searchPage,search) { + function makeTree(data,relPath) { + var result=''; + if ('children' in data) { + result+=''; + } + return result; + } + + $('#main-nav').append(makeTree(menudata,relPath)); + $('#main-nav').children(':first').addClass('sm sm-dox').attr('id','main-menu'); + if (searchEnabled) { + if (serverSide) { + $('#main-menu').append('
    • '); + } else { + $('#main-menu').append('
    • '); + } + } + $('#main-menu').smartmenus(); +} +/* @license-end */ diff --git a/docs/menudata.js b/docs/menudata.js new file mode 100644 index 00000000..9058e7b3 --- /dev/null +++ b/docs/menudata.js @@ -0,0 +1,126 @@ +/* +@licstart The following is the entire license notice for the +JavaScript code in this file. + +Copyright (C) 1997-2019 by Dimitri van Heesch + +This program is free software; you can redistribute it and/or modify +it under the terms of version 2 of the GNU General Public License as published by +the Free Software Foundation + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +@licend The above is the entire license notice +for the JavaScript code in this file +*/ +var menudata={children:[ +{text:"Main Page",url:"index.html"}, +{text:"Data Structures",url:"annotated.html",children:[ +{text:"Data Structures",url:"annotated.html"}, +{text:"Data Structure Index",url:"classes.html"}, +{text:"Data Fields",url:"functions.html",children:[ +{text:"All",url:"functions.html",children:[ +{text:"_",url:"functions.html#index__5F"}, +{text:"a",url:"functions_a.html#index_a"}, +{text:"c",url:"functions_c.html#index_c"}, +{text:"d",url:"functions_d.html#index_d"}, +{text:"e",url:"functions_e.html#index_e"}, +{text:"f",url:"functions_f.html#index_f"}, +{text:"g",url:"functions_g.html#index_g"}, +{text:"h",url:"functions_h.html#index_h"}, +{text:"i",url:"functions_i.html#index_i"}, +{text:"j",url:"functions_j.html#index_j"}, +{text:"k",url:"functions_k.html#index_k"}, +{text:"l",url:"functions_l.html#index_l"}, +{text:"m",url:"functions_m.html#index_m"}, +{text:"n",url:"functions_n.html#index_n"}, +{text:"o",url:"functions_o.html#index_o"}, +{text:"p",url:"functions_p.html#index_p"}, +{text:"q",url:"functions_q.html#index_q"}, +{text:"r",url:"functions_r.html#index_r"}, +{text:"s",url:"functions_s.html#index_s"}, +{text:"u",url:"functions_u.html#index_u"}, +{text:"v",url:"functions_v.html#index_v"}, +{text:"x",url:"functions_x.html#index_x"}, +{text:"z",url:"functions_z.html#index_z"}]}, +{text:"Variables",url:"functions_vars.html",children:[ +{text:"_",url:"functions_vars.html#index__5F"}, +{text:"a",url:"functions_vars_a.html#index_a"}, +{text:"c",url:"functions_vars_c.html#index_c"}, +{text:"d",url:"functions_vars_d.html#index_d"}, +{text:"e",url:"functions_vars_e.html#index_e"}, +{text:"f",url:"functions_vars_f.html#index_f"}, +{text:"g",url:"functions_vars_g.html#index_g"}, +{text:"h",url:"functions_vars_h.html#index_h"}, +{text:"i",url:"functions_vars_i.html#index_i"}, +{text:"j",url:"functions_vars_j.html#index_j"}, +{text:"k",url:"functions_vars_k.html#index_k"}, +{text:"l",url:"functions_vars_l.html#index_l"}, +{text:"m",url:"functions_vars_m.html#index_m"}, +{text:"n",url:"functions_vars_n.html#index_n"}, +{text:"o",url:"functions_vars_o.html#index_o"}, +{text:"p",url:"functions_vars_p.html#index_p"}, +{text:"q",url:"functions_vars_q.html#index_q"}, +{text:"r",url:"functions_vars_r.html#index_r"}, +{text:"s",url:"functions_vars_s.html#index_s"}, +{text:"u",url:"functions_vars_u.html#index_u"}, +{text:"v",url:"functions_vars_v.html#index_v"}, +{text:"x",url:"functions_vars_x.html#index_x"}, +{text:"z",url:"functions_vars_z.html#index_z"}]}]}]}, +{text:"Files",url:"files.html",children:[ +{text:"File List",url:"files.html"}, +{text:"Globals",url:"globals.html",children:[ +{text:"All",url:"globals.html",children:[ +{text:"_",url:"globals.html#index__5F"}, +{text:"a",url:"globals_a.html#index_a"}, +{text:"c",url:"globals_c.html#index_c"}, +{text:"d",url:"globals_d.html#index_d"}, +{text:"f",url:"globals_f.html#index_f"}, +{text:"g",url:"globals_g.html#index_g"}, +{text:"h",url:"globals_h.html#index_h"}, +{text:"i",url:"globals_i.html#index_i"}, +{text:"k",url:"globals_k.html#index_k"}, +{text:"m",url:"globals_m.html#index_m"}, +{text:"q",url:"globals_q.html#index_q"}, +{text:"s",url:"globals_s.html#index_s"}, +{text:"x",url:"globals_x.html#index_x"}, +{text:"z",url:"globals_z.html#index_z"}]}, +{text:"Functions",url:"globals_func.html",children:[ +{text:"a",url:"globals_func.html#index_a"}, +{text:"d",url:"globals_func_d.html#index_d"}, +{text:"f",url:"globals_func_f.html#index_f"}, +{text:"g",url:"globals_func_g.html#index_g"}, +{text:"i",url:"globals_func_i.html#index_i"}, +{text:"k",url:"globals_func_k.html#index_k"}, +{text:"m",url:"globals_func_m.html#index_m"}, +{text:"q",url:"globals_func_q.html#index_q"}, +{text:"s",url:"globals_func_s.html#index_s"}, +{text:"z",url:"globals_func_z.html#index_z"}]}, +{text:"Typedefs",url:"globals_type.html",children:[ +{text:"a",url:"globals_type.html#index_a"}, +{text:"c",url:"globals_type.html#index_c"}, +{text:"d",url:"globals_type.html#index_d"}, +{text:"f",url:"globals_type.html#index_f"}, +{text:"g",url:"globals_type.html#index_g"}, +{text:"h",url:"globals_type.html#index_h"}, +{text:"i",url:"globals_type.html#index_i"}, +{text:"k",url:"globals_type.html#index_k"}, +{text:"q",url:"globals_type.html#index_q"}, +{text:"s",url:"globals_type.html#index_s"}, +{text:"x",url:"globals_type.html#index_x"}, +{text:"z",url:"globals_type.html#index_z"}]}, +{text:"Enumerations",url:"globals_enum.html"}, +{text:"Enumerator",url:"globals_eval.html",children:[ +{text:"i",url:"globals_eval.html#index_i"}]}, +{text:"Macros",url:"globals_defs.html",children:[ +{text:"_",url:"globals_defs.html#index__5F"}, +{text:"d",url:"globals_defs.html#index_d"}, +{text:"i",url:"globals_defs.html#index_i"}, +{text:"k",url:"globals_defs.html#index_k"}]}]}]}]} diff --git a/docs/nav_f.png b/docs/nav_f.png new file mode 100644 index 0000000000000000000000000000000000000000..72a58a529ed3a9ed6aa0c51a79cf207e026deee2 GIT binary patch literal 153 zcmeAS@N?(olHy`uVBq!ia0vp^j6iI`!2~2XGqLUlQVE_ejv*C{Z|{2ZH7M}7UYxc) zn!W8uqtnIQ>_z8U literal 0 HcmV?d00001 diff --git a/docs/nav_g.png b/docs/nav_g.png new file mode 100644 index 0000000000000000000000000000000000000000..2093a237a94f6c83e19ec6e5fd42f7ddabdafa81 GIT binary patch literal 95 zcmeAS@N?(olHy`uVBq!ia0vp^j6lrB!3HFm1ilyoDK$?Q$B+ufw|5PB85lU25BhtE tr?otc=hd~V+ws&_A@j8Fiv!KF$B+ufw|5=67#uj90@pIL wZ=Q8~_Ju`#59=RjDrmm`tMD@M=!-l18IR?&vFVdQ&MBb@0HFXL=0 ? varName.substring(i+1) : varName; + return eval(n.replace(/\-/g,'_')); +} + +function stripPath(uri) +{ + return uri.substring(uri.lastIndexOf('/')+1); +} + +function stripPath2(uri) +{ + var i = uri.lastIndexOf('/'); + var s = uri.substring(i+1); + var m = uri.substring(0,i+1).match(/\/d\w\/d\w\w\/$/); + return m ? uri.substring(i-6) : s; +} + +function hashValue() +{ + return $(location).attr('hash').substring(1).replace(/[^\w\-]/g,''); +} + +function hashUrl() +{ + return '#'+hashValue(); +} + +function pathName() +{ + return $(location).attr('pathname').replace(/[^-A-Za-z0-9+&@#/%?=~_|!:,.;\(\)]/g, ''); +} + +function localStorageSupported() +{ + try { + return 'localStorage' in window && window['localStorage'] !== null && window.localStorage.getItem; + } + catch(e) { + return false; + } +} + +function storeLink(link) +{ + if (!$("#nav-sync").hasClass('sync') && localStorageSupported()) { + window.localStorage.setItem('navpath',link); + } +} + +function deleteLink() +{ + if (localStorageSupported()) { + window.localStorage.setItem('navpath',''); + } +} + +function cachedLink() +{ + if (localStorageSupported()) { + return window.localStorage.getItem('navpath'); + } else { + return ''; + } +} + +function getScript(scriptName,func,show) +{ + var head = document.getElementsByTagName("head")[0]; + var script = document.createElement('script'); + script.id = scriptName; + script.type = 'text/javascript'; + script.onload = func; + script.src = scriptName+'.js'; + head.appendChild(script); +} + +function createIndent(o,domNode,node,level) +{ + var level=-1; + var n = node; + while (n.parentNode) { level++; n=n.parentNode; } + if (node.childrenData) { + var imgNode = document.createElement("span"); + imgNode.className = 'arrow'; + imgNode.style.paddingLeft=(16*level).toString()+'px'; + imgNode.innerHTML=arrowRight; + node.plus_img = imgNode; + node.expandToggle = document.createElement("a"); + node.expandToggle.href = "javascript:void(0)"; + node.expandToggle.onclick = function() { + if (node.expanded) { + $(node.getChildrenUL()).slideUp("fast"); + node.plus_img.innerHTML=arrowRight; + node.expanded = false; + } else { + expandNode(o, node, false, false); + } + } + node.expandToggle.appendChild(imgNode); + domNode.appendChild(node.expandToggle); + } else { + var span = document.createElement("span"); + span.className = 'arrow'; + span.style.width = 16*(level+1)+'px'; + span.innerHTML = ' '; + domNode.appendChild(span); + } +} + +var animationInProgress = false; + +function gotoAnchor(anchor,aname,updateLocation) +{ + var pos, docContent = $('#doc-content'); + var ancParent = $(anchor.parent()); + if (ancParent.hasClass('memItemLeft') || + ancParent.hasClass('memtitle') || + ancParent.hasClass('fieldname') || + ancParent.hasClass('fieldtype') || + ancParent.is(':header')) + { + pos = ancParent.position().top; + } else if (anchor.position()) { + pos = anchor.position().top; + } + if (pos) { + var dist = Math.abs(Math.min( + pos-docContent.offset().top, + docContent[0].scrollHeight- + docContent.height()-docContent.scrollTop())); + animationInProgress=true; + docContent.animate({ + scrollTop: pos + docContent.scrollTop() - docContent.offset().top + },Math.max(50,Math.min(500,dist)),function(){ + if (updateLocation) window.location.href=aname; + animationInProgress=false; + }); + } +} + +function newNode(o, po, text, link, childrenData, lastNode) +{ + var node = new Object(); + node.children = Array(); + node.childrenData = childrenData; + node.depth = po.depth + 1; + node.relpath = po.relpath; + node.isLast = lastNode; + + node.li = document.createElement("li"); + po.getChildrenUL().appendChild(node.li); + node.parentNode = po; + + node.itemDiv = document.createElement("div"); + node.itemDiv.className = "item"; + + node.labelSpan = document.createElement("span"); + node.labelSpan.className = "label"; + + createIndent(o,node.itemDiv,node,0); + node.itemDiv.appendChild(node.labelSpan); + node.li.appendChild(node.itemDiv); + + var a = document.createElement("a"); + node.labelSpan.appendChild(a); + node.label = document.createTextNode(text); + node.expanded = false; + a.appendChild(node.label); + if (link) { + var url; + if (link.substring(0,1)=='^') { + url = link.substring(1); + link = url; + } else { + url = node.relpath+link; + } + a.className = stripPath(link.replace('#',':')); + if (link.indexOf('#')!=-1) { + var aname = '#'+link.split('#')[1]; + var srcPage = stripPath(pathName()); + var targetPage = stripPath(link.split('#')[0]); + a.href = srcPage!=targetPage ? url : "javascript:void(0)"; + a.onclick = function(){ + storeLink(link); + if (!$(a).parent().parent().hasClass('selected')) + { + $('.item').removeClass('selected'); + $('.item').removeAttr('id'); + $(a).parent().parent().addClass('selected'); + $(a).parent().parent().attr('id','selected'); + } + var anchor = $(aname); + gotoAnchor(anchor,aname,true); + }; + } else { + a.href = url; + a.onclick = function() { storeLink(link); } + } + } else { + if (childrenData != null) + { + a.className = "nolink"; + a.href = "javascript:void(0)"; + a.onclick = node.expandToggle.onclick; + } + } + + node.childrenUL = null; + node.getChildrenUL = function() { + if (!node.childrenUL) { + node.childrenUL = document.createElement("ul"); + node.childrenUL.className = "children_ul"; + node.childrenUL.style.display = "none"; + node.li.appendChild(node.childrenUL); + } + return node.childrenUL; + }; + + return node; +} + +function showRoot() +{ + var headerHeight = $("#top").height(); + var footerHeight = $("#nav-path").height(); + var windowHeight = $(window).height() - headerHeight - footerHeight; + (function (){ // retry until we can scroll to the selected item + try { + var navtree=$('#nav-tree'); + navtree.scrollTo('#selected',100,{offset:-windowHeight/2}); + } catch (err) { + setTimeout(arguments.callee, 0); + } + })(); +} + +function expandNode(o, node, imm, showRoot) +{ + if (node.childrenData && !node.expanded) { + if (typeof(node.childrenData)==='string') { + var varName = node.childrenData; + getScript(node.relpath+varName,function(){ + node.childrenData = getData(varName); + expandNode(o, node, imm, showRoot); + }, showRoot); + } else { + if (!node.childrenVisited) { + getNode(o, node); + } + $(node.getChildrenUL()).slideDown("fast"); + node.plus_img.innerHTML = arrowDown; + node.expanded = true; + } + } +} + +function glowEffect(n,duration) +{ + n.addClass('glow').delay(duration).queue(function(next){ + $(this).removeClass('glow');next(); + }); +} + +function highlightAnchor() +{ + var aname = hashUrl(); + var anchor = $(aname); + if (anchor.parent().attr('class')=='memItemLeft'){ + var rows = $('.memberdecls tr[class$="'+hashValue()+'"]'); + glowEffect(rows.children(),300); // member without details + } else if (anchor.parent().attr('class')=='fieldname'){ + glowEffect(anchor.parent().parent(),1000); // enum value + } else if (anchor.parent().attr('class')=='fieldtype'){ + glowEffect(anchor.parent().parent(),1000); // struct field + } else if (anchor.parent().is(":header")) { + glowEffect(anchor.parent(),1000); // section header + } else { + glowEffect(anchor.next(),1000); // normal member + } +} + +function selectAndHighlight(hash,n) +{ + var a; + if (hash) { + var link=stripPath(pathName())+':'+hash.substring(1); + a=$('.item a[class$="'+link+'"]'); + } + if (a && a.length) { + a.parent().parent().addClass('selected'); + a.parent().parent().attr('id','selected'); + highlightAnchor(); + } else if (n) { + $(n.itemDiv).addClass('selected'); + $(n.itemDiv).attr('id','selected'); + } + var topOffset=5; + if (typeof page_layout!=='undefined' && page_layout==1) { + topOffset+=$('#top').outerHeight(); + } + if ($('#nav-tree-contents .item:first').hasClass('selected')) { + topOffset+=25; + } + $('#nav-sync').css('top',topOffset+'px'); + showRoot(); +} + +function showNode(o, node, index, hash) +{ + if (node && node.childrenData) { + if (typeof(node.childrenData)==='string') { + var varName = node.childrenData; + getScript(node.relpath+varName,function(){ + node.childrenData = getData(varName); + showNode(o,node,index,hash); + },true); + } else { + if (!node.childrenVisited) { + getNode(o, node); + } + $(node.getChildrenUL()).css({'display':'block'}); + node.plus_img.innerHTML = arrowDown; + node.expanded = true; + var n = node.children[o.breadcrumbs[index]]; + if (index+11) hash = '#'+parts[1].replace(/[^\w\-]/g,''); + else hash=''; + } + if (hash.match(/^#l\d+$/)) { + var anchor=$('a[name='+hash.substring(1)+']'); + glowEffect(anchor.parent(),1000); // line number + hash=''; // strip line number anchors + } + var url=root+hash; + var i=-1; + while (NAVTREEINDEX[i+1]<=url) i++; + if (i==-1) { i=0; root=NAVTREE[0][1]; } // fallback: show index + if (navTreeSubIndices[i]) { + gotoNode(o,i,root,hash,relpath) + } else { + getScript(relpath+'navtreeindex'+i,function(){ + navTreeSubIndices[i] = eval('NAVTREEINDEX'+i); + if (navTreeSubIndices[i]) { + gotoNode(o,i,root,hash,relpath); + } + },true); + } +} + +function showSyncOff(n,relpath) +{ + n.html(''); +} + +function showSyncOn(n,relpath) +{ + n.html(''); +} + +function toggleSyncButton(relpath) +{ + var navSync = $('#nav-sync'); + if (navSync.hasClass('sync')) { + navSync.removeClass('sync'); + showSyncOff(navSync,relpath); + storeLink(stripPath2(pathName())+hashUrl()); + } else { + navSync.addClass('sync'); + showSyncOn(navSync,relpath); + deleteLink(); + } +} + +var loadTriggered = false; +var readyTriggered = false; +var loadObject,loadToRoot,loadUrl,loadRelPath; + +$(window).on('load',function(){ + if (readyTriggered) { // ready first + navTo(loadObject,loadToRoot,loadUrl,loadRelPath); + showRoot(); + } + loadTriggered=true; +}); + +function initNavTree(toroot,relpath) +{ + var o = new Object(); + o.toroot = toroot; + o.node = new Object(); + o.node.li = document.getElementById("nav-tree-contents"); + o.node.childrenData = NAVTREE; + o.node.children = new Array(); + o.node.childrenUL = document.createElement("ul"); + o.node.getChildrenUL = function() { return o.node.childrenUL; }; + o.node.li.appendChild(o.node.childrenUL); + o.node.depth = 0; + o.node.relpath = relpath; + o.node.expanded = false; + o.node.isLast = true; + o.node.plus_img = document.createElement("span"); + o.node.plus_img.className = 'arrow'; + o.node.plus_img.innerHTML = arrowRight; + + if (localStorageSupported()) { + var navSync = $('#nav-sync'); + if (cachedLink()) { + showSyncOff(navSync,relpath); + navSync.removeClass('sync'); + } else { + showSyncOn(navSync,relpath); + } + navSync.click(function(){ toggleSyncButton(relpath); }); + } + + if (loadTriggered) { // load before ready + navTo(o,toroot,hashUrl(),relpath); + showRoot(); + } else { // ready before load + loadObject = o; + loadToRoot = toroot; + loadUrl = hashUrl(); + loadRelPath = relpath; + readyTriggered=true; + } + + $(window).bind('hashchange', function(){ + if (window.location.hash && window.location.hash.length>1){ + var a; + if ($(location).attr('hash')){ + var clslink=stripPath(pathName())+':'+hashValue(); + a=$('.item a[class$="'+clslink.replace(/1|%O$WD@{VPM$7~Ar*{o?;hlAFyLXmaDC0y znK1_#cQqJWPES%4Uujug^TE?jMft$}Eq^WaR~)%f)vSNs&gek&x%A9X9sM + + + + + + +intel-ipsec-mb: Related Pages + + + + + + + + + +
    +
    + + + + + + +
    +
    intel-ipsec-mb +
    +
    +
    + + + + + + + +
    + +
    +
    + + +
    + +
    + +
    +
    +
    Related Pages
    +
    +
    +
    Here is a list of all related documentation pages:
    + + +
     README
    +
    +
    + +
    +Generated by   +doxygen + 1.8.17 +
    + + diff --git a/docs/resize.js b/docs/resize.js new file mode 100644 index 00000000..03d914f9 --- /dev/null +++ b/docs/resize.js @@ -0,0 +1,156 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +function initResizable() +{ + var cookie_namespace = 'doxygen'; + var sidenav,navtree,content,header,collapsed,collapsedWidth=0,barWidth=6,desktop_vp=768,titleHeight; + + function readCookie(cookie) + { + var myCookie = cookie_namespace+"_"+cookie+"="; + if (document.cookie) { + var index = document.cookie.indexOf(myCookie); + if (index != -1) { + var valStart = index + myCookie.length; + var valEnd = document.cookie.indexOf(";", valStart); + if (valEnd == -1) { + valEnd = document.cookie.length; + } + var val = document.cookie.substring(valStart, valEnd); + return val; + } + } + return 0; + } + + function writeCookie(cookie, val, expiration) + { + if (val==undefined) return; + if (expiration == null) { + var date = new Date(); + date.setTime(date.getTime()+(10*365*24*60*60*1000)); // default expiration is one week + expiration = date.toGMTString(); + } + document.cookie = cookie_namespace + "_" + cookie + "=" + val + "; SameSite=Lax; expires=" + expiration+"; path=/"; + } + + function resizeWidth() + { + var windowWidth = $(window).width() + "px"; + var sidenavWidth = $(sidenav).outerWidth(); + content.css({marginLeft:parseInt(sidenavWidth)+"px"}); + if (typeof page_layout!=='undefined' && page_layout==1) { + footer.css({marginLeft:parseInt(sidenavWidth)+"px"}); + } + writeCookie('width',sidenavWidth-barWidth, null); + } + + function restoreWidth(navWidth) + { + var windowWidth = $(window).width() + "px"; + content.css({marginLeft:parseInt(navWidth)+barWidth+"px"}); + if (typeof page_layout!=='undefined' && page_layout==1) { + footer.css({marginLeft:parseInt(navWidth)+barWidth+"px"}); + } + sidenav.css({width:navWidth + "px"}); + } + + function resizeHeight() + { + var headerHeight = header.outerHeight(); + var footerHeight = footer.outerHeight(); + var windowHeight = $(window).height(); + var contentHeight,navtreeHeight,sideNavHeight; + if (typeof page_layout==='undefined' || page_layout==0) { /* DISABLE_INDEX=NO */ + contentHeight = windowHeight - headerHeight - footerHeight; + navtreeHeight = contentHeight; + sideNavHeight = contentHeight; + } else if (page_layout==1) { /* DISABLE_INDEX=YES */ + contentHeight = windowHeight - footerHeight; + navtreeHeight = windowHeight - headerHeight; + sideNavHeight = windowHeight; + } + content.css({height:contentHeight + "px"}); + navtree.css({height:navtreeHeight + "px"}); + sidenav.css({height:sideNavHeight + "px"}); + var width=$(window).width(); + if (width!=collapsedWidth) { + if (width=desktop_vp) { + if (!collapsed) { + collapseExpand(); + } + } else if (width>desktop_vp && collapsedWidth0) { + restoreWidth(0); + collapsed=true; + } + else { + var width = readCookie('width'); + if (width>200 && width<$(window).width()) { restoreWidth(width); } else { restoreWidth(200); } + collapsed=false; + } + } + + header = $("#top"); + sidenav = $("#side-nav"); + content = $("#doc-content"); + navtree = $("#nav-tree"); + footer = $("#nav-path"); + $(".side-nav-resizable").resizable({resize: function(e, ui) { resizeWidth(); } }); + $(sidenav).resizable({ minWidth: 0 }); + $(window).resize(function() { resizeHeight(); }); + var device = navigator.userAgent.toLowerCase(); + var touch_device = device.match(/(iphone|ipod|ipad|android)/); + if (touch_device) { /* wider split bar for touch only devices */ + $(sidenav).css({ paddingRight:'20px' }); + $('.ui-resizable-e').css({ width:'20px' }); + $('#nav-sync').css({ right:'34px' }); + barWidth=20; + } + var width = readCookie('width'); + if (width) { restoreWidth(width); } else { resizeWidth(); } + resizeHeight(); + var url = location.href; + var i=url.indexOf("#"); + if (i>=0) window.location.hash=url.substr(i); + var _preventDefault = function(evt) { evt.preventDefault(); }; + $("#splitbar").bind("dragstart", _preventDefault).bind("selectstart", _preventDefault); + $(".ui-resizable-handle").dblclick(collapseExpand); + $(window).on('load',resizeHeight); +} +/* @license-end */ diff --git a/docs/search/all_0.html b/docs/search/all_0.html new file mode 100644 index 00000000..98038bd0 --- /dev/null +++ b/docs/search/all_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_0.js b/docs/search/all_0.js new file mode 100644 index 00000000..b5d42e4c --- /dev/null +++ b/docs/search/all_0.js @@ -0,0 +1,16 @@ +var searchData= +[ + ['_5f_5fforceinline_0',['__forceinline',['../intel-ipsec-mb_8h.html#af93b819ac40799ac392e16f6a90729fd',1,'intel-ipsec-mb.h']]], + ['_5fhashed_5fauth_5fkey_5fxor_5fipad_1',['_hashed_auth_key_xor_ipad',['../structIMB__JOB.html#a2e74bbeeed2c70f97e6b30a0a12aec0e',1,'IMB_JOB']]], + ['_5fhashed_5fauth_5fkey_5fxor_5fopad_2',['_hashed_auth_key_xor_opad',['../structIMB__JOB.html#a684b751d0a275a28d345a3d1051204ef',1,'IMB_JOB']]], + ['_5finit_5ftag_3',['_init_tag',['../structIMB__JOB.html#a1e2fc881ce9c84173168df1dbda851f8',1,'IMB_JOB']]], + ['_5fiv_4',['_iv',['../structIMB__JOB.html#ae571147592c332cea0d90a36e133e6a5',1,'IMB_JOB::_iv()'],['../structIMB__JOB.html#a4415e6851e770734c5ec1b46a828ce55',1,'IMB_JOB::_iv()']]], + ['_5fiv23_5',['_iv23',['../structIMB__JOB.html#a2bd093b1609dad73bbef2a728995d12f',1,'IMB_JOB']]], + ['_5fk1_5fexpanded_6',['_k1_expanded',['../structIMB__JOB.html#a76d55a988c71e40c577c6f8f5d0496de',1,'IMB_JOB']]], + ['_5fk2_7',['_k2',['../structIMB__JOB.html#adc47d4af15c88dcb7abf01bb07e79e63',1,'IMB_JOB']]], + ['_5fk3_8',['_k3',['../structIMB__JOB.html#afa62af4d5f1ecbb80f5216ec298fcfe8',1,'IMB_JOB']]], + ['_5fkey_9',['_key',['../structIMB__JOB.html#a7ba1fdbcc6ed2e3840990d2e34dac8e8',1,'IMB_JOB::_key()'],['../structIMB__JOB.html#a7a62dc4b2ef35231a93ce035561c4153',1,'IMB_JOB::_key()'],['../structIMB__JOB.html#a3b3f616759b854553a351e6b44e08471',1,'IMB_JOB::_key()']]], + ['_5fkey_5fexpanded_10',['_key_expanded',['../structIMB__JOB.html#a95cc77e0c079f43dec1a46a2b75422a0',1,'IMB_JOB']]], + ['_5fskey1_11',['_skey1',['../structIMB__JOB.html#ab64e8f6bf8dfaf56305b9023103ea5e2',1,'IMB_JOB']]], + ['_5fskey2_12',['_skey2',['../structIMB__JOB.html#a8e8143e461eab3b2b04333c5f1c17a82',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_1.html b/docs/search/all_1.html new file mode 100644 index 00000000..ed8ad8a6 --- /dev/null +++ b/docs/search/all_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_1.js b/docs/search/all_1.js new file mode 100644 index 00000000..78ce251f --- /dev/null +++ b/docs/search/all_1.js @@ -0,0 +1,151 @@ +var searchData= +[ + ['aad_0',['aad',['../structIMB__JOB.html#a4c943fcac7f5c4f5d6d030547840071e',1,'IMB_JOB']]], + ['aad_5fhash_1',['aad_hash',['../structgcm__context__data.html#a98c22f35eb0b4d368e2f454b347ed3b9',1,'gcm_context_data']]], + ['aad_5flen_2',['aad_len',['../structchacha20__poly1305__context__data.html#a7b5edcebc9a6bec3776daf6d13bf950c',1,'chacha20_poly1305_context_data']]], + ['aad_5flen_5fin_5fbytes_3',['aad_len_in_bytes',['../structIMB__JOB.html#aa607a780bc2c8509270625c7520a11f1',1,'IMB_JOB']]], + ['aad_5flength_4',['aad_length',['../structgcm__context__data.html#ab1dadabcbd2002738ec7db7da332e04f',1,'gcm_context_data']]], + ['aes128_5fcbcs_5fooo_5',['aes128_cbcs_ooo',['../structIMB__MGR.html#a9c72f3341a9a4837595338d06c3eda5c',1,'IMB_MGR']]], + ['aes128_5fcfb_5fone_6',['aes128_cfb_one',['../structIMB__MGR.html#abd468de03563a40da0b21b0885f44cbe',1,'IMB_MGR']]], + ['aes128_5fooo_7',['aes128_ooo',['../structIMB__MGR.html#a97fd05a4057fc898f647955a7ba8501b',1,'IMB_MGR']]], + ['aes192_5fooo_8',['aes192_ooo',['../structIMB__MGR.html#afe48187946dff59f6c69103501691921',1,'IMB_MGR']]], + ['aes256_5fccm_5fooo_9',['aes256_ccm_ooo',['../structIMB__MGR.html#a04c28bfe763544b68dac98e23aa0a8d9',1,'IMB_MGR']]], + ['aes256_5fcmac_5fooo_10',['aes256_cmac_ooo',['../structIMB__MGR.html#acbd18bab8176262daab2bd808ca30616',1,'IMB_MGR']]], + ['aes256_5fooo_11',['aes256_ooo',['../structIMB__MGR.html#a6abbf129001cb44003a10105115e52b7',1,'IMB_MGR']]], + ['aes_5fccm_5fooo_12',['aes_ccm_ooo',['../structIMB__MGR.html#a3eeeead99056e46d8cb1ed316f587aaf',1,'IMB_MGR']]], + ['aes_5fcfb_5f128_5fone_5favx_13',['aes_cfb_128_one_avx',['../intel-ipsec-mb_8h.html#a5a330bd4ed945f830c88dd6e1066652d',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5favx2_14',['aes_cfb_128_one_avx2',['../intel-ipsec-mb_8h.html#a7046666312bdba42b05bfbf63d0e1c6a',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5favx512_15',['aes_cfb_128_one_avx512',['../intel-ipsec-mb_8h.html#abb7c6395420af2addd92a3fa46e0fd28',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5fsse_16',['aes_cfb_128_one_sse',['../intel-ipsec-mb_8h.html#a7bc28362ae2fb8a4d624b8a7a00adebd',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5ft_17',['aes_cfb_t',['../intel-ipsec-mb_8h.html#aeb837fda81cdf56ffcfa4e68b5877a35',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fooo_18',['aes_cmac_ooo',['../structIMB__MGR.html#a0bd2acac3a6c3f90a998ed53a7d28e57',1,'IMB_MGR']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx_19',['aes_cmac_subkey_gen_avx',['../intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx2_20',['aes_cmac_subkey_gen_avx2',['../intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx512_21',['aes_cmac_subkey_gen_avx512',['../intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5fsse_22',['aes_cmac_subkey_gen_sse',['../intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen2_23',['aes_gcm_dec_128_avx_gen2',['../intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen4_24',['aes_gcm_dec_128_avx_gen4',['../intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen2_25',['aes_gcm_dec_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen4_26',['aes_gcm_dec_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5fsse_27',['aes_gcm_dec_128_finalize_sse',['../intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fsse_28',['aes_gcm_dec_128_sse',['../intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen2_29',['aes_gcm_dec_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen4_30',['aes_gcm_dec_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5fsse_31',['aes_gcm_dec_128_update_sse',['../intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen2_32',['aes_gcm_dec_192_avx_gen2',['../intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen4_33',['aes_gcm_dec_192_avx_gen4',['../intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen2_34',['aes_gcm_dec_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen4_35',['aes_gcm_dec_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5fsse_36',['aes_gcm_dec_192_finalize_sse',['../intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fsse_37',['aes_gcm_dec_192_sse',['../intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen2_38',['aes_gcm_dec_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen4_39',['aes_gcm_dec_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5fsse_40',['aes_gcm_dec_192_update_sse',['../intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen2_41',['aes_gcm_dec_256_avx_gen2',['../intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen4_42',['aes_gcm_dec_256_avx_gen4',['../intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen2_43',['aes_gcm_dec_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen4_44',['aes_gcm_dec_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5fsse_45',['aes_gcm_dec_256_finalize_sse',['../intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fsse_46',['aes_gcm_dec_256_sse',['../intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen2_47',['aes_gcm_dec_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen4_48',['aes_gcm_dec_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5fsse_49',['aes_gcm_dec_256_update_sse',['../intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen2_50',['aes_gcm_enc_128_avx_gen2',['../intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen4_51',['aes_gcm_enc_128_avx_gen4',['../intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen2_52',['aes_gcm_enc_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen4_53',['aes_gcm_enc_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5fsse_54',['aes_gcm_enc_128_finalize_sse',['../intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fsse_55',['aes_gcm_enc_128_sse',['../intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen2_56',['aes_gcm_enc_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen4_57',['aes_gcm_enc_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5fsse_58',['aes_gcm_enc_128_update_sse',['../intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen2_59',['aes_gcm_enc_192_avx_gen2',['../intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen4_60',['aes_gcm_enc_192_avx_gen4',['../intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen2_61',['aes_gcm_enc_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen4_62',['aes_gcm_enc_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5fsse_63',['aes_gcm_enc_192_finalize_sse',['../intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fsse_64',['aes_gcm_enc_192_sse',['../intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen2_65',['aes_gcm_enc_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen4_66',['aes_gcm_enc_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5fsse_67',['aes_gcm_enc_192_update_sse',['../intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen2_68',['aes_gcm_enc_256_avx_gen2',['../intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen4_69',['aes_gcm_enc_256_avx_gen4',['../intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen2_70',['aes_gcm_enc_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen4_71',['aes_gcm_enc_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5fsse_72',['aes_gcm_enc_256_finalize_sse',['../intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fsse_73',['aes_gcm_enc_256_sse',['../intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen2_74',['aes_gcm_enc_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen4_75',['aes_gcm_enc_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5fsse_76',['aes_gcm_enc_256_update_sse',['../intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ffinalize_5ft_77',['aes_gcm_enc_dec_finalize_t',['../intel-ipsec-mb_8h.html#a8c85430db7a8b695a2836dcb96653bc5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fiv_5ft_78',['aes_gcm_enc_dec_iv_t',['../intel-ipsec-mb_8h.html#ac5bfb31d18fdc55451cd325c4af032c7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ft_79',['aes_gcm_enc_dec_t',['../intel-ipsec-mb_8h.html#a71fb9f25bcab533a98f3d1c5cc2c7f48',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fupdate_5ft_80',['aes_gcm_enc_dec_update_t',['../intel-ipsec-mb_8h.html#af701adcd98fd95084d76e0cb0f926fef',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen2_81',['aes_gcm_init_128_avx_gen2',['../intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen4_82',['aes_gcm_init_128_avx_gen4',['../intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5fsse_83',['aes_gcm_init_128_sse',['../intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen2_84',['aes_gcm_init_192_avx_gen2',['../intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen4_85',['aes_gcm_init_192_avx_gen4',['../intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5fsse_86',['aes_gcm_init_192_sse',['../intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen2_87',['aes_gcm_init_256_avx_gen2',['../intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen4_88',['aes_gcm_init_256_avx_gen4',['../intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5fsse_89',['aes_gcm_init_256_sse',['../intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5ft_90',['aes_gcm_init_t',['../intel-ipsec-mb_8h.html#adeecd4bac067493d8465e8d7b6b7dd6d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5fvar_5fiv_5ft_91',['aes_gcm_init_var_iv_t',['../intel-ipsec-mb_8h.html#a65f909001359ad986640155f8756a11d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen2_92',['aes_gcm_pre_128_avx_gen2',['../intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen4_93',['aes_gcm_pre_128_avx_gen4',['../intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5fsse_94',['aes_gcm_pre_128_sse',['../intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen2_95',['aes_gcm_pre_192_avx_gen2',['../intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen4_96',['aes_gcm_pre_192_avx_gen4',['../intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5fsse_97',['aes_gcm_pre_192_sse',['../intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen2_98',['aes_gcm_pre_256_avx_gen2',['../intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen4_99',['aes_gcm_pre_256_avx_gen4',['../intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5fsse_100',['aes_gcm_pre_256_sse',['../intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5ft_101',['aes_gcm_pre_t',['../intel-ipsec-mb_8h.html#a11e10a3ac7a08801dc68949fb0f428af',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen2_102',['aes_gcm_precomp_128_avx_gen2',['../intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen4_103',['aes_gcm_precomp_128_avx_gen4',['../intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5fsse_104',['aes_gcm_precomp_128_sse',['../intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen2_105',['aes_gcm_precomp_192_avx_gen2',['../intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen4_106',['aes_gcm_precomp_192_avx_gen4',['../intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5fsse_107',['aes_gcm_precomp_192_sse',['../intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen2_108',['aes_gcm_precomp_256_avx_gen2',['../intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen4_109',['aes_gcm_precomp_256_avx_gen4',['../intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5fsse_110',['aes_gcm_precomp_256_sse',['../intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5ft_111',['aes_gcm_precomp_t',['../intel-ipsec-mb_8h.html#ab50906d68121cb8e6edf2a3493a60c5c',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5ffinalize_5ft_112',['aes_gmac_finalize_t',['../intel-ipsec-mb_8h.html#af3f75897a0ec8b3f22c4e2b01f7f48b7',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5finit_5ft_113',['aes_gmac_init_t',['../intel-ipsec-mb_8h.html#a31987251c968435d720e8656ae4621a0',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5fupdate_5ft_114',['aes_gmac_update_t',['../intel-ipsec-mb_8h.html#aba0ac59064802c6d0a182a5164568c18',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx_115',['aes_keyexp_128_avx',['../intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx2_116',['aes_keyexp_128_avx2',['../intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx512_117',['aes_keyexp_128_avx512',['../intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx_118',['aes_keyexp_128_enc_avx',['../intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx2_119',['aes_keyexp_128_enc_avx2',['../intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx512_120',['aes_keyexp_128_enc_avx512',['../intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5fsse_121',['aes_keyexp_128_enc_sse',['../intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fsse_122',['aes_keyexp_128_sse',['../intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx_123',['aes_keyexp_192_avx',['../intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx2_124',['aes_keyexp_192_avx2',['../intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx512_125',['aes_keyexp_192_avx512',['../intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx_126',['aes_keyexp_192_enc_avx',['../intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx2_127',['aes_keyexp_192_enc_avx2',['../intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx512_128',['aes_keyexp_192_enc_avx512',['../intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5fsse_129',['aes_keyexp_192_enc_sse',['../intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fsse_130',['aes_keyexp_192_sse',['../intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx_131',['aes_keyexp_256_avx',['../intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx2_132',['aes_keyexp_256_avx2',['../intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx512_133',['aes_keyexp_256_avx512',['../intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx_134',['aes_keyexp_256_enc_avx',['../intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx2_135',['aes_keyexp_256_enc_avx2',['../intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx512_136',['aes_keyexp_256_enc_avx512',['../intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5fsse_137',['aes_keyexp_256_enc_sse',['../intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fsse_138',['aes_keyexp_256_sse',['../intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx_139',['aes_xcbc_expand_key_avx',['../intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx2_140',['aes_xcbc_expand_key_avx2',['../intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx512_141',['aes_xcbc_expand_key_avx512',['../intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5fsse_142',['aes_xcbc_expand_key_sse',['../intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fooo_143',['aes_xcbc_ooo',['../structIMB__MGR.html#ad554a25793a904ae6cf7ca87250adbd3',1,'IMB_MGR']]], + ['alloc_5fmb_5fmgr_144',['alloc_mb_mgr',['../intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e',1,'intel-ipsec-mb.h']]], + ['auth_5ftag_5foutput_145',['auth_tag_output',['../structIMB__JOB.html#ac37080ddb0baf56c9589e65960430b32',1,'IMB_JOB']]], + ['auth_5ftag_5foutput_5flen_5fin_5fbytes_146',['auth_tag_output_len_in_bytes',['../structIMB__JOB.html#a750df4fa7330e2387138409d0eb50e0f',1,'IMB_JOB']]], + ['avx2_5favx512_147',['avx2_avx512',['../structgcm__key__data.html#a3f231fdc1d521cd15c111667e4aea818',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_10.html b/docs/search/all_10.html new file mode 100644 index 00000000..2703ec10 --- /dev/null +++ b/docs/search/all_10.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_10.js b/docs/search/all_10.js new file mode 100644 index 00000000..a3966989 --- /dev/null +++ b/docs/search/all_10.js @@ -0,0 +1,9 @@ +var searchData= +[ + ['queue_5fsize_0',['queue_size',['../structIMB__MGR.html#a5a35c9c2a1c8eb5b4aa239a3b04f4fbf',1,'IMB_MGR']]], + ['queue_5fsize_5favx_1',['queue_size_avx',['../intel-ipsec-mb_8h.html#a800c395d326e718a2b520d0bb91cbb89',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx2_2',['queue_size_avx2',['../intel-ipsec-mb_8h.html#a8a540000c08366ad47d95e23ed5b2918',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx512_3',['queue_size_avx512',['../intel-ipsec-mb_8h.html#a51c0426a4b57c9e7db2b091692dff472',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5fsse_4',['queue_size_sse',['../intel-ipsec-mb_8h.html#a80f792744558de173ea97d722c58e5c1',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5ft_5',['queue_size_t',['../intel-ipsec-mb_8h.html#a9884ae92b598dd504f7d0ef8be5051d6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_11.html b/docs/search/all_11.html new file mode 100644 index 00000000..986a99b6 --- /dev/null +++ b/docs/search/all_11.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_11.js b/docs/search/all_11.js new file mode 100644 index 00000000..7f1d6ce3 --- /dev/null +++ b/docs/search/all_11.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['readme_2emd_0',['README.md',['../README_8md.html',1,'']]], + ['remain_5fct_5fbytes_1',['remain_ct_bytes',['../structchacha20__poly1305__context__data.html#a74e971fe76c8b946e30e8f0c8cce111a',1,'chacha20_poly1305_context_data']]], + ['remain_5fks_5fbytes_2',['remain_ks_bytes',['../structchacha20__poly1305__context__data.html#a84693b96048f6130a749b633c65e2105',1,'chacha20_poly1305_context_data']]], + ['reserved_3',['reserved',['../structIMB__JOB.html#a9d5e67861edf38dfa23160b650c7caa9',1,'IMB_JOB::reserved()'],['../structIMB__MGR.html#a136697eb37d5a06eeada0f338abd8b51',1,'IMB_MGR::reserved()']]] +]; diff --git a/docs/search/all_12.html b/docs/search/all_12.html new file mode 100644 index 00000000..d0cff191 --- /dev/null +++ b/docs/search/all_12.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_12.js b/docs/search/all_12.js new file mode 100644 index 00000000..6a4e73cf --- /dev/null +++ b/docs/search/all_12.js @@ -0,0 +1,116 @@ +var searchData= +[ + ['sgl_5fio_5fsegs_0',['sgl_io_segs',['../structIMB__JOB.html#a2462f9efb16c31ae615df7493dddb343',1,'IMB_JOB']]], + ['sgl_5fstate_1',['sgl_state',['../structIMB__JOB.html#a3d7e363be004999d02db464c721e9751',1,'IMB_JOB']]], + ['sha1_2',['sha1',['../structIMB__MGR.html#a82aaedc800096aaa55062c56131e0b13',1,'IMB_MGR']]], + ['sha1_5favx_3',['sha1_avx',['../intel-ipsec-mb_8h.html#a9478828dde5161c2a1b43cb0da0f9f67',1,'intel-ipsec-mb.h']]], + ['sha1_5favx2_4',['sha1_avx2',['../intel-ipsec-mb_8h.html#ac13d60c1ddbd3c770f9d85d59821e520',1,'intel-ipsec-mb.h']]], + ['sha1_5favx512_5',['sha1_avx512',['../intel-ipsec-mb_8h.html#ad9b42865aff95b5bcde6efa4b4e83fe8',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_6',['sha1_one_block',['../structIMB__MGR.html#a80a26be9a12f561a68b70d65e096cde9',1,'IMB_MGR']]], + ['sha1_5fone_5fblock_5favx_7',['sha1_one_block_avx',['../intel-ipsec-mb_8h.html#a532582e6d066b9c6b1c2189ba1b36f27',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5favx2_8',['sha1_one_block_avx2',['../intel-ipsec-mb_8h.html#af1a63afff082c9d14bbe22f14bd6b5d9',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5favx512_9',['sha1_one_block_avx512',['../intel-ipsec-mb_8h.html#a745c97ce8263ebc67f4d8070f5c9ee50',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5fsse_10',['sha1_one_block_sse',['../intel-ipsec-mb_8h.html#a99f608f81f6b692eaeea1d68cac25a28',1,'intel-ipsec-mb.h']]], + ['sha1_5fsse_11',['sha1_sse',['../intel-ipsec-mb_8h.html#a37491294dc799c5477e3979e89e0cf92',1,'intel-ipsec-mb.h']]], + ['sha224_12',['sha224',['../structIMB__MGR.html#a52e26b8805b28d1298ac48e5762412b5',1,'IMB_MGR']]], + ['sha224_5favx_13',['sha224_avx',['../intel-ipsec-mb_8h.html#af0e48b520099db5a76190aa5395c90d3',1,'intel-ipsec-mb.h']]], + ['sha224_5favx2_14',['sha224_avx2',['../intel-ipsec-mb_8h.html#ad5945d85d451ccf9b3ba06c06c10d4e9',1,'intel-ipsec-mb.h']]], + ['sha224_5favx512_15',['sha224_avx512',['../intel-ipsec-mb_8h.html#a51c522a828b36d9c69dd7c357b556fc9',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_16',['sha224_one_block',['../structIMB__MGR.html#aacf8209094d12ceceed96d4d0fae193c',1,'IMB_MGR']]], + ['sha224_5fone_5fblock_5favx_17',['sha224_one_block_avx',['../intel-ipsec-mb_8h.html#a76d64c2094522011f741de4ac8a03125',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5favx2_18',['sha224_one_block_avx2',['../intel-ipsec-mb_8h.html#a58916afb9e72f42a1c8c2658e248703a',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5favx512_19',['sha224_one_block_avx512',['../intel-ipsec-mb_8h.html#a16f7bd134f4a667f14261a89ca541126',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5fsse_20',['sha224_one_block_sse',['../intel-ipsec-mb_8h.html#a6a15b62259d7475edd02b11b775b7a38',1,'intel-ipsec-mb.h']]], + ['sha224_5fsse_21',['sha224_sse',['../intel-ipsec-mb_8h.html#a2ef2c1513632e69da882a895a075aa2c',1,'intel-ipsec-mb.h']]], + ['sha256_22',['sha256',['../structIMB__MGR.html#a8d978764a953e220b3c799990bae490b',1,'IMB_MGR']]], + ['sha256_5favx_23',['sha256_avx',['../intel-ipsec-mb_8h.html#a7976461101f011a77d19f4fa480d5be3',1,'intel-ipsec-mb.h']]], + ['sha256_5favx2_24',['sha256_avx2',['../intel-ipsec-mb_8h.html#aec411a6b29d7adff67e6eaa1b13b3975',1,'intel-ipsec-mb.h']]], + ['sha256_5favx512_25',['sha256_avx512',['../intel-ipsec-mb_8h.html#a1020a70f15b89b7146733d8cbc2bcee8',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_26',['sha256_one_block',['../structIMB__MGR.html#ade4a72c39980f0bba790b70529b60158',1,'IMB_MGR']]], + ['sha256_5fone_5fblock_5favx_27',['sha256_one_block_avx',['../intel-ipsec-mb_8h.html#a2e481e815eb9c8109460e8a8dac62ce9',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5favx2_28',['sha256_one_block_avx2',['../intel-ipsec-mb_8h.html#a0a3c025cfbe9b6e72e9b9f4805f0ca1f',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5favx512_29',['sha256_one_block_avx512',['../intel-ipsec-mb_8h.html#ac2bfb5dca839523118362b48115d7c18',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5fsse_30',['sha256_one_block_sse',['../intel-ipsec-mb_8h.html#ae5ce647166e6f9f8b1d616584ee5d1d6',1,'intel-ipsec-mb.h']]], + ['sha256_5fsse_31',['sha256_sse',['../intel-ipsec-mb_8h.html#a4d9d41b8311d2649ec1b43d6e6edd398',1,'intel-ipsec-mb.h']]], + ['sha384_32',['sha384',['../structIMB__MGR.html#a9ffcce0ba9aef3cd68bdd7f6e9e3e2a0',1,'IMB_MGR']]], + ['sha384_5favx_33',['sha384_avx',['../intel-ipsec-mb_8h.html#a7b656d085ad03ff43a6c3a4751bfbf56',1,'intel-ipsec-mb.h']]], + ['sha384_5favx2_34',['sha384_avx2',['../intel-ipsec-mb_8h.html#a389b88f838e79306acea6c8bd157c712',1,'intel-ipsec-mb.h']]], + ['sha384_5favx512_35',['sha384_avx512',['../intel-ipsec-mb_8h.html#a908b9e33df6e434c4bb0c74fa20ceece',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_36',['sha384_one_block',['../structIMB__MGR.html#a92b7ff145f0016dd46eabf9271ef03f1',1,'IMB_MGR']]], + ['sha384_5fone_5fblock_5favx_37',['sha384_one_block_avx',['../intel-ipsec-mb_8h.html#aef8af95a47ec01a8c280802a16d0f0c7',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5favx2_38',['sha384_one_block_avx2',['../intel-ipsec-mb_8h.html#ae5cf8bca3e0f1c7c40c110b88168669e',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5favx512_39',['sha384_one_block_avx512',['../intel-ipsec-mb_8h.html#a39fa05d7d02e8911e3d882f379d8661e',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5fsse_40',['sha384_one_block_sse',['../intel-ipsec-mb_8h.html#aa3bf3d1bd2a5d62035c972fe38827549',1,'intel-ipsec-mb.h']]], + ['sha384_5fsse_41',['sha384_sse',['../intel-ipsec-mb_8h.html#ad63e21495f9b798250f5c68371af4f37',1,'intel-ipsec-mb.h']]], + ['sha512_42',['sha512',['../structIMB__MGR.html#a62ebdd5e2b5a59c9e648511caf56f3f1',1,'IMB_MGR']]], + ['sha512_5favx_43',['sha512_avx',['../intel-ipsec-mb_8h.html#a17c4f20de186e1f04580d04d6551e844',1,'intel-ipsec-mb.h']]], + ['sha512_5favx2_44',['sha512_avx2',['../intel-ipsec-mb_8h.html#afc6b16cbe9cf1f840723aa6be3913131',1,'intel-ipsec-mb.h']]], + ['sha512_5favx512_45',['sha512_avx512',['../intel-ipsec-mb_8h.html#a7fa788fbcd353c13fd1eac251f658f1c',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_46',['sha512_one_block',['../structIMB__MGR.html#a88f1060a2adb8098fc085fc2b812addc',1,'IMB_MGR']]], + ['sha512_5fone_5fblock_5favx_47',['sha512_one_block_avx',['../intel-ipsec-mb_8h.html#a7d95ff83cbadc815cbe8c11b0af74b7e',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5favx2_48',['sha512_one_block_avx2',['../intel-ipsec-mb_8h.html#ada14329d2331d02241efaafc3ba26ffd',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5favx512_49',['sha512_one_block_avx512',['../intel-ipsec-mb_8h.html#a5d2e35fbfc14d979d87b5b9a63bf19c9',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5fsse_50',['sha512_one_block_sse',['../intel-ipsec-mb_8h.html#aa5e8c4861339d101c718449cf544d5c4',1,'intel-ipsec-mb.h']]], + ['sha512_5fsse_51',['sha512_sse',['../intel-ipsec-mb_8h.html#a392a99d0622122fe0e3c17611ef276a7',1,'intel-ipsec-mb.h']]], + ['sha_5f1_5fooo_52',['sha_1_ooo',['../structIMB__MGR.html#acdb53e812f3e4a307a497eeab28c4256',1,'IMB_MGR']]], + ['sha_5f224_5fooo_53',['sha_224_ooo',['../structIMB__MGR.html#aa174bf6846067a48f3f47ee5ebd74877',1,'IMB_MGR']]], + ['sha_5f256_5fooo_54',['sha_256_ooo',['../structIMB__MGR.html#a37fc5ccae7af099a8517566a20751265',1,'IMB_MGR']]], + ['sha_5f384_5fooo_55',['sha_384_ooo',['../structIMB__MGR.html#a51a499114f84eed88cb48bd3b461bd1c',1,'IMB_MGR']]], + ['sha_5f512_5fooo_56',['sha_512_ooo',['../structIMB__MGR.html#a6e21e034cd011b695bac5a711be65da1',1,'IMB_MGR']]], + ['shifted_5fhkey_57',['shifted_hkey',['../structgcm__key__data.html#a73b61a77635c0357b36b83aaf69d1237',1,'gcm_key_data']]], + ['shifted_5fhkey_5fk_58',['shifted_hkey_k',['../structgcm__key__data.html#a570a82f076ceb5da6719d1a554584820',1,'gcm_key_data']]], + ['sk16_59',['sk16',['../structkasumi__key__sched__s.html#a7e69de411b81753c10776442bbe03100',1,'kasumi_key_sched_s']]], + ['snow3g_5ff8_5f1_5fbuffer_60',['snow3g_f8_1_buffer',['../structIMB__MGR.html#ab70cbec09a61b5adeae37260c10234ce',1,'IMB_MGR']]], + ['snow3g_5ff8_5f1_5fbuffer_5fbit_61',['snow3g_f8_1_buffer_bit',['../structIMB__MGR.html#ad85fdcb101391b0a2488a3244d5f442d',1,'IMB_MGR']]], + ['snow3g_5ff8_5f1_5fbuffer_5fbit_5ft_62',['snow3g_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a7bd00b31d12260305accf0d37b60c958',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f1_5fbuffer_5ft_63',['snow3g_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a281315fb8b441d837ebed541b3a23380',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f2_5fbuffer_64',['snow3g_f8_2_buffer',['../structIMB__MGR.html#a25f46509b2f8926902a73e2210d697fd',1,'IMB_MGR']]], + ['snow3g_5ff8_5f2_5fbuffer_5ft_65',['snow3g_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#a4e8912e7e55a2ac891f85c19e7f29328',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f4_5fbuffer_66',['snow3g_f8_4_buffer',['../structIMB__MGR.html#a194a0f15a7345ee369780dbdeb4f52f6',1,'IMB_MGR']]], + ['snow3g_5ff8_5f4_5fbuffer_5ft_67',['snow3g_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#ae13b1154dcc13ca562e81471659d6099',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_68',['snow3g_f8_8_buffer',['../structIMB__MGR.html#a620d2d75bb50784aa0000b458197afaf',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_69',['snow3g_f8_8_buffer_multikey',['../structIMB__MGR.html#a898a4c56e1a746a328529c3665c5682e',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_5ft_70',['snow3g_f8_8_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a109b2a72d05bbe05a373ed5cf3c1f201',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_5ft_71',['snow3g_f8_8_buffer_t',['../intel-ipsec-mb_8h.html#a6155f472c6794e88289235752d83154a',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fiv_5fgen_72',['snow3g_f8_iv_gen',['../intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_73',['snow3g_f8_n_buffer',['../structIMB__MGR.html#a829c4cf1157dfc3fcb829df31e408e9d',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_74',['snow3g_f8_n_buffer_multikey',['../structIMB__MGR.html#a63a2a5f74e7cc2fc637896bdaeb6b262',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_5ft_75',['snow3g_f8_n_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a4049b601f9da6542e25d903efe524bc1',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_5ft_76',['snow3g_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a6a4a1da950207c9ed04f3f18ae1b7e43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5f1_5fbuffer_77',['snow3g_f9_1_buffer',['../structIMB__MGR.html#a9a4d2148b6e8b4426b2946e9fe695461',1,'IMB_MGR']]], + ['snow3g_5ff9_5f1_5fbuffer_5ft_78',['snow3g_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#ae7a48386f7190b4e46cd113eee413733',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5fiv_5fgen_79',['snow3g_f9_iv_gen',['../intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd',1,'intel-ipsec-mb.h']]], + ['snow3g_5finit_5fkey_5fsched_80',['snow3g_init_key_sched',['../structIMB__MGR.html#a38ff0528815be4d92481948664366ca9',1,'IMB_MGR']]], + ['snow3g_5finit_5fkey_5fsched_5ft_81',['snow3g_init_key_sched_t',['../intel-ipsec-mb_8h.html#adfa182361f1c179d75129b4e46841af5',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fsched_5fsize_82',['snow3g_key_sched_size',['../structIMB__MGR.html#a5e0002330399f643e8f0ca0c1b96a7a0',1,'IMB_MGR']]], + ['snow3g_5fkey_5fsched_5fsize_5ft_83',['snow3g_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad78a4f1c0d6de808f7046c0fbcd98d8f',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fschedule_5fs_84',['snow3g_key_schedule_s',['../structsnow3g__key__schedule__s.html',1,'']]], + ['snow3g_5fkey_5fschedule_5ft_85',['snow3g_key_schedule_t',['../intel-ipsec-mb_8h.html#ae50b48832b41b4b468cf933b44849213',1,'intel-ipsec-mb.h']]], + ['snow3g_5fuea2_5fooo_86',['snow3g_uea2_ooo',['../structIMB__MGR.html#ab0193bf46978026a82803c25e11acbfb',1,'IMB_MGR']]], + ['snow3g_5fuia2_87',['SNOW3G_UIA2',['../structIMB__JOB.html#a2a48f64b9344d0b6ca6775a4fdbe21b7',1,'IMB_JOB']]], + ['snow3g_5fuia2_5fooo_88',['snow3g_uia2_ooo',['../structIMB__MGR.html#aa7d4cfe53e0d70c22d3a274b8cd218af',1,'IMB_MGR']]], + ['snow_5fv_5faead_89',['SNOW_V_AEAD',['../structIMB__JOB.html#abf4a3f2db05c6583de5a99d9590ddca1',1,'IMB_JOB']]], + ['src_90',['src',['../structIMB__JOB.html#af83d63eb71a4f26b4915562bea554f28',1,'IMB_JOB']]], + ['sse_5favx_91',['sse_avx',['../structgcm__key__data.html#a908988e25607cd3e092bfcea9964deb1',1,'gcm_key_data']]], + ['status_92',['status',['../structIMB__JOB.html#ad4e08611eee5188cb84cb2dc337ef9f0',1,'IMB_JOB']]], + ['submit_5fburst_93',['submit_burst',['../structIMB__MGR.html#a6a790b2aa04138fce78c3347e285a31b',1,'IMB_MGR']]], + ['submit_5fburst_5fnocheck_94',['submit_burst_nocheck',['../structIMB__MGR.html#ad16230a6a671adbd597675abbc0f0a87',1,'IMB_MGR']]], + ['submit_5fburst_5ft_95',['submit_burst_t',['../intel-ipsec-mb_8h.html#a0cf6acb86385bc984ec5f916fe6e1066',1,'intel-ipsec-mb.h']]], + ['submit_5fcipher_5fburst_96',['submit_cipher_burst',['../structIMB__MGR.html#a8b52f1b3ca92eff48b945929a0eab839',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_5fnocheck_97',['submit_cipher_burst_nocheck',['../structIMB__MGR.html#acb2ee84d30247bb257635063db3ae308',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_5ft_98',['submit_cipher_burst_t',['../intel-ipsec-mb_8h.html#ac4bd89ab3cb1341eb4f30c4c501a9009',1,'intel-ipsec-mb.h']]], + ['submit_5fhash_5fburst_99',['submit_hash_burst',['../structIMB__MGR.html#a20a3d4d1e40aed26d83db5395b7ccc83',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_5fnocheck_100',['submit_hash_burst_nocheck',['../structIMB__MGR.html#a809782eb9dcbd264442bcd9edd2f4c2f',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_5ft_101',['submit_hash_burst_t',['../intel-ipsec-mb_8h.html#a4f6fca648737cec9ee869786e9c240b5',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_102',['submit_job',['../structIMB__MGR.html#ab143534d96e0d585b19dfc19553e929b',1,'IMB_MGR']]], + ['submit_5fjob_5favx_103',['submit_job_avx',['../intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx2_104',['submit_job_avx2',['../intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx512_105',['submit_job_avx512',['../intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_106',['submit_job_nocheck',['../structIMB__MGR.html#aab7350eeecc56b1ef45606b83519a72c',1,'IMB_MGR']]], + ['submit_5fjob_5fnocheck_5favx_107',['submit_job_nocheck_avx',['../intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx2_108',['submit_job_nocheck_avx2',['../intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx512_109',['submit_job_nocheck_avx512',['../intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5fsse_110',['submit_job_nocheck_sse',['../intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fsse_111',['submit_job_sse',['../intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5ft_112',['submit_job_t',['../intel-ipsec-mb_8h.html#a41297d789b434157e746e338f0af96ec',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_13.html b/docs/search/all_13.html new file mode 100644 index 00000000..bff7ab5c --- /dev/null +++ b/docs/search/all_13.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_13.js b/docs/search/all_13.js new file mode 100644 index 00000000..6cf3c83e --- /dev/null +++ b/docs/search/all_13.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['u_0',['u',['../structIMB__JOB.html#a5e6d8702069bf88ce81cd750df66bcbc',1,'IMB_JOB']]], + ['used_5farch_1',['used_arch',['../structIMB__MGR.html#a44b6c95f8ce611828b6acacdb459cdc9',1,'IMB_MGR']]], + ['user_5fdata_2',['user_data',['../structIMB__JOB.html#abd56907c9485830817f7b30870726aeb',1,'IMB_JOB']]], + ['user_5fdata2_3',['user_data2',['../structIMB__JOB.html#a4e68c8cf14552db625e26477650dfb8b',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_14.html b/docs/search/all_14.html new file mode 100644 index 00000000..a4ec4103 --- /dev/null +++ b/docs/search/all_14.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_14.js b/docs/search/all_14.js new file mode 100644 index 00000000..835924a7 --- /dev/null +++ b/docs/search/all_14.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['vaes_5favx512_0',['vaes_avx512',['../structgcm__key__data.html#a7ad44e105b23e41b7f79530b6370e509',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_15.html b/docs/search/all_15.html new file mode 100644 index 00000000..bafc8a26 --- /dev/null +++ b/docs/search/all_15.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_15.js b/docs/search/all_15.js new file mode 100644 index 00000000..63d1f09e --- /dev/null +++ b/docs/search/all_15.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['xcbc_0',['XCBC',['../structIMB__JOB.html#a3508ac7d49477dffb93b6b9cfbf0ebf3',1,'IMB_JOB']]], + ['xcbc_5fkeyexp_1',['xcbc_keyexp',['../structIMB__MGR.html#ae575c2350f5fca9a182c541795bb7d4d',1,'IMB_MGR']]], + ['xcbc_5fkeyexp_5ft_2',['xcbc_keyexp_t',['../intel-ipsec-mb_8h.html#abb36f71b9fb4d928c2ad9fa462b80ae6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_16.html b/docs/search/all_16.html new file mode 100644 index 00000000..5f7d3ea5 --- /dev/null +++ b/docs/search/all_16.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_16.js b/docs/search/all_16.js new file mode 100644 index 00000000..6057aa1c --- /dev/null +++ b/docs/search/all_16.js @@ -0,0 +1,15 @@ +var searchData= +[ + ['zuc256_5feea3_5fooo_0',['zuc256_eea3_ooo',['../structIMB__MGR.html#a6d05b0a20196be514182920710dce2a8',1,'IMB_MGR']]], + ['zuc256_5feia3_5fooo_1',['zuc256_eia3_ooo',['../structIMB__MGR.html#a5f8744a8d64273d3f4cbcd9ad41ce3d5',1,'IMB_MGR']]], + ['zuc_5feea3_5f1_5fbuffer_5ft_2',['zuc_eea3_1_buffer_t',['../intel-ipsec-mb_8h.html#a0a3032f0698c42bd486d053e90c44fdd',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5f4_5fbuffer_5ft_3',['zuc_eea3_4_buffer_t',['../intel-ipsec-mb_8h.html#a7df2e3f063333d677ef7668626126c4d',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fiv_5fgen_4',['zuc_eea3_iv_gen',['../intel-ipsec-mb_8h.html#a4931dd6b1b91456bfe885e28ada0aaf3',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fn_5fbuffer_5ft_5',['zuc_eea3_n_buffer_t',['../intel-ipsec-mb_8h.html#a303f2dd11b1193938596b4ec69edf8cc',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fooo_6',['zuc_eea3_ooo',['../structIMB__MGR.html#a26d22c5ff963b9c7ffc8af19292bce5c',1,'IMB_MGR']]], + ['zuc_5feia3_7',['ZUC_EIA3',['../structIMB__JOB.html#ab42f79355b9173c63aca9b0b577a2b61',1,'IMB_JOB']]], + ['zuc_5feia3_5f1_5fbuffer_5ft_8',['zuc_eia3_1_buffer_t',['../intel-ipsec-mb_8h.html#a95ed5580686f708ba2313179cd6064a8',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fiv_5fgen_9',['zuc_eia3_iv_gen',['../intel-ipsec-mb_8h.html#ae3cefd0e992f1390110f0ef9a3aa510f',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fn_5fbuffer_5ft_10',['zuc_eia3_n_buffer_t',['../intel-ipsec-mb_8h.html#ada45082bb04dd727c0deec5c4fc96fed',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fooo_11',['zuc_eia3_ooo',['../structIMB__MGR.html#a910f747aa1a0e6a46f87b0dfd2189219',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_2.html b/docs/search/all_2.html new file mode 100644 index 00000000..7e0d7b7a --- /dev/null +++ b/docs/search/all_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_2.js b/docs/search/all_2.js new file mode 100644 index 00000000..aac78601 --- /dev/null +++ b/docs/search/all_2.js @@ -0,0 +1,41 @@ +var searchData= +[ + ['cbcs_0',['CBCS',['../structIMB__JOB.html#a832c73263792bf8895cf028035730c4e',1,'IMB_JOB']]], + ['ccm_1',['CCM',['../structIMB__JOB.html#ad43a9c9149ec16b8e9164771ec38d379',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_2',['CHACHA20_POLY1305',['../structIMB__JOB.html#ace537bcd7b871fddd4514b7fd4f678a3',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_5fcontext_5fdata_3',['chacha20_poly1305_context_data',['../structchacha20__poly1305__context__data.html',1,'']]], + ['chacha20_5fpoly1305_5fdec_5fupdate_4',['chacha20_poly1305_dec_update',['../structIMB__MGR.html#a715aede4176564ac475e42c8a377bd8d',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5fenc_5fupdate_5',['chacha20_poly1305_enc_update',['../structIMB__MGR.html#ab2c87c40f9844795e3987536c602ce03',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5ffinalize_6',['chacha20_poly1305_finalize',['../structIMB__MGR.html#ab18b3fcba2aa252bb31f66093bc62abb',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5finit_7',['chacha20_poly1305_init',['../structIMB__MGR.html#a67b96b1b26fa25391f9f4b83a7a4b6c0',1,'IMB_MGR']]], + ['chacha_5fpoly_5fenc_5fdec_5fupdate_5ft_8',['chacha_poly_enc_dec_update_t',['../intel-ipsec-mb_8h.html#a437de868b8dba4f9b878d99a82f80548',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5ffinalize_5ft_9',['chacha_poly_finalize_t',['../intel-ipsec-mb_8h.html#a1a96f063bf1bd07de158985b57cff4a0',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5finit_5ft_10',['chacha_poly_init_t',['../intel-ipsec-mb_8h.html#a4b0b9ffa331f8d675d24911639f0374f',1,'intel-ipsec-mb.h']]], + ['chain_5forder_11',['chain_order',['../structIMB__JOB.html#a41e9fa248ebb040ce1e074a2444b55d2',1,'IMB_JOB']]], + ['cipher_5fdirection_12',['cipher_direction',['../structIMB__JOB.html#a1449c51b88ccdb2f855525b89c05eb84',1,'IMB_JOB']]], + ['cipher_5ffields_13',['cipher_fields',['../structIMB__JOB.html#a006bf024dd3e809fad23c3df813c6fb2',1,'IMB_JOB']]], + ['cipher_5ffunc_14',['cipher_func',['../structIMB__JOB.html#a652eb3eee14e59be703bf28e69ca0671',1,'IMB_JOB']]], + ['cipher_5fmode_15',['cipher_mode',['../structIMB__JOB.html#a78113e875fdb681e775fd32ee67c7597',1,'IMB_JOB']]], + ['cipher_5fstart_5foffset_5fin_5fbits_16',['cipher_start_offset_in_bits',['../structIMB__JOB.html#a60a01da0056367a6f0a326265e4da4fb',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbits_17',['cipher_start_src_offset_in_bits',['../structIMB__JOB.html#af5c226d1c875c4e30a8f7bd7c03f05b1',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbytes_18',['cipher_start_src_offset_in_bytes',['../structIMB__JOB.html#adda0b9bd22d445402bb9866e423707c4',1,'IMB_JOB']]], + ['cmac_19',['CMAC',['../structIMB__JOB.html#a499c1e90c9004ab7bb50fac98b6a9048',1,'IMB_JOB']]], + ['cmac_5fsubkey_5fgen_5f128_20',['cmac_subkey_gen_128',['../structIMB__MGR.html#aa3404b7811d57821ea574994e0c88995',1,'IMB_MGR']]], + ['cmac_5fsubkey_5fgen_5f256_21',['cmac_subkey_gen_256',['../structIMB__MGR.html#abd500d2cc03f6470ac6988785bebdf76',1,'IMB_MGR']]], + ['cmac_5fsubkey_5fgen_5ft_22',['cmac_subkey_gen_t',['../intel-ipsec-mb_8h.html#aab07996d622d682adc1db8286e633cb3',1,'intel-ipsec-mb.h']]], + ['crc10_5fiuup_5fdata_23',['crc10_iuup_data',['../structIMB__MGR.html#a7d0979deda4e2ff7ac202c82611ce94c',1,'IMB_MGR']]], + ['crc11_5ffp_5fheader_24',['crc11_fp_header',['../structIMB__MGR.html#a4b901916c2f262f1c873e4cec7b7f474',1,'IMB_MGR']]], + ['crc16_5ffp_5fdata_25',['crc16_fp_data',['../structIMB__MGR.html#ad566ac43002c2ba8ecb93048d5edc41b',1,'IMB_MGR']]], + ['crc16_5fx25_26',['crc16_x25',['../structIMB__MGR.html#a52a7ab10d6434820e9b31d0e772ea0bc',1,'IMB_MGR']]], + ['crc24_5flte_5fa_27',['crc24_lte_a',['../structIMB__MGR.html#a058cfc60f2fdd68678b36f4f038270ac',1,'IMB_MGR']]], + ['crc24_5flte_5fb_28',['crc24_lte_b',['../structIMB__MGR.html#abe8406f3582d3540edb770e153b74c3e',1,'IMB_MGR']]], + ['crc32_5fethernet_5ffcs_29',['crc32_ethernet_fcs',['../structIMB__MGR.html#a8483109fbe42f8b7fa6e7f53e016d5ec',1,'IMB_MGR']]], + ['crc32_5ffn_5ft_30',['crc32_fn_t',['../intel-ipsec-mb_8h.html#a56dd197c301ad27dc9ec0862721cb73d',1,'intel-ipsec-mb.h']]], + ['crc32_5fsctp_31',['crc32_sctp',['../structIMB__MGR.html#a4aed66c9e73808a65a062e1a5b0f66aa',1,'IMB_MGR']]], + ['crc32_5fwimax_5fofdma_5fdata_32',['crc32_wimax_ofdma_data',['../structIMB__MGR.html#ad08e44bc4edc54c556ea2245f848b974',1,'IMB_MGR']]], + ['crc6_5fiuup_5fheader_33',['crc6_iuup_header',['../structIMB__MGR.html#aa610dc7226673020be410762b995b725',1,'IMB_MGR']]], + ['crc7_5ffp_5fheader_34',['crc7_fp_header',['../structIMB__MGR.html#ac819ddcc968604186983d58f275bd24b',1,'IMB_MGR']]], + ['crc8_5fwimax_5fofdma_5fhcs_35',['crc8_wimax_ofdma_hcs',['../structIMB__MGR.html#a5e5be572c46ed01ac11fc95738373e2a',1,'IMB_MGR']]], + ['ctx_36',['ctx',['../structIMB__JOB.html#a053ceeb65e611c48a97a3e10d03d47eb',1,'IMB_JOB::ctx()'],['../structIMB__JOB.html#a4abdf3311d861979dc39669b3b7f1ed8',1,'IMB_JOB::ctx()']]], + ['current_5fcounter_37',['current_counter',['../structgcm__context__data.html#a2a454a5e44422aaec135dc7807aa55a4',1,'gcm_context_data']]] +]; diff --git a/docs/search/all_3.html b/docs/search/all_3.html new file mode 100644 index 00000000..a56694f0 --- /dev/null +++ b/docs/search/all_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_3.js b/docs/search/all_3.js new file mode 100644 index 00000000..0be3da31 --- /dev/null +++ b/docs/search/all_3.js @@ -0,0 +1,19 @@ +var searchData= +[ + ['dec_5fkeys_0',['dec_keys',['../structIMB__JOB.html#af78a3df5a9f79f329c8d1095b9ecea05',1,'IMB_JOB']]], + ['declare_5faligned_1',['DECLARE_ALIGNED',['../intel-ipsec-mb_8h.html#ad920a47ed8f57af8b2cf6bcd18a3e41c',1,'intel-ipsec-mb.h']]], + ['des3_5fdec_5fooo_2',['des3_dec_ooo',['../structIMB__MGR.html#a2ab69fdbef6a9a0574fcbb5ea9a7da3b',1,'IMB_MGR']]], + ['des3_5fenc_5fooo_3',['des3_enc_ooo',['../structIMB__MGR.html#a87f7566634911b4dba48a19a0c3a4993',1,'IMB_MGR']]], + ['des_5fdec_5fooo_4',['des_dec_ooo',['../structIMB__MGR.html#af03e401b002e2e415c8a927c46e1517b',1,'IMB_MGR']]], + ['des_5fenc_5fooo_5',['des_enc_ooo',['../structIMB__MGR.html#a54ec50411917f1aa9016db268ac638c1',1,'IMB_MGR']]], + ['des_5fkey_5fsched_6',['des_key_sched',['../structIMB__MGR.html#aae9afac5c2c1bf7b27d187d10778a1ee',1,'IMB_MGR']]], + ['des_5fkey_5fschedule_7',['des_key_schedule',['../intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440',1,'intel-ipsec-mb.h']]], + ['des_5fkeysched_5ft_8',['des_keysched_t',['../intel-ipsec-mb_8h.html#a60a93569304a978c798bb8c4a4f7f52f',1,'intel-ipsec-mb.h']]], + ['docsis128_5fcrc32_5fsec_5fooo_9',['docsis128_crc32_sec_ooo',['../structIMB__MGR.html#acf438b50cb95a4fff1276b6666d22ad4',1,'IMB_MGR']]], + ['docsis128_5fsec_5fooo_10',['docsis128_sec_ooo',['../structIMB__MGR.html#a93cb5904608e179c49917f80c64fabd1',1,'IMB_MGR']]], + ['docsis256_5fcrc32_5fsec_5fooo_11',['docsis256_crc32_sec_ooo',['../structIMB__MGR.html#af6531598b8dd0131a11e0cf95854605d',1,'IMB_MGR']]], + ['docsis256_5fsec_5fooo_12',['docsis256_sec_ooo',['../structIMB__MGR.html#acb6e3b70e16194274e2eaefdff8f9e89',1,'IMB_MGR']]], + ['docsis_5fdes_5fdec_5fooo_13',['docsis_des_dec_ooo',['../structIMB__MGR.html#a3d5173cea3d4932576f0a5b3abbc1f40',1,'IMB_MGR']]], + ['docsis_5fdes_5fenc_5fooo_14',['docsis_des_enc_ooo',['../structIMB__MGR.html#a937fb0d3b113d203ee1a4beaf5b5b98f',1,'IMB_MGR']]], + ['dst_15',['dst',['../structIMB__JOB.html#ac1f7c363e7709cf1a7de1364aa8625e2',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_4.html b/docs/search/all_4.html new file mode 100644 index 00000000..61724dff --- /dev/null +++ b/docs/search/all_4.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_4.js b/docs/search/all_4.js new file mode 100644 index 00000000..086cbee1 --- /dev/null +++ b/docs/search/all_4.js @@ -0,0 +1,12 @@ +var searchData= +[ + ['earliest_5fjob_0',['earliest_job',['../structIMB__MGR.html#ac5cd33ac3bf985dc31ccc716fb3a7747',1,'IMB_MGR']]], + ['eea3_5f1_5fbuffer_1',['eea3_1_buffer',['../structIMB__MGR.html#aeff54b370fbe495ce96370811d280140',1,'IMB_MGR']]], + ['eea3_5f4_5fbuffer_2',['eea3_4_buffer',['../structIMB__MGR.html#ac5ad1bf8ecdcfc51eaf681fb353c9621',1,'IMB_MGR']]], + ['eea3_5fn_5fbuffer_3',['eea3_n_buffer',['../structIMB__MGR.html#a67d8a55ba87742b8cf282d23ed9e3ef5',1,'IMB_MGR']]], + ['eia3_5f1_5fbuffer_4',['eia3_1_buffer',['../structIMB__MGR.html#a3ed58bdf37ab6d5f0d28271882d52327',1,'IMB_MGR']]], + ['eia3_5fn_5fbuffer_5',['eia3_n_buffer',['../structIMB__MGR.html#ab515c5aec39b1a9a31454b945e5ccd35',1,'IMB_MGR']]], + ['enc_5fkeys_6',['enc_keys',['../structIMB__JOB.html#ac656e2d98f36bd5547cc9fa297c1de62',1,'IMB_JOB']]], + ['end_5fooo_7',['end_ooo',['../structIMB__MGR.html#a6e1ab772182b97f931af3d4685785353',1,'IMB_MGR']]], + ['expanded_5fkeys_8',['expanded_keys',['../structgcm__key__data.html#aa5bda4f55b8393be151cfe28a9dc1ca9',1,'gcm_key_data']]] +]; diff --git a/docs/search/all_5.html b/docs/search/all_5.html new file mode 100644 index 00000000..b03a4e7a --- /dev/null +++ b/docs/search/all_5.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_5.js b/docs/search/all_5.js new file mode 100644 index 00000000..2bcdf24a --- /dev/null +++ b/docs/search/all_5.js @@ -0,0 +1,20 @@ +var searchData= +[ + ['f8_5f1_5fbuffer_0',['f8_1_buffer',['../structIMB__MGR.html#af854e664cef84cb8be4495a1af5e9cb5',1,'IMB_MGR']]], + ['f8_5f1_5fbuffer_5fbit_1',['f8_1_buffer_bit',['../structIMB__MGR.html#aacf6ce3f508fef115dcd2dc415ca7379',1,'IMB_MGR']]], + ['f8_5f2_5fbuffer_2',['f8_2_buffer',['../structIMB__MGR.html#ac733fc4c5ee6e80e8e71b20a9bb2dc6e',1,'IMB_MGR']]], + ['f8_5f3_5fbuffer_3',['f8_3_buffer',['../structIMB__MGR.html#a929b9c50bc98bf37daec7a1603f10118',1,'IMB_MGR']]], + ['f8_5f4_5fbuffer_4',['f8_4_buffer',['../structIMB__MGR.html#af0b1c5d3b269b252a75a55df8bdf472c',1,'IMB_MGR']]], + ['f8_5fn_5fbuffer_5',['f8_n_buffer',['../structIMB__MGR.html#ae0f2316d96e46b64d08e3cf73a93b39d',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_6',['f9_1_buffer',['../structIMB__MGR.html#afafb95a00fe6633fb6901855928b08bb',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_5fuser_7',['f9_1_buffer_user',['../structIMB__MGR.html#a4fbdac066c9d0fbd1d586a2e5b2319c3',1,'IMB_MGR']]], + ['features_8',['features',['../structIMB__MGR.html#af877c7669265775604857c6b81063bdb',1,'IMB_MGR']]], + ['flags_9',['flags',['../structIMB__MGR.html#ae1a596c9c9d6ca1292c6116575021c14',1,'IMB_MGR']]], + ['flush_5fjob_10',['flush_job',['../structIMB__MGR.html#a225e3b35eef8e18fd09c429fbb2f70b5',1,'IMB_MGR']]], + ['flush_5fjob_5favx_11',['flush_job_avx',['../intel-ipsec-mb_8h.html#abc6b0f1c857f26b834f22b6bd8f51846',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx2_12',['flush_job_avx2',['../intel-ipsec-mb_8h.html#ae242d142065172898129f30e6bc40d8c',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx512_13',['flush_job_avx512',['../intel-ipsec-mb_8h.html#ac7aec54f653aeee2554ba980d62fd491',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5fsse_14',['flush_job_sse',['../intel-ipsec-mb_8h.html#a96b880b8a32b0466f880d4f22d95cc3b',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5ft_15',['flush_job_t',['../intel-ipsec-mb_8h.html#ae64d0f677e42b8f8d50289113cf5e01e',1,'intel-ipsec-mb.h']]], + ['free_5fmb_5fmgr_16',['free_mb_mgr',['../intel-ipsec-mb_8h.html#a238b96cc36c0dc64798a1244b94add8b',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_6.html b/docs/search/all_6.html new file mode 100644 index 00000000..690aaafe --- /dev/null +++ b/docs/search/all_6.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_6.js b/docs/search/all_6.js new file mode 100644 index 00000000..b334655c --- /dev/null +++ b/docs/search/all_6.js @@ -0,0 +1,63 @@ +var searchData= +[ + ['gcm_0',['GCM',['../structIMB__JOB.html#ac9fc7aef831cfc1ca598a55b250dd858',1,'IMB_JOB']]], + ['gcm128_5fdec_1',['gcm128_dec',['../structIMB__MGR.html#a2ce2f6e6d61953b24a5859279534b02a',1,'IMB_MGR']]], + ['gcm128_5fdec_5ffinalize_2',['gcm128_dec_finalize',['../structIMB__MGR.html#af95fa831fb24629e19613e8594e5d32a',1,'IMB_MGR']]], + ['gcm128_5fdec_5fupdate_3',['gcm128_dec_update',['../structIMB__MGR.html#a28ddfed1a807fedb9c77c99736278dcb',1,'IMB_MGR']]], + ['gcm128_5fenc_4',['gcm128_enc',['../structIMB__MGR.html#a9f8b9687fbb17f7a78fc4ba79e2749fd',1,'IMB_MGR']]], + ['gcm128_5fenc_5ffinalize_5',['gcm128_enc_finalize',['../structIMB__MGR.html#a8efc0cf95907805289a297c024ce0cac',1,'IMB_MGR']]], + ['gcm128_5fenc_5fupdate_6',['gcm128_enc_update',['../structIMB__MGR.html#ad2c64d9d5f9ffd65d13e73205cb803cd',1,'IMB_MGR']]], + ['gcm128_5finit_7',['gcm128_init',['../structIMB__MGR.html#ad8c004899d29974a3f4c034370422c31',1,'IMB_MGR']]], + ['gcm128_5finit_5fvar_5fiv_8',['gcm128_init_var_iv',['../structIMB__MGR.html#aebd9eab76cd056add559e6bea328dbf0',1,'IMB_MGR']]], + ['gcm128_5fpre_9',['gcm128_pre',['../structIMB__MGR.html#a207c78e01e4aa1a35de1bbd2cc0a6ee2',1,'IMB_MGR']]], + ['gcm128_5fprecomp_10',['gcm128_precomp',['../structIMB__MGR.html#a69e4dd044703cd3aa9c4c36e04b2a701',1,'IMB_MGR']]], + ['gcm192_5fdec_11',['gcm192_dec',['../structIMB__MGR.html#adff2cf46e7cbb5e2926f348623d8ad2a',1,'IMB_MGR']]], + ['gcm192_5fdec_5ffinalize_12',['gcm192_dec_finalize',['../structIMB__MGR.html#a4d7879172344be144b92be68b2e2a81a',1,'IMB_MGR']]], + ['gcm192_5fdec_5fupdate_13',['gcm192_dec_update',['../structIMB__MGR.html#a66fa94f9c07061f3d93b78ed505931e3',1,'IMB_MGR']]], + ['gcm192_5fenc_14',['gcm192_enc',['../structIMB__MGR.html#aaaeb1676e0cf19e3558d10e537f71bda',1,'IMB_MGR']]], + ['gcm192_5fenc_5ffinalize_15',['gcm192_enc_finalize',['../structIMB__MGR.html#a91ad9ce7a4b3c9ceeffa9dd712013915',1,'IMB_MGR']]], + ['gcm192_5fenc_5fupdate_16',['gcm192_enc_update',['../structIMB__MGR.html#a3b5c3407cc4595420dbe43e98090e558',1,'IMB_MGR']]], + ['gcm192_5finit_17',['gcm192_init',['../structIMB__MGR.html#a006315a7045996d451c7dfd03b96a076',1,'IMB_MGR']]], + ['gcm192_5finit_5fvar_5fiv_18',['gcm192_init_var_iv',['../structIMB__MGR.html#a7bfa7b17c0fb36dc832138a86948991f',1,'IMB_MGR']]], + ['gcm192_5fpre_19',['gcm192_pre',['../structIMB__MGR.html#a149b463ecf68ee78d39c33e183ed8997',1,'IMB_MGR']]], + ['gcm192_5fprecomp_20',['gcm192_precomp',['../structIMB__MGR.html#abec8bdb319ca132028d5b787bf60a895',1,'IMB_MGR']]], + ['gcm256_5fdec_21',['gcm256_dec',['../structIMB__MGR.html#acfdfb511771cac71e032c5bf1c9273b7',1,'IMB_MGR']]], + ['gcm256_5fdec_5ffinalize_22',['gcm256_dec_finalize',['../structIMB__MGR.html#a5c2e4a1296e33f5c9eaff2067a6432d4',1,'IMB_MGR']]], + ['gcm256_5fdec_5fupdate_23',['gcm256_dec_update',['../structIMB__MGR.html#a99fcb1d18fa45044063c6c288b0fc769',1,'IMB_MGR']]], + ['gcm256_5fenc_24',['gcm256_enc',['../structIMB__MGR.html#a79767326692a8d4faabe5ab435dc5df8',1,'IMB_MGR']]], + ['gcm256_5fenc_5ffinalize_25',['gcm256_enc_finalize',['../structIMB__MGR.html#a9764d27efef2491a86d517f38dd9b6ef',1,'IMB_MGR']]], + ['gcm256_5fenc_5fupdate_26',['gcm256_enc_update',['../structIMB__MGR.html#ac131ceec0dcb0d08097b76c8abe9e024',1,'IMB_MGR']]], + ['gcm256_5finit_27',['gcm256_init',['../structIMB__MGR.html#a8987d7d491628d9d80f12820468ac541',1,'IMB_MGR']]], + ['gcm256_5finit_5fvar_5fiv_28',['gcm256_init_var_iv',['../structIMB__MGR.html#afa275308dab467d0c68dc921bd68d1a4',1,'IMB_MGR']]], + ['gcm256_5fpre_29',['gcm256_pre',['../structIMB__MGR.html#a44b0b9d218043be965a84c29ced2584b',1,'IMB_MGR']]], + ['gcm256_5fprecomp_30',['gcm256_precomp',['../structIMB__MGR.html#a1ea15abc11b618ceb438c7f282643c52',1,'IMB_MGR']]], + ['gcm_5fcontext_5fdata_31',['gcm_context_data',['../structgcm__context__data.html',1,'']]], + ['gcm_5fkey_5fdata_32',['gcm_key_data',['../structgcm__key__data.html',1,'']]], + ['get_5fcompleted_5fjob_33',['get_completed_job',['../structIMB__MGR.html#a99b7111c8fc59161e0bf291d13b1f195',1,'IMB_MGR']]], + ['get_5fcompleted_5fjob_5favx_34',['get_completed_job_avx',['../intel-ipsec-mb_8h.html#a9d115c47670330bd01b960f2188169d9',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx2_35',['get_completed_job_avx2',['../intel-ipsec-mb_8h.html#ad85bdbf56038fd72ceda7e50deea586e',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx512_36',['get_completed_job_avx512',['../intel-ipsec-mb_8h.html#a693afd528fcc0809cc667b6c38de84dc',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5fsse_37',['get_completed_job_sse',['../intel-ipsec-mb_8h.html#a267a718472e0c1b6585402e81e44e7e9',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5ft_38',['get_completed_job_t',['../intel-ipsec-mb_8h.html#ac8cb9a545cdef14f60f36fe8888e6e7c',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_39',['get_next_job',['../structIMB__MGR.html#a35939892cd168eb7ce9c6c29c5401709',1,'IMB_MGR']]], + ['get_5fnext_5fjob_5favx_40',['get_next_job_avx',['../intel-ipsec-mb_8h.html#a3518b02f06a82e9f83cd0435f3cf25c9',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx2_41',['get_next_job_avx2',['../intel-ipsec-mb_8h.html#a49a3d54979188b3c68bdd621043125f7',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx512_42',['get_next_job_avx512',['../intel-ipsec-mb_8h.html#a40c08ba2c387e24f2096b1213c46c812',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5fsse_43',['get_next_job_sse',['../intel-ipsec-mb_8h.html#a055db1f6f16bc17724bdc64d0f313871',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5ft_44',['get_next_job_t',['../intel-ipsec-mb_8h.html#a5a533d852e3b65e9e5978fec37da43b5',1,'intel-ipsec-mb.h']]], + ['ghash_45',['GHASH',['../structIMB__JOB.html#af24c379b5414a5d3ec290954b98b1c72',1,'IMB_JOB']]], + ['ghash_46',['ghash',['../structIMB__MGR.html#afc7d7ae10e8e6f37b9ec894059818fad',1,'IMB_MGR']]], + ['ghash_5fkeys_47',['ghash_keys',['../structgcm__key__data.html#af5adb4225d569050207cfcfe00fe273a',1,'gcm_key_data']]], + ['ghash_5fpre_48',['ghash_pre',['../structIMB__MGR.html#af6a1c323981b8c3cfe09f94cd9dd58e2',1,'IMB_MGR']]], + ['ghash_5ft_49',['ghash_t',['../intel-ipsec-mb_8h.html#a12e0c05f0ecec8d75d4b288a7ec11c84',1,'intel-ipsec-mb.h']]], + ['gmac_50',['GMAC',['../structIMB__JOB.html#a07832e4571ec56f377dffa477b7aafe0',1,'IMB_JOB']]], + ['gmac128_5ffinalize_51',['gmac128_finalize',['../structIMB__MGR.html#a00ce9a8422a961166ce9855d4815cc2e',1,'IMB_MGR']]], + ['gmac128_5finit_52',['gmac128_init',['../structIMB__MGR.html#a1098c38784f6c6e4e3a3380f8be8ecc1',1,'IMB_MGR']]], + ['gmac128_5fupdate_53',['gmac128_update',['../structIMB__MGR.html#a90aad4f47448a1959b7cf747438d7ce9',1,'IMB_MGR']]], + ['gmac192_5ffinalize_54',['gmac192_finalize',['../structIMB__MGR.html#a000b24d7fbb9b57ed6bd4147d225e015',1,'IMB_MGR']]], + ['gmac192_5finit_55',['gmac192_init',['../structIMB__MGR.html#a3dd58a9d9b4fb6cf85fa26175ffac9a6',1,'IMB_MGR']]], + ['gmac192_5fupdate_56',['gmac192_update',['../structIMB__MGR.html#a5220f0d243291ae5a2f43fbc93d18725',1,'IMB_MGR']]], + ['gmac256_5ffinalize_57',['gmac256_finalize',['../structIMB__MGR.html#a3bb98c9a9c8545f363b74627870c7038',1,'IMB_MGR']]], + ['gmac256_5finit_58',['gmac256_init',['../structIMB__MGR.html#aca7d6388863eb76b60651e7b16c41b82',1,'IMB_MGR']]], + ['gmac256_5fupdate_59',['gmac256_update',['../structIMB__MGR.html#a0720a1fc9c0652c6b3aa7dfab070f73d',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_7.html b/docs/search/all_7.html new file mode 100644 index 00000000..b355aaaa --- /dev/null +++ b/docs/search/all_7.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_7.js b/docs/search/all_7.js new file mode 100644 index 00000000..21cb71d4 --- /dev/null +++ b/docs/search/all_7.js @@ -0,0 +1,22 @@ +var searchData= +[ + ['hash_0',['hash',['../structchacha20__poly1305__context__data.html#ae12447b2fd2e9f8a6f09f443bd303887',1,'chacha20_poly1305_context_data']]], + ['hash_5falg_1',['hash_alg',['../structIMB__JOB.html#aa4d9b62d3353150aba88aec5d8d08853',1,'IMB_JOB']]], + ['hash_5ffn_5ft_2',['hash_fn_t',['../intel-ipsec-mb_8h.html#afacacc8b74c05c80a0ee255c68637490',1,'intel-ipsec-mb.h']]], + ['hash_5ffunc_3',['hash_func',['../structIMB__JOB.html#a62883bf8707d09816753832bef5d2902',1,'IMB_JOB']]], + ['hash_5flen_4',['hash_len',['../structchacha20__poly1305__context__data.html#ade4568d041695b318df773787872127f',1,'chacha20_poly1305_context_data']]], + ['hash_5fone_5fblock_5ft_5',['hash_one_block_t',['../intel-ipsec-mb_8h.html#aade58eb541a45650ed7d875dc2be97fa',1,'intel-ipsec-mb.h']]], + ['hash_5fstart_5fsrc_5foffset_5fin_5fbytes_6',['hash_start_src_offset_in_bytes',['../structIMB__JOB.html#af7cdc3d086f6e2b12d21c1383b2bd152',1,'IMB_JOB']]], + ['hec_5f32_7',['hec_32',['../structIMB__MGR.html#a0863c9f64126aba5cb47628493dae582',1,'IMB_MGR']]], + ['hec_5f32_5ft_8',['hec_32_t',['../intel-ipsec-mb_8h.html#aa34b11bf4be97c764b283aedc0589575',1,'intel-ipsec-mb.h']]], + ['hec_5f64_9',['hec_64',['../structIMB__MGR.html#a4128ee02f4cbef5aa1e6b0bb5fd46097',1,'IMB_MGR']]], + ['hec_5f64_5ft_10',['hec_64_t',['../intel-ipsec-mb_8h.html#aa9afd4ee91f0d82729afcf5f1cdcb808',1,'intel-ipsec-mb.h']]], + ['high_11',['high',['../structimb__uint128__t.html#aa65815c9b383e888758ae21d0c6159c9',1,'imb_uint128_t']]], + ['hmac_12',['HMAC',['../structIMB__JOB.html#aa84f5eefd378ab65ebe0b302ccb0df92',1,'IMB_JOB']]], + ['hmac_5fmd5_5fooo_13',['hmac_md5_ooo',['../structIMB__MGR.html#a855d14df536b22ceb175dc495ced891c',1,'IMB_MGR']]], + ['hmac_5fsha_5f1_5fooo_14',['hmac_sha_1_ooo',['../structIMB__MGR.html#a49be1d77b96d338c08200dce82e452d3',1,'IMB_MGR']]], + ['hmac_5fsha_5f224_5fooo_15',['hmac_sha_224_ooo',['../structIMB__MGR.html#ab9226cf37e41c2a13ae6003305654193',1,'IMB_MGR']]], + ['hmac_5fsha_5f256_5fooo_16',['hmac_sha_256_ooo',['../structIMB__MGR.html#a633807f7eaae35ada650a2c22f1366be',1,'IMB_MGR']]], + ['hmac_5fsha_5f384_5fooo_17',['hmac_sha_384_ooo',['../structIMB__MGR.html#afd72d6af1d8ff605e8aaaf984f0700ed',1,'IMB_MGR']]], + ['hmac_5fsha_5f512_5fooo_18',['hmac_sha_512_ooo',['../structIMB__MGR.html#af0cf9149495b0805a4f6f511c0fbff5b',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_8.html b/docs/search/all_8.html new file mode 100644 index 00000000..26cb33bb --- /dev/null +++ b/docs/search/all_8.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_8.js b/docs/search/all_8.js new file mode 100644 index 00000000..338e64c9 --- /dev/null +++ b/docs/search/all_8.js @@ -0,0 +1,363 @@ +var searchData= +[ + ['imb_5faes128_5fcfb_5fone_0',['IMB_AES128_CFB_ONE',['../intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_1',['IMB_AES128_GCM_DEC',['../intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5ffinalize_2',['IMB_AES128_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5fupdate_3',['IMB_AES128_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_4',['IMB_AES128_GCM_ENC',['../intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5ffinalize_5',['IMB_AES128_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5fupdate_6',['IMB_AES128_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_7',['IMB_AES128_GCM_INIT',['../intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_5fvar_5fiv_8',['IMB_AES128_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fpre_9',['IMB_AES128_GCM_PRE',['../intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fprecomp_10',['IMB_AES128_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5ffinalize_11',['IMB_AES128_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5finit_12',['IMB_AES128_GMAC_INIT',['../intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5fupdate_13',['IMB_AES128_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_14',['IMB_AES192_GCM_DEC',['../intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5ffinalize_15',['IMB_AES192_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5fupdate_16',['IMB_AES192_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_17',['IMB_AES192_GCM_ENC',['../intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5ffinalize_18',['IMB_AES192_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5fupdate_19',['IMB_AES192_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_20',['IMB_AES192_GCM_INIT',['../intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_5fvar_5fiv_21',['IMB_AES192_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fpre_22',['IMB_AES192_GCM_PRE',['../intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fprecomp_23',['IMB_AES192_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5ffinalize_24',['IMB_AES192_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5finit_25',['IMB_AES192_GMAC_INIT',['../intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5fupdate_26',['IMB_AES192_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_27',['IMB_AES256_GCM_DEC',['../intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5ffinalize_28',['IMB_AES256_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5fupdate_29',['IMB_AES256_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_30',['IMB_AES256_GCM_ENC',['../intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5ffinalize_31',['IMB_AES256_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5fupdate_32',['IMB_AES256_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_33',['IMB_AES256_GCM_INIT',['../intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_5fvar_5fiv_34',['IMB_AES256_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fpre_35',['IMB_AES256_GCM_PRE',['../intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fprecomp_36',['IMB_AES256_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5ffinalize_37',['IMB_AES256_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5finit_38',['IMB_AES256_GMAC_INIT',['../intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5fupdate_39',['IMB_AES256_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fblock_5fsize_40',['IMB_AES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f128_41',['IMB_AES_CMAC_SUBKEY_GEN_128',['../intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f256_42',['IMB_AES_CMAC_SUBKEY_GEN_256',['../intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f128_43',['IMB_AES_KEYEXP_128',['../intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f192_44',['IMB_AES_KEYEXP_192',['../intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f256_45',['IMB_AES_KEYEXP_256',['../intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fxcbc_5fkeyexp_46',['IMB_AES_XCBC_KEYEXP',['../intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_47',['IMB_ARCH',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx_48',['IMB_ARCH_AVX',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx2_49',['IMB_ARCH_AVX2',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx512_50',['IMB_ARCH_AVX512',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnoaesni_51',['IMB_ARCH_NOAESNI',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnone_52',['IMB_ARCH_NONE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnum_53',['IMB_ARCH_NUM',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fsse_54',['IMB_ARCH_SSE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16',1,'intel-ipsec-mb.h']]], + ['imb_5fassert_55',['IMB_ASSERT',['../intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fccm_56',['IMB_AUTH_AES_CCM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_57',['IMB_AUTH_AES_CMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5f256_58',['IMB_AUTH_AES_CMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5fbitlen_59',['IMB_AUTH_AES_CMAC_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_60',['IMB_AUTH_AES_GMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f128_61',['IMB_AUTH_AES_GMAC_128',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f192_62',['IMB_AUTH_AES_GMAC_192',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f256_63',['IMB_AUTH_AES_GMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fxcbc_64',['IMB_AUTH_AES_XCBC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_65',['IMB_AUTH_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_5fsgl_66',['IMB_AUTH_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc10_5fiuup_5fdata_67',['IMB_AUTH_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc11_5ffp_5fheader_68',['IMB_AUTH_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5ffp_5fdata_69',['IMB_AUTH_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5fx25_70',['IMB_AUTH_CRC16_X25',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fa_71',['IMB_AUTH_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fb_72',['IMB_AUTH_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fethernet_5ffcs_73',['IMB_AUTH_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fsctp_74',['IMB_AUTH_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fwimax_5fofdma_5fdata_75',['IMB_AUTH_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc6_5fiuup_5fheader_76',['IMB_AUTH_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc7_5ffp_5fheader_77',['IMB_AUTH_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc8_5fwimax_5fofdma_5fhcs_78',['IMB_AUTH_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcustom_79',['IMB_AUTH_CUSTOM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fdocsis_5fcrc32_80',['IMB_AUTH_DOCSIS_CRC32',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fgcm_5fsgl_81',['IMB_AUTH_GCM_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fghash_82',['IMB_AUTH_GHASH',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f1_83',['IMB_AUTH_HMAC_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f224_84',['IMB_AUTH_HMAC_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f256_85',['IMB_AUTH_HMAC_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f384_86',['IMB_AUTH_HMAC_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f512_87',['IMB_AUTH_HMAC_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fkasumi_5fuia1_88',['IMB_AUTH_KASUMI_UIA1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fmd5_89',['IMB_AUTH_MD5',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnull_90',['IMB_AUTH_NULL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnum_91',['IMB_AUTH_NUM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpoly1305_92',['IMB_AUTH_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpon_5fcrc_5fbip_93',['IMB_AUTH_PON_CRC_BIP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f1_94',['IMB_AUTH_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f224_95',['IMB_AUTH_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f256_96',['IMB_AUTH_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f384_97',['IMB_AUTH_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f512_98',['IMB_AUTH_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow3g_5fuia2_5fbitlen_99',['IMB_AUTH_SNOW3G_UIA2_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow_5fv_5faead_100',['IMB_AUTH_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc256_5feia3_5fbitlen_101',['IMB_AUTH_ZUC256_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc_5feia3_5fbitlen_102',['IMB_AUTH_ZUC_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5ffinalize_103',['IMB_CHACHA20_POLY1305_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5fupdate_104',['IMB_CHACHA20_POLY1305_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5ffinalize_105',['IMB_CHACHA20_POLY1305_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5fupdate_106',['IMB_CHACHA20_POLY1305_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5finit_107',['IMB_CHACHA20_POLY1305_INIT',['../intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643',1,'intel-ipsec-mb.h']]], + ['imb_5fchain_5forder_108',['IMB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbc_109',['IMB_CIPHER_CBC',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbcs_5f1_5f9_110',['IMB_CIPHER_CBCS_1_9',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fccm_111',['IMB_CIPHER_CCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_112',['IMB_CIPHER_CHACHA20',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_113',['IMB_CIPHER_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_5fsgl_114',['IMB_CIPHER_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_115',['IMB_CIPHER_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_5fbitlen_116',['IMB_CIPHER_CNTR_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcustom_117',['IMB_CIPHER_CUSTOM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes_118',['IMB_CIPHER_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes3_119',['IMB_CIPHER_DES3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdirection_120',['IMB_CIPHER_DIRECTION',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9ee',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fdes_121',['IMB_CIPHER_DOCSIS_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fsec_5fbpi_122',['IMB_CIPHER_DOCSIS_SEC_BPI',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fecb_123',['IMB_CIPHER_ECB',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_124',['IMB_CIPHER_GCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_5fsgl_125',['IMB_CIPHER_GCM_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fkasumi_5fuea1_5fbitlen_126',['IMB_CIPHER_KASUMI_UEA1_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fmode_127',['IMB_CIPHER_MODE',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334c',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnull_128',['IMB_CIPHER_NULL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnum_129',['IMB_CIPHER_NUM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fpon_5faes_5fcntr_130',['IMB_CIPHER_PON_AES_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow3g_5fuea2_5fbitlen_131',['IMB_CIPHER_SNOW3G_UEA2_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_132',['IMB_CIPHER_SNOW_V',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_5faead_133',['IMB_CIPHER_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fzuc_5feea3_134',['IMB_CIPHER_ZUC_EEA3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc',1,'intel-ipsec-mb.h']]], + ['imb_5fclear_5fmem_135',['imb_clear_mem',['../intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_136',['IMB_CPUFLAGS_AVX',['../intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_137',['IMB_CPUFLAGS_AVX2',['../intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_138',['IMB_CPUFLAGS_AVX512',['../intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fno_5faesni_139',['IMB_CPUFLAGS_NO_AESNI',['../intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_140',['IMB_CPUFLAGS_SSE',['../intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc10_5fiuup_5fdata_141',['IMB_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc11_5ffp_5fheader_142',['IMB_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5ffp_5fdata_143',['IMB_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5fx25_144',['IMB_CRC16_X25',['../intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fa_145',['IMB_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fb_146',['IMB_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fethernet_5ffcs_147',['IMB_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fsctp_148',['IMB_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fwimax_5fofdma_5fdata_149',['IMB_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc6_5fiuup_5fheader_150',['IMB_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc7_5ffp_5fheader_151',['IMB_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc8_5fwimax_5fofdma_5fhcs_152',['IMB_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fblock_5fsize_153',['IMB_DES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkey_5fsched_5fsize_154',['IMB_DES_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkeysched_155',['IMB_DES_KEYSCHED',['../intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6',1,'intel-ipsec-mb.h']]], + ['imb_5fdim_156',['IMB_DIM',['../intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fdecrypt_157',['IMB_DIR_DECRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fencrypt_158',['IMB_DIR_ENCRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5fexport_159',['IMB_DLL_EXPORT',['../intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5flocal_160',['IMB_DLL_LOCAL',['../intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5fmin_5feth_5fpdu_5fsize_161',['IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE',['../intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5ftag_5fsize_162',['IMB_DOCSIS_CRC32_TAG_SIZE',['../intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_163',['IMB_ERR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0b',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5faad_5flen_164',['IMB_ERR_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5flen_165',['IMB_ERR_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5ftag_5flen_166',['IMB_ERR_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5flen_167',['IMB_ERR_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5fmode_168',['IMB_ERR_CIPH_MODE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fhash_5falgo_169',['IMB_ERR_HASH_ALGO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fiv_5flen_170',['IMB_ERR_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5faad_5flen_171',['IMB_ERR_JOB_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5flen_172',['IMB_ERR_JOB_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5ftag_5flen_173',['IMB_ERR_JOB_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fchain_5forder_174',['IMB_ERR_JOB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5fdir_175',['IMB_ERR_JOB_CIPH_DIR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5flen_176',['IMB_ERR_JOB_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fiv_5flen_177',['IMB_ERR_JOB_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fkey_5flen_178',['IMB_ERR_JOB_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5faad_179',['IMB_ERR_JOB_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_180',['IMB_ERR_JOB_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_5fkey_181',['IMB_ERR_JOB_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fdst_182',['IMB_ERR_JOB_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fghash_5finit_5ftag_183',['IMB_ERR_JOB_NULL_GHASH_INIT_TAG',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fipad_184',['IMB_ERR_JOB_NULL_HMAC_IPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fopad_185',['IMB_ERR_JOB_NULL_HMAC_OPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fiv_186',['IMB_ERR_JOB_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fkey_187',['IMB_ERR_JOB_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fnext_5fiv_188',['IMB_ERR_JOB_NULL_NEXT_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsgl_5fctx_189',['IMB_ERR_JOB_NULL_SGL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsrc_190',['IMB_ERR_JOB_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk1_5fexp_191',['IMB_ERR_JOB_NULL_XCBC_K1_EXP',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk2_192',['IMB_ERR_JOB_NULL_XCBC_K2',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk3_193',['IMB_ERR_JOB_NULL_XCBC_K3',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fpon_5fpli_194',['IMB_ERR_JOB_PON_PLI',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fsrc_5foffset_195',['IMB_ERR_JOB_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fkey_5flen_196',['IMB_ERR_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmax_197',['IMB_ERR_MAX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmin_198',['IMB_ERR_MIN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmissing_5fcpuflags_5finit_5fmgr_199',['IMB_ERR_MISSING_CPUFLAGS_INIT_MGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fno_5faesni_5femu_200',['IMB_ERR_NO_AESNI_EMU',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5faad_201',['IMB_ERR_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_202',['IMB_ERR_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_5fkey_203',['IMB_ERR_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fctx_204',['IMB_ERR_NULL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fdst_205',['IMB_ERR_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fexp_5fkey_206',['IMB_ERR_NULL_EXP_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fiv_207',['IMB_ERR_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fjob_208',['IMB_ERR_NULL_JOB',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fkey_209',['IMB_ERR_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fmbmgr_210',['IMB_ERR_NULL_MBMGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fsrc_211',['IMB_ERR_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fsrc_5foffset_212',['IMB_ERR_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09',1,'intel-ipsec-mb.h']]], + ['imb_5ferrno_213',['imb_errno',['../structIMB__MGR.html#a0ab7a3dfdfd8442f591699e2310df11f',1,'IMB_MGR']]], + ['imb_5ffeature_5faesni_214',['IMB_FEATURE_AESNI',['../intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_5femu_215',['IMB_FEATURE_AESNI_EMU',['../intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_216',['IMB_FEATURE_AVX',['../intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx2_217',['IMB_FEATURE_AVX2',['../intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fifma_218',['IMB_FEATURE_AVX512_IFMA',['../intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fskx_219',['IMB_FEATURE_AVX512_SKX',['../intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512bw_220',['IMB_FEATURE_AVX512BW',['../intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512cd_221',['IMB_FEATURE_AVX512CD',['../intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512dq_222',['IMB_FEATURE_AVX512DQ',['../intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512f_223',['IMB_FEATURE_AVX512F',['../intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512vl_224',['IMB_FEATURE_AVX512VL',['../intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fbmi2_225',['IMB_FEATURE_BMI2',['../intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fcmov_226',['IMB_FEATURE_CMOV',['../intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fgfni_227',['IMB_FEATURE_GFNI',['../intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fpclmulqdq_228',['IMB_FEATURE_PCLMULQDQ',['../intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fdata_229',['IMB_FEATURE_SAFE_DATA',['../intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fparam_230',['IMB_FEATURE_SAFE_PARAM',['../intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fshani_231',['IMB_FEATURE_SHANI',['../intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsse4_5f2_232',['IMB_FEATURE_SSE4_2',['../intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvaes_233',['IMB_FEATURE_VAES',['../intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvpclmulqdq_234',['IMB_FEATURE_VPCLMULQDQ',['../intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5faesni_5foff_235',['IMB_FLAG_AESNI_OFF',['../intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fshani_5foff_236',['IMB_FLAG_SHANI_OFF',['../intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fjob_237',['IMB_FLUSH_JOB',['../intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f128_5fkey_5flen_238',['IMB_GCM_128_KEY_LEN',['../intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f192_5fkey_5flen_239',['IMB_GCM_192_KEY_LEN',['../intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f256_5fkey_5flen_240',['IMB_GCM_256_KEY_LEN',['../intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fblock_5flen_241',['IMB_GCM_BLOCK_LEN',['../intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fenc_5fkey_5flen_242',['IMB_GCM_ENC_KEY_LEN',['../intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fiv_5fdata_5flen_243',['IMB_GCM_IV_DATA_LEN',['../intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fkey_5fsets_244',['IMB_GCM_KEY_SETS',['../intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fcompleted_5fjob_245',['IMB_GET_COMPLETED_JOB',['../intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ferrno_246',['imb_get_errno',['../intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ffeature_5fflags_247',['imb_get_feature_flags',['../intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fmb_5fmgr_5fsize_248',['imb_get_mb_mgr_size',['../intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fjob_249',['IMB_GET_NEXT_JOB',['../intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fstrerror_250',['imb_get_strerror',['../intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_251',['imb_get_version',['../intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5fstr_252',['imb_get_version_str',['../intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_253',['IMB_GHASH',['../intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_5fpre_254',['IMB_GHASH_PRE',['../intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2',1,'intel-ipsec-mb.h']]], + ['imb_5fhash_5falg_255',['IMB_HASH_ALG',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f32_256',['IMB_HEC_32',['../intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f64_257',['IMB_HEC_64',['../intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4',1,'intel-ipsec-mb.h']]], + ['imb_5fjob_258',['IMB_JOB',['../intel-ipsec-mb_8h.html#aed2387ba36d72c3d148fcb9d2802ccf5',1,'IMB_JOB(): intel-ipsec-mb.h'],['../structIMB__JOB.html',1,'IMB_JOB']]], + ['imb_5fkasumi_5fblock_5fsize_259',['IMB_KASUMI_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fdigest_5fsize_260',['IMB_KASUMI_DIGEST_SIZE',['../intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_261',['IMB_KASUMI_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_5fbit_262',['IMB_KASUMI_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f2_5fbuffer_263',['IMB_KASUMI_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f3_5fbuffer_264',['IMB_KASUMI_F8_3_BUFFER',['../intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f4_5fbuffer_265',['IMB_KASUMI_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5fn_5fbuffer_266',['IMB_KASUMI_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_267',['IMB_KASUMI_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_5fuser_268',['IMB_KASUMI_F9_1_BUFFER_USER',['../intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff8_5fkey_5fsched_269',['IMB_KASUMI_INIT_F8_KEY_SCHED',['../intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff9_5fkey_5fsched_270',['IMB_KASUMI_INIT_F9_KEY_SCHED',['../intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fiv_5fsize_271',['IMB_KASUMI_IV_SIZE',['../intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsched_5fsize_272',['IMB_KASUMI_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsize_273',['IMB_KASUMI_KEY_SIZE',['../intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f128_5fbytes_274',['IMB_KEY_128_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f192_5fbytes_275',['IMB_KEY_192_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f256_5fbytes_276',['IMB_KEY_256_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5fsize_5fbytes_277',['IMB_KEY_SIZE_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cc',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fjobs_278',['IMB_MAX_JOBS',['../intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5ftag_5flen_279',['IMB_MAX_TAG_LEN',['../intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fone_5fblock_280',['IMB_MD5_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2',1,'intel-ipsec-mb.h']]], + ['imb_5fmgr_281',['IMB_MGR',['../structIMB__MGR.html',1,'IMB_MGR'],['../intel-ipsec-mb_8h.html#a6d466496b4adfea3f9bc0881f11fe551',1,'IMB_MGR(): intel-ipsec-mb.h']]], + ['imb_5forder_5fcipher_5fhash_282',['IMB_ORDER_CIPHER_HASH',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fhash_5fcipher_283',['IMB_ORDER_HASH_CIPHER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d',1,'intel-ipsec-mb.h']]], + ['imb_5fqueue_5fsize_284',['IMB_QUEUE_SIZE',['../intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fpointers_5fmb_5fmgr_285',['imb_set_pointers_mb_mgr',['../intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fall_286',['IMB_SGL_ALL',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fcomplete_287',['IMB_SGL_COMPLETE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5finit_288',['IMB_SGL_INIT',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fiov_289',['IMB_SGL_IOV',['../structIMB__SGL__IOV.html',1,'']]], + ['imb_5fsgl_5fstate_290',['IMB_SGL_STATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3d',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fupdate_291',['IMB_SGL_UPDATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_292',['IMB_SHA1',['../intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fblock_5fsize_293',['IMB_SHA1_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fdigest_5fsize_5fin_5fbytes_294',['IMB_SHA1_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fone_5fblock_295',['IMB_SHA1_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_296',['IMB_SHA224',['../intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fdigest_5fsize_5fin_5fbytes_297',['IMB_SHA224_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fone_5fblock_298',['IMB_SHA224_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_299',['IMB_SHA256',['../intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fdigest_5fsize_5fin_5fbytes_300',['IMB_SHA256_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fone_5fblock_301',['IMB_SHA256_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_302',['IMB_SHA384',['../intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fdigest_5fsize_5fin_5fbytes_303',['IMB_SHA384_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fone_5fblock_304',['IMB_SHA384_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_305',['IMB_SHA512',['../intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fdigest_5fsize_5fin_5fbytes_306',['IMB_SHA512_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fone_5fblock_307',['IMB_SHA512_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f256_5fblock_5fsize_308',['IMB_SHA_256_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f384_5fblock_5fsize_309',['IMB_SHA_384_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f512_5fblock_5fsize_310',['IMB_SHA_512_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_311',['IMB_SNOW3G_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_5fbit_312',['IMB_SNOW3G_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f2_5fbuffer_313',['IMB_SNOW3G_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f4_5fbuffer_314',['IMB_SNOW3G_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_315',['IMB_SNOW3G_F8_8_BUFFER',['../intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_5fmultikey_316',['IMB_SNOW3G_F8_8_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_317',['IMB_SNOW3G_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_5fmultikey_318',['IMB_SNOW3G_F8_N_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff9_5f1_5fbuffer_319',['IMB_SNOW3G_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5finit_5fkey_5fsched_320',['IMB_SNOW3G_INIT_KEY_SCHED',['../intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5fkey_5fsched_5fsize_321',['IMB_SNOW3G_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_322',['IMB_STATUS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066ab',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fbeing_5fprocessed_323',['IMB_STATUS_BEING_PROCESSED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_324',['IMB_STATUS_COMPLETED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fauth_325',['IMB_STATUS_COMPLETED_AUTH',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fcipher_326',['IMB_STATUS_COMPLETED_CIPHER',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5ferror_327',['IMB_STATUS_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finternal_5ferror_328',['IMB_STATUS_INTERNAL_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finvalid_5fargs_329',['IMB_STATUS_INVALID_ARGS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_330',['IMB_SUBMIT_BURST',['../intel-ipsec-mb_8h.html#ad95138f7b201a9d7f8e57a155af720f4',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_5fnocheck_331',['IMB_SUBMIT_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#ab043c084d20c19d9952dee899b693085',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_332',['IMB_SUBMIT_CIPHER_BURST',['../intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_5fnocheck_333',['IMB_SUBMIT_CIPHER_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_334',['IMB_SUBMIT_HASH_BURST',['../intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_5fnocheck_335',['IMB_SUBMIT_HASH_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_336',['IMB_SUBMIT_JOB',['../intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_5fnocheck_337',['IMB_SUBMIT_JOB_NOCHECK',['../intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d',1,'intel-ipsec-mb.h']]], + ['imb_5fuint128_5ft_338',['imb_uint128_t',['../structimb__uint128__t.html',1,'']]], + ['imb_5fversion_339',['IMB_VERSION',['../intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fnum_340',['IMB_VERSION_NUM',['../intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fstr_341',['IMB_VERSION_STR',['../intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f1_5fbuffer_342',['IMB_ZUC_EEA3_1_BUFFER',['../intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f4_5fbuffer_343',['IMB_ZUC_EEA3_4_BUFFER',['../intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5fn_5fbuffer_344',['IMB_ZUC_EEA3_N_BUFFER',['../intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5f1_5fbuffer_345',['IMB_ZUC_EIA3_1_BUFFER',['../intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5fn_5fbuffer_346',['IMB_ZUC_EIA3_N_BUFFER',['../intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c',1,'intel-ipsec-mb.h']]], + ['in_347',['in',['../structIMB__SGL__IOV.html#a56c5b0a033d949869cee2d03f1df3129',1,'IMB_SGL_IOV']]], + ['in_5flength_348',['in_length',['../structgcm__context__data.html#a0d9e72f85aba9363fcb4f2d757342aa2',1,'gcm_context_data']]], + ['init_5fmb_5fmgr_5fauto_349',['init_mb_mgr_auto',['../intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx_350',['init_mb_mgr_avx',['../intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx2_351',['init_mb_mgr_avx2',['../intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx512_352',['init_mb_mgr_avx512',['../intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fsse_353',['init_mb_mgr_sse',['../intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5ft_354',['init_mb_mgr_t',['../intel-ipsec-mb_8h.html#af0c38ca0a5aac81d8db7e0cc2811426b',1,'intel-ipsec-mb.h']]], + ['intel_2dipsec_2dmb_355',['intel-ipsec-mb',['../index.html',1,'']]], + ['intel_2dipsec_2dmb_2eh_356',['intel-ipsec-mb.h',['../intel-ipsec-mb_8h.html',1,'']]], + ['iv_357',['IV',['../structchacha20__poly1305__context__data.html#ac0d1657fc57308900691369362c56658',1,'chacha20_poly1305_context_data']]], + ['iv_358',['iv',['../structIMB__JOB.html#a960fdd62c1c10d2d3cc0a765c0a6531d',1,'IMB_JOB']]], + ['iv_5flen_5fin_5fbytes_359',['iv_len_in_bytes',['../structIMB__JOB.html#a25b4ce8aec20f64cfc393b245d62a09e',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_9.html b/docs/search/all_9.html new file mode 100644 index 00000000..ac197d32 --- /dev/null +++ b/docs/search/all_9.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_9.js b/docs/search/all_9.js new file mode 100644 index 00000000..f0cb16fe --- /dev/null +++ b/docs/search/all_9.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['jobs_0',['jobs',['../structIMB__MGR.html#ab342224dce812fb13a9e935e7a7b0617',1,'IMB_MGR']]] +]; diff --git a/docs/search/all_a.html b/docs/search/all_a.html new file mode 100644 index 00000000..9f6e47a3 --- /dev/null +++ b/docs/search/all_a.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_a.js b/docs/search/all_a.js new file mode 100644 index 00000000..4e6bfd2e --- /dev/null +++ b/docs/search/all_a.js @@ -0,0 +1,29 @@ +var searchData= +[ + ['k_0',['k',['../structsnow3g__key__schedule__s.html#ae0ca8cd2581836c0297b34eea1effb12',1,'snow3g_key_schedule_s']]], + ['kasumi_5ff8_5f1_5fbuffer_5fbit_5ft_1',['kasumi_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a81afa744fb5669fe37cf0dce2193d8a7',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f1_5fbuffer_5ft_2',['kasumi_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a89fe975f0388b71765be494d04b034f4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f2_5fbuffer_5ft_3',['kasumi_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#af1eb074f5da469fa655d1404be8701dd',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f3_5fbuffer_5ft_4',['kasumi_f8_3_buffer_t',['../intel-ipsec-mb_8h.html#a53ecac7e009fcbeee603f975a45d3012',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f4_5fbuffer_5ft_5',['kasumi_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#accf08a616e22a2c56e6a37d56f38b354',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5fiv_5fgen_6',['kasumi_f8_iv_gen',['../intel-ipsec-mb_8h.html#a3876dca52c5c210dd0827b2a472e6d0b',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5fn_5fbuffer_5ft_7',['kasumi_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a51bbde186cdd437b6f9448e19a58e8a1',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5ft_8',['kasumi_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#a8b02114f003b5fe0e1ccbfdc13ac55e4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5fuser_5ft_9',['kasumi_f9_1_buffer_user_t',['../intel-ipsec-mb_8h.html#ac17147d04602964bea59322ff31ebf59',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5fiv_5fgen_10',['kasumi_f9_iv_gen',['../intel-ipsec-mb_8h.html#ac886170fc4e70b467383fdf09f0724cf',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_11',['kasumi_init_f8_key_sched',['../structIMB__MGR.html#ab32d67f27cb6f06980080e09db133b24',1,'IMB_MGR']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_5ft_12',['kasumi_init_f8_key_sched_t',['../intel-ipsec-mb_8h.html#a7bec62666456d8962676961cf187ac80',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_13',['kasumi_init_f9_key_sched',['../structIMB__MGR.html#a58ad87104b08ba5b56fc65928677a639',1,'IMB_MGR']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_5ft_14',['kasumi_init_f9_key_sched_t',['../intel-ipsec-mb_8h.html#a8b30db8462aca1c039c3fd20646f342c',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5fs_15',['kasumi_key_sched_s',['../structkasumi__key__sched__s.html',1,'']]], + ['kasumi_5fkey_5fsched_5fsize_16',['kasumi_key_sched_size',['../structIMB__MGR.html#abd0c855a3e43df48f5f21686a095f358',1,'IMB_MGR']]], + ['kasumi_5fkey_5fsched_5fsize_5ft_17',['kasumi_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad7444c36024a63025e7682c16946a00a',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5ft_18',['kasumi_key_sched_t',['../intel-ipsec-mb_8h.html#a82d6866ebcf1bd46b684fe473b4b73f2',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fschedule_5fsize_19',['KASUMI_KEY_SCHEDULE_SIZE',['../intel-ipsec-mb_8h.html#a071e04977525b67c8d1accc60937eb67',1,'intel-ipsec-mb.h']]], + ['kasumi_5fuia1_20',['KASUMI_UIA1',['../structIMB__JOB.html#a11f452fba9d5cad2b2cdb929b7f244e9',1,'IMB_JOB']]], + ['key_5flen_5fin_5fbytes_21',['key_len_in_bytes',['../structIMB__JOB.html#abdd8a58a37e8d2daa1c2d333b826a65c',1,'IMB_JOB']]], + ['keyexp_5f128_22',['keyexp_128',['../structIMB__MGR.html#a7c97c5361b90abe325db55530b6f198e',1,'IMB_MGR']]], + ['keyexp_5f192_23',['keyexp_192',['../structIMB__MGR.html#a2bd7444a318de06b5f53178abf9c37ff',1,'IMB_MGR']]], + ['keyexp_5f256_24',['keyexp_256',['../structIMB__MGR.html#a431e99ddb3837a32bedb6cd426054bcc',1,'IMB_MGR']]], + ['keyexp_5ft_25',['keyexp_t',['../intel-ipsec-mb_8h.html#a60de89a391c8d6d9c51c742a87effb6f',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/all_b.html b/docs/search/all_b.html new file mode 100644 index 00000000..552a1499 --- /dev/null +++ b/docs/search/all_b.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_b.js b/docs/search/all_b.js new file mode 100644 index 00000000..85f6540f --- /dev/null +++ b/docs/search/all_b.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['last_5fblock_5fcount_0',['last_block_count',['../structchacha20__poly1305__context__data.html#add03511d1f95376b3ac64a7916c27368',1,'chacha20_poly1305_context_data']]], + ['last_5fks_1',['last_ks',['../structchacha20__poly1305__context__data.html#afc74af4af99449f0b9f01338ae723c1a',1,'chacha20_poly1305_context_data']]], + ['len_2',['len',['../structIMB__SGL__IOV.html#a3a3662456b91011eb0585667492acc88',1,'IMB_SGL_IOV']]], + ['low_3',['low',['../structimb__uint128__t.html#a30e60cc86acb6eafaa4079c0e46ba622',1,'imb_uint128_t']]] +]; diff --git a/docs/search/all_c.html b/docs/search/all_c.html new file mode 100644 index 00000000..e15651c4 --- /dev/null +++ b/docs/search/all_c.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_c.js b/docs/search/all_c.js new file mode 100644 index 00000000..478ff214 --- /dev/null +++ b/docs/search/all_c.js @@ -0,0 +1,13 @@ +var searchData= +[ + ['md5_5fone_5fblock_0',['md5_one_block',['../structIMB__MGR.html#a54b5f7b625ce226ebd46a2a19e9bc724',1,'IMB_MGR']]], + ['md5_5fone_5fblock_5favx_1',['md5_one_block_avx',['../intel-ipsec-mb_8h.html#a7addeb4d0c6ddcdaa031b9e7b4dd35c2',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx2_2',['md5_one_block_avx2',['../intel-ipsec-mb_8h.html#a8b912812c65f4d3b592875cdd233704d',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx512_3',['md5_one_block_avx512',['../intel-ipsec-mb_8h.html#ac7f4da2fc0601ad50851e7e43f53e0cf',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5fsse_4',['md5_one_block_sse',['../intel-ipsec-mb_8h.html#a9d883a3a629240fe86efedd180294433',1,'intel-ipsec-mb.h']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbits_5',['msg_len_to_cipher_in_bits',['../structIMB__JOB.html#a425d57ff565bb2ff76e2b0351b00c751',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbytes_6',['msg_len_to_cipher_in_bytes',['../structIMB__JOB.html#a9c2e008fdb76df46a33300f692f92265',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbits_7',['msg_len_to_hash_in_bits',['../structIMB__JOB.html#ae45d78dcef87042ffa1daf951f715d42',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbytes_8',['msg_len_to_hash_in_bytes',['../structIMB__JOB.html#a40caf02dcd7e45dd91cfae33159ca26e',1,'IMB_JOB']]], + ['msk16_9',['msk16',['../structkasumi__key__sched__s.html#a091988e7975015d654ccb6904c4fdf39',1,'kasumi_key_sched_s']]] +]; diff --git a/docs/search/all_d.html b/docs/search/all_d.html new file mode 100644 index 00000000..4c7462d6 --- /dev/null +++ b/docs/search/all_d.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_d.js b/docs/search/all_d.js new file mode 100644 index 00000000..0c972f70 --- /dev/null +++ b/docs/search/all_d.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['next_5fiv_0',['next_iv',['../structIMB__JOB.html#a5d473d9cb01f8e2818a6f5f0c9483bd7',1,'IMB_JOB']]], + ['next_5fjob_1',['next_job',['../structIMB__MGR.html#a106ebd60d876e56bfc6bcc03d20acce4',1,'IMB_MGR']]], + ['num_5fsgl_5fio_5fsegs_2',['num_sgl_io_segs',['../structIMB__JOB.html#a1d0a6baa8d8f7d9714fe86fe7085b8bb',1,'IMB_JOB']]] +]; diff --git a/docs/search/all_e.html b/docs/search/all_e.html new file mode 100644 index 00000000..26f19e81 --- /dev/null +++ b/docs/search/all_e.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_e.js b/docs/search/all_e.js new file mode 100644 index 00000000..3f451b03 --- /dev/null +++ b/docs/search/all_e.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['orig_5fiv_0',['orig_IV',['../structgcm__context__data.html#a84bac4eabc12b81d26898b6fe6728a63',1,'gcm_context_data']]], + ['out_1',['out',['../structIMB__SGL__IOV.html#a54b8da76fb8a400f9f7972bdd97f6f53',1,'IMB_SGL_IOV']]] +]; diff --git a/docs/search/all_f.html b/docs/search/all_f.html new file mode 100644 index 00000000..8caf8845 --- /dev/null +++ b/docs/search/all_f.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/all_f.js b/docs/search/all_f.js new file mode 100644 index 00000000..b274ce7e --- /dev/null +++ b/docs/search/all_f.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['partial_5fblock_5fenc_5fkey_0',['partial_block_enc_key',['../structgcm__context__data.html#a2e52ea863c681afe1d3370a987e252f0',1,'gcm_context_data']]], + ['partial_5fblock_5flength_1',['partial_block_length',['../structgcm__context__data.html#a97411247efd14ab01e7e0da6587bfa0c',1,'gcm_context_data']]], + ['poly1305_2',['POLY1305',['../structIMB__JOB.html#a15f9c1a5ab25dca6e3a550ff00ad2419',1,'IMB_JOB']]], + ['poly_5fkey_3',['poly_key',['../structchacha20__poly1305__context__data.html#a9ac4fafe2ca37395264f728c8e99a9b1',1,'chacha20_poly1305_context_data']]], + ['poly_5fscratch_4',['poly_scratch',['../structchacha20__poly1305__context__data.html#a908d80ce2c35792099dbc967818b9abd',1,'chacha20_poly1305_context_data']]] +]; diff --git a/docs/search/classes_0.html b/docs/search/classes_0.html new file mode 100644 index 00000000..e50ba583 --- /dev/null +++ b/docs/search/classes_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/classes_0.js b/docs/search/classes_0.js new file mode 100644 index 00000000..0cba4879 --- /dev/null +++ b/docs/search/classes_0.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['chacha20_5fpoly1305_5fcontext_5fdata_0',['chacha20_poly1305_context_data',['../structchacha20__poly1305__context__data.html',1,'']]] +]; diff --git a/docs/search/classes_1.html b/docs/search/classes_1.html new file mode 100644 index 00000000..4d53ed21 --- /dev/null +++ b/docs/search/classes_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/classes_1.js b/docs/search/classes_1.js new file mode 100644 index 00000000..eabce0ec --- /dev/null +++ b/docs/search/classes_1.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['gcm_5fcontext_5fdata_0',['gcm_context_data',['../structgcm__context__data.html',1,'']]], + ['gcm_5fkey_5fdata_1',['gcm_key_data',['../structgcm__key__data.html',1,'']]] +]; diff --git a/docs/search/classes_2.html b/docs/search/classes_2.html new file mode 100644 index 00000000..65e37e27 --- /dev/null +++ b/docs/search/classes_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/classes_2.js b/docs/search/classes_2.js new file mode 100644 index 00000000..6a6b26a0 --- /dev/null +++ b/docs/search/classes_2.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['imb_5fjob_0',['IMB_JOB',['../structIMB__JOB.html',1,'']]], + ['imb_5fmgr_1',['IMB_MGR',['../structIMB__MGR.html',1,'']]], + ['imb_5fsgl_5fiov_2',['IMB_SGL_IOV',['../structIMB__SGL__IOV.html',1,'']]], + ['imb_5fuint128_5ft_3',['imb_uint128_t',['../structimb__uint128__t.html',1,'']]] +]; diff --git a/docs/search/classes_3.html b/docs/search/classes_3.html new file mode 100644 index 00000000..17322b2c --- /dev/null +++ b/docs/search/classes_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/classes_3.js b/docs/search/classes_3.js new file mode 100644 index 00000000..d493f075 --- /dev/null +++ b/docs/search/classes_3.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['kasumi_5fkey_5fsched_5fs_0',['kasumi_key_sched_s',['../structkasumi__key__sched__s.html',1,'']]] +]; diff --git a/docs/search/classes_4.html b/docs/search/classes_4.html new file mode 100644 index 00000000..58da4630 --- /dev/null +++ b/docs/search/classes_4.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/classes_4.js b/docs/search/classes_4.js new file mode 100644 index 00000000..dd9067d2 --- /dev/null +++ b/docs/search/classes_4.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['snow3g_5fkey_5fschedule_5fs_0',['snow3g_key_schedule_s',['../structsnow3g__key__schedule__s.html',1,'']]] +]; diff --git a/docs/search/close.png b/docs/search/close.png new file mode 100644 index 0000000000000000000000000000000000000000..9342d3dfeea7b7c4ee610987e717804b5a42ceb9 GIT binary patch literal 273 zcmV+s0q*{ZP)4(RlMby96)VwnbG{ zbe&}^BDn7x>$<{ck4zAK-=nT;=hHG)kmplIF${xqm8db3oX6wT3bvp`TE@m0cg;b) zBuSL}5?N7O(iZLdAlz@)b)Rd~DnSsSX&P5qC`XwuFwcAYLC+d2>+1(8on;wpt8QIC X2MT$R4iQDd00000NkvXXu0mjfia~GN literal 0 HcmV?d00001 diff --git a/docs/search/close.svg b/docs/search/close.svg new file mode 100644 index 00000000..a933eea1 --- /dev/null +++ b/docs/search/close.svg @@ -0,0 +1,31 @@ + + + + + + image/svg+xml + + + + + + + + diff --git a/docs/search/defines_0.html b/docs/search/defines_0.html new file mode 100644 index 00000000..b42218f2 --- /dev/null +++ b/docs/search/defines_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/defines_0.js b/docs/search/defines_0.js new file mode 100644 index 00000000..f0aeaf9d --- /dev/null +++ b/docs/search/defines_0.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['_5f_5fforceinline_0',['__forceinline',['../intel-ipsec-mb_8h.html#af93b819ac40799ac392e16f6a90729fd',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/defines_1.html b/docs/search/defines_1.html new file mode 100644 index 00000000..64a33798 --- /dev/null +++ b/docs/search/defines_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/defines_1.js b/docs/search/defines_1.js new file mode 100644 index 00000000..1c8507ed --- /dev/null +++ b/docs/search/defines_1.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['declare_5faligned_0',['DECLARE_ALIGNED',['../intel-ipsec-mb_8h.html#ad920a47ed8f57af8b2cf6bcd18a3e41c',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/defines_2.html b/docs/search/defines_2.html new file mode 100644 index 00000000..f183653c --- /dev/null +++ b/docs/search/defines_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/defines_2.js b/docs/search/defines_2.js new file mode 100644 index 00000000..6a7fdb33 --- /dev/null +++ b/docs/search/defines_2.js @@ -0,0 +1,183 @@ +var searchData= +[ + ['imb_5faes128_5fcfb_5fone_0',['IMB_AES128_CFB_ONE',['../intel-ipsec-mb_8h.html#a1e122b460a714f2981f94bf7f4bba547',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_1',['IMB_AES128_GCM_DEC',['../intel-ipsec-mb_8h.html#a2b026b2a901c76a1512eb9c8c80f9709',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5ffinalize_2',['IMB_AES128_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a280b9231c76c525a62d0ef819abd2133',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fdec_5fupdate_3',['IMB_AES128_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a2f6014cfaaa0ee5f454e2771ff6b5829',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_4',['IMB_AES128_GCM_ENC',['../intel-ipsec-mb_8h.html#adb65fc01b63a475cca4cf532ace005b6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5ffinalize_5',['IMB_AES128_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#aaf13d46f5832cf9e0cb50f7fcde020be',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fenc_5fupdate_6',['IMB_AES128_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#ab1f4c52b4145946e80ef698e469d67db',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_7',['IMB_AES128_GCM_INIT',['../intel-ipsec-mb_8h.html#ad0a8ea6b81df94d066b028ca7bf656ef',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5finit_5fvar_5fiv_8',['IMB_AES128_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a04142f11a678c0419dff242a0ccbd2e6',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fpre_9',['IMB_AES128_GCM_PRE',['../intel-ipsec-mb_8h.html#a5a442c1fcd7d427bf78e804da32bdbd1',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgcm_5fprecomp_10',['IMB_AES128_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a1c63dfec645afa671f93f4fa65a6996e',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5ffinalize_11',['IMB_AES128_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#af0bd4f5841d41433339717e6b97c15ad',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5finit_12',['IMB_AES128_GMAC_INIT',['../intel-ipsec-mb_8h.html#af315c30a8cbd3c20735e22f462b47964',1,'intel-ipsec-mb.h']]], + ['imb_5faes128_5fgmac_5fupdate_13',['IMB_AES128_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#af0b2ba1a0f9b099a06a1ba04380b0e2a',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_14',['IMB_AES192_GCM_DEC',['../intel-ipsec-mb_8h.html#aedd2b84ebfbcb56ac5176e8793490047',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5ffinalize_15',['IMB_AES192_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a759bbe1b8494c3157f1a506c003b700b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fdec_5fupdate_16',['IMB_AES192_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#ae25c4f893fc29f09d03a8ece5fddc82f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_17',['IMB_AES192_GCM_ENC',['../intel-ipsec-mb_8h.html#af7548f17077a0b187a4ecfde4e0ff5d9',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5ffinalize_18',['IMB_AES192_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1e36a361be137a5d49b0798f054403bd',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fenc_5fupdate_19',['IMB_AES192_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a456f5bed8eb67603bdebfda230f76957',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_20',['IMB_AES192_GCM_INIT',['../intel-ipsec-mb_8h.html#a8cd402fe539de3fc8ff4062c2804096f',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5finit_5fvar_5fiv_21',['IMB_AES192_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a97791c976660a673405e6fa22ef3d262',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fpre_22',['IMB_AES192_GCM_PRE',['../intel-ipsec-mb_8h.html#a003f16aac382baff7a6b83040e58e471',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgcm_5fprecomp_23',['IMB_AES192_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#ac7b3bcfd3ecbe7b2b8c7da963310b87b',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5ffinalize_24',['IMB_AES192_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#a16a6f89fd4131c406d6ee7a1aeb663eb',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5finit_25',['IMB_AES192_GMAC_INIT',['../intel-ipsec-mb_8h.html#a25ed49507e66a6e98eab4a3d34e75231',1,'intel-ipsec-mb.h']]], + ['imb_5faes192_5fgmac_5fupdate_26',['IMB_AES192_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a113d0f4320e929f89786773f7d11c5c8',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_27',['IMB_AES256_GCM_DEC',['../intel-ipsec-mb_8h.html#a1eb77771626f83eba8482473d29ffbb1',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5ffinalize_28',['IMB_AES256_GCM_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a64d9473bc4f28c0b783baa4a31b9930e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fdec_5fupdate_29',['IMB_AES256_GCM_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a70847fd3bb5d1be79ebfa4e9104edabd',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_30',['IMB_AES256_GCM_ENC',['../intel-ipsec-mb_8h.html#a556333acdc2db1a2540a1cd1af644ca9',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5ffinalize_31',['IMB_AES256_GCM_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a1f49fe7109315eb479ac1c9969e2f877',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fenc_5fupdate_32',['IMB_AES256_GCM_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a223c00c636e2552e7a3dd1a97952cc6a',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_33',['IMB_AES256_GCM_INIT',['../intel-ipsec-mb_8h.html#a8049111340bf2694799eb4e42b0a82c3',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5finit_5fvar_5fiv_34',['IMB_AES256_GCM_INIT_VAR_IV',['../intel-ipsec-mb_8h.html#a61ddc1485781aba1969fe3ca97a22ded',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fpre_35',['IMB_AES256_GCM_PRE',['../intel-ipsec-mb_8h.html#a5771f5169d3f3af3a2419ccd8275a754',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgcm_5fprecomp_36',['IMB_AES256_GCM_PRECOMP',['../intel-ipsec-mb_8h.html#a4fa417a748ca30a961334bf91297dfda',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5ffinalize_37',['IMB_AES256_GMAC_FINALIZE',['../intel-ipsec-mb_8h.html#abfbb7469f7d0240bb3d9a0e5f368b19e',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5finit_38',['IMB_AES256_GMAC_INIT',['../intel-ipsec-mb_8h.html#a3d57940a54e9d94fa14dc008cd959092',1,'intel-ipsec-mb.h']]], + ['imb_5faes256_5fgmac_5fupdate_39',['IMB_AES256_GMAC_UPDATE',['../intel-ipsec-mb_8h.html#a91b8d98c7ffce8179ef953493a959d4b',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fblock_5fsize_40',['IMB_AES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#ab1b5db5ac58714b6f69e754a0c4ca793',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f128_41',['IMB_AES_CMAC_SUBKEY_GEN_128',['../intel-ipsec-mb_8h.html#aa1e9006a8a76ba7fb09b97816d7fdfe5',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fcmac_5fsubkey_5fgen_5f256_42',['IMB_AES_CMAC_SUBKEY_GEN_256',['../intel-ipsec-mb_8h.html#aca7b22139c297858794c7268439e88f0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f128_43',['IMB_AES_KEYEXP_128',['../intel-ipsec-mb_8h.html#afa111339421e794c2643df2ecf86a8c0',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f192_44',['IMB_AES_KEYEXP_192',['../intel-ipsec-mb_8h.html#a574d86849e2c9ca402305eb479ca558d',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fkeyexp_5f256_45',['IMB_AES_KEYEXP_256',['../intel-ipsec-mb_8h.html#a003306124dae4920161ed6d4a7bf4e2f',1,'intel-ipsec-mb.h']]], + ['imb_5faes_5fxcbc_5fkeyexp_46',['IMB_AES_XCBC_KEYEXP',['../intel-ipsec-mb_8h.html#a5a5f2342f2b1e65e41084d915c3b2cda',1,'intel-ipsec-mb.h']]], + ['imb_5fassert_47',['IMB_ASSERT',['../intel-ipsec-mb_8h.html#af0f2f0bfad62f740bc2e22e60f48af8d',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5ffinalize_48',['IMB_CHACHA20_POLY1305_DEC_FINALIZE',['../intel-ipsec-mb_8h.html#a8f6a258b6a13952fad9c217194c716f0',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fdec_5fupdate_49',['IMB_CHACHA20_POLY1305_DEC_UPDATE',['../intel-ipsec-mb_8h.html#a4ac8fd429d9f356003522200811b0bed',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5ffinalize_50',['IMB_CHACHA20_POLY1305_ENC_FINALIZE',['../intel-ipsec-mb_8h.html#a7575d2fafb2b970bd08a49a556602031',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5fenc_5fupdate_51',['IMB_CHACHA20_POLY1305_ENC_UPDATE',['../intel-ipsec-mb_8h.html#a2abd7cd460b99467836188d62272cd2b',1,'intel-ipsec-mb.h']]], + ['imb_5fchacha20_5fpoly1305_5finit_52',['IMB_CHACHA20_POLY1305_INIT',['../intel-ipsec-mb_8h.html#adb8ac3238e6dfe39a10e64602a96e643',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx_53',['IMB_CPUFLAGS_AVX',['../intel-ipsec-mb_8h.html#a0329fede890d95a97423a714410e1576',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx2_54',['IMB_CPUFLAGS_AVX2',['../intel-ipsec-mb_8h.html#ac0a92622a85423541536321231a6cb7d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5favx512_55',['IMB_CPUFLAGS_AVX512',['../intel-ipsec-mb_8h.html#a0a250932ede6ca7b69f5c763c24cf51d',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fno_5faesni_56',['IMB_CPUFLAGS_NO_AESNI',['../intel-ipsec-mb_8h.html#a6afb888ab232bab1c3ca027eb071f388',1,'intel-ipsec-mb.h']]], + ['imb_5fcpuflags_5fsse_57',['IMB_CPUFLAGS_SSE',['../intel-ipsec-mb_8h.html#a5654f563aaad731df22c8e8517c9154c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc10_5fiuup_5fdata_58',['IMB_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#ac7406ab5572b4af93ff8dbbc8ee0c961',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc11_5ffp_5fheader_59',['IMB_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#ae35b6b1b19ef6d599d47591f6c1e1732',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5ffp_5fdata_60',['IMB_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#abe747d2365eb0393ab748c218029e3c6',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc16_5fx25_61',['IMB_CRC16_X25',['../intel-ipsec-mb_8h.html#a78217e8116876c0ecbe82fd3d417cc1c',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fa_62',['IMB_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a6ff60117e0ff260865cd80323306479b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc24_5flte_5fb_63',['IMB_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#ae07e15be0652ca8faa23c6f907d83e03',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fethernet_5ffcs_64',['IMB_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a075757c7f7b132a8f2ae13c18f36e2af',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fsctp_65',['IMB_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a86753ed15fcd8aab177d276dd4ebf169',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc32_5fwimax_5fofdma_5fdata_66',['IMB_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#ab0a9f19127f3417cc78cfdf9091af33e',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc6_5fiuup_5fheader_67',['IMB_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#ac7c45b0788456473390ca178275f9a34',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc7_5ffp_5fheader_68',['IMB_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a23fe73b1f9d40348bc98f92f859df60b',1,'intel-ipsec-mb.h']]], + ['imb_5fcrc8_5fwimax_5fofdma_5fhcs_69',['IMB_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a0edd5fa8f3351ee214f7f2a267949f6d',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fblock_5fsize_70',['IMB_DES_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a95f4ece0b318cb36bdbe66089d9ef649',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkey_5fsched_5fsize_71',['IMB_DES_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a7d156b1b9d715299e08cca76e790e2b7',1,'intel-ipsec-mb.h']]], + ['imb_5fdes_5fkeysched_72',['IMB_DES_KEYSCHED',['../intel-ipsec-mb_8h.html#a6fbb59f06f60f1900b591735ad691ac6',1,'intel-ipsec-mb.h']]], + ['imb_5fdim_73',['IMB_DIM',['../intel-ipsec-mb_8h.html#a1beec59f9694a41c22b6b099263c070d',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5fexport_74',['IMB_DLL_EXPORT',['../intel-ipsec-mb_8h.html#a986643689f56109161597f67eb9d7b40',1,'intel-ipsec-mb.h']]], + ['imb_5fdll_5flocal_75',['IMB_DLL_LOCAL',['../intel-ipsec-mb_8h.html#a479e22216a6be5645af96c4b1433339e',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5fmin_5feth_5fpdu_5fsize_76',['IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE',['../intel-ipsec-mb_8h.html#a74dca2bb8b0c87dec90f3b985c0266af',1,'intel-ipsec-mb.h']]], + ['imb_5fdocsis_5fcrc32_5ftag_5fsize_77',['IMB_DOCSIS_CRC32_TAG_SIZE',['../intel-ipsec-mb_8h.html#a59c620471ea8c9577ed593127f82c758',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_78',['IMB_FEATURE_AESNI',['../intel-ipsec-mb_8h.html#a19c147087d2ae918baa22f90fa9b3b05',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5faesni_5femu_79',['IMB_FEATURE_AESNI_EMU',['../intel-ipsec-mb_8h.html#a421158db6d19c97a6bceb32eb4a839ed',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx_80',['IMB_FEATURE_AVX',['../intel-ipsec-mb_8h.html#a7af992d12b4f9a6e789384969a066056',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx2_81',['IMB_FEATURE_AVX2',['../intel-ipsec-mb_8h.html#a225e2cd9c1445795fc62879eabad2530',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fifma_82',['IMB_FEATURE_AVX512_IFMA',['../intel-ipsec-mb_8h.html#a0134dfaffe12427a8f2182247c6f1f48',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512_5fskx_83',['IMB_FEATURE_AVX512_SKX',['../intel-ipsec-mb_8h.html#ae6d6e3427c81b278d64f0189401c4964',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512bw_84',['IMB_FEATURE_AVX512BW',['../intel-ipsec-mb_8h.html#a00caae81d425acb6b946bebb25ac267f',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512cd_85',['IMB_FEATURE_AVX512CD',['../intel-ipsec-mb_8h.html#abbdb390b80ff1a92f31a7a9adc9beeb9',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512dq_86',['IMB_FEATURE_AVX512DQ',['../intel-ipsec-mb_8h.html#a8ca2c970faa0f62a3f4a6b6c18432bb1',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512f_87',['IMB_FEATURE_AVX512F',['../intel-ipsec-mb_8h.html#ab673c08624d868ca0dd89e1a9ec28f00',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5favx512vl_88',['IMB_FEATURE_AVX512VL',['../intel-ipsec-mb_8h.html#a803871900c08e2dc1b3f805ed4ecefd8',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fbmi2_89',['IMB_FEATURE_BMI2',['../intel-ipsec-mb_8h.html#a557f3de4bcc0bd7e9bd90e5e1d1b9b09',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fcmov_90',['IMB_FEATURE_CMOV',['../intel-ipsec-mb_8h.html#a5eef394935c651931b1cf560f31c841c',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fgfni_91',['IMB_FEATURE_GFNI',['../intel-ipsec-mb_8h.html#aa67ab4088ac2f8cadbe839101d276567',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fpclmulqdq_92',['IMB_FEATURE_PCLMULQDQ',['../intel-ipsec-mb_8h.html#a1af54a4a862936a6793af2ff44163ba5',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fdata_93',['IMB_FEATURE_SAFE_DATA',['../intel-ipsec-mb_8h.html#abd58fdbe1f1ef53cf509a0b37ba8aa4b',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsafe_5fparam_94',['IMB_FEATURE_SAFE_PARAM',['../intel-ipsec-mb_8h.html#a4e9506dc72ab4e6a2af463238ccb3080',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fshani_95',['IMB_FEATURE_SHANI',['../intel-ipsec-mb_8h.html#acb9c15eabfb9bafeb15a06886fa91e90',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fsse4_5f2_96',['IMB_FEATURE_SSE4_2',['../intel-ipsec-mb_8h.html#abcc243fc505aca3eac6963eff25e9b99',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvaes_97',['IMB_FEATURE_VAES',['../intel-ipsec-mb_8h.html#af4321e4e3c947f9b9fb7f204aaecf2c3',1,'intel-ipsec-mb.h']]], + ['imb_5ffeature_5fvpclmulqdq_98',['IMB_FEATURE_VPCLMULQDQ',['../intel-ipsec-mb_8h.html#a153d695ecd1af69a0d23385a50654614',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5faesni_5foff_99',['IMB_FLAG_AESNI_OFF',['../intel-ipsec-mb_8h.html#a4c53fb8e39bbc6e3f39d0558934e5d55',1,'intel-ipsec-mb.h']]], + ['imb_5fflag_5fshani_5foff_100',['IMB_FLAG_SHANI_OFF',['../intel-ipsec-mb_8h.html#a3d45fcff99a9ea952b4dd6ef70e36e0c',1,'intel-ipsec-mb.h']]], + ['imb_5fflush_5fjob_101',['IMB_FLUSH_JOB',['../intel-ipsec-mb_8h.html#a4726e3683b3579b1e643134278386d72',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f128_5fkey_5flen_102',['IMB_GCM_128_KEY_LEN',['../intel-ipsec-mb_8h.html#ae97681c203afbd818a39fb797613a232',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f192_5fkey_5flen_103',['IMB_GCM_192_KEY_LEN',['../intel-ipsec-mb_8h.html#a7201a0be67f1e6f3757169b0166dd47d',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5f256_5fkey_5flen_104',['IMB_GCM_256_KEY_LEN',['../intel-ipsec-mb_8h.html#a4c4c40ec87c2354832f5b182b22d2111',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fblock_5flen_105',['IMB_GCM_BLOCK_LEN',['../intel-ipsec-mb_8h.html#a59def6ed420a4ef91ce4a6562cc2c4d5',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fenc_5fkey_5flen_106',['IMB_GCM_ENC_KEY_LEN',['../intel-ipsec-mb_8h.html#a0c6f54ab6663187fa401c766a981a41e',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fiv_5fdata_5flen_107',['IMB_GCM_IV_DATA_LEN',['../intel-ipsec-mb_8h.html#a87884da318b3013893b725d047b53702',1,'intel-ipsec-mb.h']]], + ['imb_5fgcm_5fkey_5fsets_108',['IMB_GCM_KEY_SETS',['../intel-ipsec-mb_8h.html#a73715b8c1ba918ce6d5c21749841b47b',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fcompleted_5fjob_109',['IMB_GET_COMPLETED_JOB',['../intel-ipsec-mb_8h.html#a18e5a7f85573f0006448deda41014117',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fnext_5fjob_110',['IMB_GET_NEXT_JOB',['../intel-ipsec-mb_8h.html#a34b6006fc39dd4ea23d3fec49237d382',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_111',['IMB_GHASH',['../intel-ipsec-mb_8h.html#a477ad08a4c55e2c6458b7e86dae803ec',1,'intel-ipsec-mb.h']]], + ['imb_5fghash_5fpre_112',['IMB_GHASH_PRE',['../intel-ipsec-mb_8h.html#a230f16784519d7e2b69de87871d35cb2',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f32_113',['IMB_HEC_32',['../intel-ipsec-mb_8h.html#a37996cede66ee8036b1b113af5f553ac',1,'intel-ipsec-mb.h']]], + ['imb_5fhec_5f64_114',['IMB_HEC_64',['../intel-ipsec-mb_8h.html#aa0b62ba2c9fb07a1ad3c8105ec4374a4',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fblock_5fsize_115',['IMB_KASUMI_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#aa31ca303e03fc548199f95e5335889e1',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fdigest_5fsize_116',['IMB_KASUMI_DIGEST_SIZE',['../intel-ipsec-mb_8h.html#a56bd44d01c91469b57efa36f360a83bf',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_117',['IMB_KASUMI_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a45ac331e1f9241e53fc35a1f88dec6ce',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f1_5fbuffer_5fbit_118',['IMB_KASUMI_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#ae3befb703f7dec2c0695e61542f3212c',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f2_5fbuffer_119',['IMB_KASUMI_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a884bfa72fc232c7e84043d0faee719ee',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f3_5fbuffer_120',['IMB_KASUMI_F8_3_BUFFER',['../intel-ipsec-mb_8h.html#af35e87948218048e4bbad7162208fe41',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5f4_5fbuffer_121',['IMB_KASUMI_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a7ed6be7362e5ff7553fb4e7633bee76a',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff8_5fn_5fbuffer_122',['IMB_KASUMI_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a65b1afbadee23111046746a4d302bb0d',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_123',['IMB_KASUMI_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#a43c13bbdf83e2c75d6dd29d01dec4a1f',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5ff9_5f1_5fbuffer_5fuser_124',['IMB_KASUMI_F9_1_BUFFER_USER',['../intel-ipsec-mb_8h.html#a66dcf1985c046ef619141a6706159940',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff8_5fkey_5fsched_125',['IMB_KASUMI_INIT_F8_KEY_SCHED',['../intel-ipsec-mb_8h.html#aafcc540a047a6036adf4da0613ea5272',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5finit_5ff9_5fkey_5fsched_126',['IMB_KASUMI_INIT_F9_KEY_SCHED',['../intel-ipsec-mb_8h.html#a80a71ab95fd99c8427e171e45cebbd92',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fiv_5fsize_127',['IMB_KASUMI_IV_SIZE',['../intel-ipsec-mb_8h.html#adf7e13e43c6bb30489d0e26f11e39440',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsched_5fsize_128',['IMB_KASUMI_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#ae3812840fbe6b9b5ea5e84a6b86a7ed3',1,'intel-ipsec-mb.h']]], + ['imb_5fkasumi_5fkey_5fsize_129',['IMB_KASUMI_KEY_SIZE',['../intel-ipsec-mb_8h.html#a46d59512e3a7254e54018181b7a31102',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5fjobs_130',['IMB_MAX_JOBS',['../intel-ipsec-mb_8h.html#ab12d22470a68b5d3852c8dd972784d97',1,'intel-ipsec-mb.h']]], + ['imb_5fmax_5ftag_5flen_131',['IMB_MAX_TAG_LEN',['../intel-ipsec-mb_8h.html#a41087ea6dce2e096ebb02a613600c2ed',1,'intel-ipsec-mb.h']]], + ['imb_5fmd5_5fone_5fblock_132',['IMB_MD5_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a52b27293e2a19809c5f7d0f2afab97a2',1,'intel-ipsec-mb.h']]], + ['imb_5fqueue_5fsize_133',['IMB_QUEUE_SIZE',['../intel-ipsec-mb_8h.html#a712f6e7ecb2a16f8e1bc96b84d188365',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_134',['IMB_SHA1',['../intel-ipsec-mb_8h.html#ae61900d14cde2836621a1421141548a4',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fblock_5fsize_135',['IMB_SHA1_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#af19fd9d6768e33720cab4ecd9aabad91',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fdigest_5fsize_5fin_5fbytes_136',['IMB_SHA1_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a8c821d8cf5520f6f46e9b835eeeb2066',1,'intel-ipsec-mb.h']]], + ['imb_5fsha1_5fone_5fblock_137',['IMB_SHA1_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ac2f9695b12c32c303faa7c93c1c24c9b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_138',['IMB_SHA224',['../intel-ipsec-mb_8h.html#a542a40ced46bf97a678c6b44c325b360',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fdigest_5fsize_5fin_5fbytes_139',['IMB_SHA224_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a836641341e86b80f67716eabdf4a64c5',1,'intel-ipsec-mb.h']]], + ['imb_5fsha224_5fone_5fblock_140',['IMB_SHA224_ONE_BLOCK',['../intel-ipsec-mb_8h.html#ab1f7eb5e9bcd0c69a70dc76747f919f6',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_141',['IMB_SHA256',['../intel-ipsec-mb_8h.html#a37338b9f91315e5de0c4f3d19859aa7b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fdigest_5fsize_5fin_5fbytes_142',['IMB_SHA256_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a4bc9eee15ba0ce09f545af8916292731',1,'intel-ipsec-mb.h']]], + ['imb_5fsha256_5fone_5fblock_143',['IMB_SHA256_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a6023b87b5d1c6dc7dc0547db540af997',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_144',['IMB_SHA384',['../intel-ipsec-mb_8h.html#ac8052b8b996f866f27b4d443bcda0d5f',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fdigest_5fsize_5fin_5fbytes_145',['IMB_SHA384_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a2538a5e82141eb5a54f78660241e186c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha384_5fone_5fblock_146',['IMB_SHA384_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a763d0cfb16263906e44f46696b16124b',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_147',['IMB_SHA512',['../intel-ipsec-mb_8h.html#ac806d6b629633e801409d3e363d83f6c',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fdigest_5fsize_5fin_5fbytes_148',['IMB_SHA512_DIGEST_SIZE_IN_BYTES',['../intel-ipsec-mb_8h.html#a9efcf370f0db0b21f7cf4a3b72eec53e',1,'intel-ipsec-mb.h']]], + ['imb_5fsha512_5fone_5fblock_149',['IMB_SHA512_ONE_BLOCK',['../intel-ipsec-mb_8h.html#a71d0a529b355a434efeeda2e01870ba9',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f256_5fblock_5fsize_150',['IMB_SHA_256_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a7eed0159b050f8d78cddcc78b4b941ef',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f384_5fblock_5fsize_151',['IMB_SHA_384_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a256cc0f9eaba38030ca3f7589e479855',1,'intel-ipsec-mb.h']]], + ['imb_5fsha_5f512_5fblock_5fsize_152',['IMB_SHA_512_BLOCK_SIZE',['../intel-ipsec-mb_8h.html#a4141433b17ade98e14a585e0409cb62d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_153',['IMB_SNOW3G_F8_1_BUFFER',['../intel-ipsec-mb_8h.html#a4bcfbb70da38433f8bcd3468779026f1',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f1_5fbuffer_5fbit_154',['IMB_SNOW3G_F8_1_BUFFER_BIT',['../intel-ipsec-mb_8h.html#a5d7a9e694b932b1c198782e22dc1ead5',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f2_5fbuffer_155',['IMB_SNOW3G_F8_2_BUFFER',['../intel-ipsec-mb_8h.html#a51085837f0775feae26fe3885f7459b7',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f4_5fbuffer_156',['IMB_SNOW3G_F8_4_BUFFER',['../intel-ipsec-mb_8h.html#a79b873457a74ad6c50e689709d79a0fb',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_157',['IMB_SNOW3G_F8_8_BUFFER',['../intel-ipsec-mb_8h.html#a8536e68d053999123eb104762ef51932',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5f8_5fbuffer_5fmultikey_158',['IMB_SNOW3G_F8_8_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#aca8fb428a3b291950cd9dc6277ad801a',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_159',['IMB_SNOW3G_F8_N_BUFFER',['../intel-ipsec-mb_8h.html#a4538b34eaf3a3d402041ada1e8d03e2c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff8_5fn_5fbuffer_5fmultikey_160',['IMB_SNOW3G_F8_N_BUFFER_MULTIKEY',['../intel-ipsec-mb_8h.html#a4a5ae1b04c7aee099c2f8fcbd488c38c',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5ff9_5f1_5fbuffer_161',['IMB_SNOW3G_F9_1_BUFFER',['../intel-ipsec-mb_8h.html#aa6a951033156fc06c25afb4f79de8851',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5finit_5fkey_5fsched_162',['IMB_SNOW3G_INIT_KEY_SCHED',['../intel-ipsec-mb_8h.html#ac5b5ff1cc9cf44ea8587a7d054b7459d',1,'intel-ipsec-mb.h']]], + ['imb_5fsnow3g_5fkey_5fsched_5fsize_163',['IMB_SNOW3G_KEY_SCHED_SIZE',['../intel-ipsec-mb_8h.html#a1ddd766994cb5d0b8e9ef5cd580cdcd2',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_164',['IMB_SUBMIT_BURST',['../intel-ipsec-mb_8h.html#ad95138f7b201a9d7f8e57a155af720f4',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fburst_5fnocheck_165',['IMB_SUBMIT_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#ab043c084d20c19d9952dee899b693085',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_166',['IMB_SUBMIT_CIPHER_BURST',['../intel-ipsec-mb_8h.html#ae487720f5112a3ecec659ab4aeada77e',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fcipher_5fburst_5fnocheck_167',['IMB_SUBMIT_CIPHER_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#af819e81f584b3b56e0201a7174b3c0eb',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_168',['IMB_SUBMIT_HASH_BURST',['../intel-ipsec-mb_8h.html#a3702b37004e38dd27a9b8a008a0c48ed',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fhash_5fburst_5fnocheck_169',['IMB_SUBMIT_HASH_BURST_NOCHECK',['../intel-ipsec-mb_8h.html#afd6b80655a6129fe063d2ffb391e3015',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_170',['IMB_SUBMIT_JOB',['../intel-ipsec-mb_8h.html#a3602a4ccaf1da4bf6787169370cef04f',1,'intel-ipsec-mb.h']]], + ['imb_5fsubmit_5fjob_5fnocheck_171',['IMB_SUBMIT_JOB_NOCHECK',['../intel-ipsec-mb_8h.html#af23c64e466272c5ae90fcf78fbcd0c3d',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_172',['IMB_VERSION',['../intel-ipsec-mb_8h.html#ad9bcb21181d1e07b4f66d8bf0514c1a3',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fnum_173',['IMB_VERSION_NUM',['../intel-ipsec-mb_8h.html#a415abfc6067c0b1663a508993a4eb0d4',1,'intel-ipsec-mb.h']]], + ['imb_5fversion_5fstr_174',['IMB_VERSION_STR',['../intel-ipsec-mb_8h.html#a874972638400dafdb542319d61bfbfbc',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f1_5fbuffer_175',['IMB_ZUC_EEA3_1_BUFFER',['../intel-ipsec-mb_8h.html#adb23f957dde69fb3bb4c6798d1a69da4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5f4_5fbuffer_176',['IMB_ZUC_EEA3_4_BUFFER',['../intel-ipsec-mb_8h.html#a57445dd863838d04cf44f4d79c4dbcaf',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feea3_5fn_5fbuffer_177',['IMB_ZUC_EEA3_N_BUFFER',['../intel-ipsec-mb_8h.html#a6382e3185f31e61f995ab90110c52991',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5f1_5fbuffer_178',['IMB_ZUC_EIA3_1_BUFFER',['../intel-ipsec-mb_8h.html#ac789fe54f3a8f92144831a67b14501d4',1,'intel-ipsec-mb.h']]], + ['imb_5fzuc_5feia3_5fn_5fbuffer_179',['IMB_ZUC_EIA3_N_BUFFER',['../intel-ipsec-mb_8h.html#ae840d62f1246c0c4270c3319ef356d3c',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/defines_3.html b/docs/search/defines_3.html new file mode 100644 index 00000000..293a0304 --- /dev/null +++ b/docs/search/defines_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/defines_3.js b/docs/search/defines_3.js new file mode 100644 index 00000000..ec4e71c1 --- /dev/null +++ b/docs/search/defines_3.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['kasumi_5fkey_5fschedule_5fsize_0',['KASUMI_KEY_SCHEDULE_SIZE',['../intel-ipsec-mb_8h.html#a071e04977525b67c8d1accc60937eb67',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/enums_0.html b/docs/search/enums_0.html new file mode 100644 index 00000000..677cf7d3 --- /dev/null +++ b/docs/search/enums_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/enums_0.js b/docs/search/enums_0.js new file mode 100644 index 00000000..b5dd5c61 --- /dev/null +++ b/docs/search/enums_0.js @@ -0,0 +1,12 @@ +var searchData= +[ + ['imb_5farch_0',['IMB_ARCH',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6',1,'intel-ipsec-mb.h']]], + ['imb_5fchain_5forder_1',['IMB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdirection_2',['IMB_CIPHER_DIRECTION',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9ee',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fmode_3',['IMB_CIPHER_MODE',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_4',['IMB_ERR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0b',1,'intel-ipsec-mb.h']]], + ['imb_5fhash_5falg_5',['IMB_HASH_ALG',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5fsize_5fbytes_6',['IMB_KEY_SIZE_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cc',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fstate_7',['IMB_SGL_STATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3d',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_8',['IMB_STATUS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066ab',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/enumvalues_0.html b/docs/search/enumvalues_0.html new file mode 100644 index 00000000..5f03ba76 --- /dev/null +++ b/docs/search/enumvalues_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/enumvalues_0.js b/docs/search/enumvalues_0.js new file mode 100644 index 00000000..689ed0fa --- /dev/null +++ b/docs/search/enumvalues_0.js @@ -0,0 +1,148 @@ +var searchData= +[ + ['imb_5farch_5favx_0',['IMB_ARCH_AVX',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6aa831f565e051aa13fcad5b224358e57d',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx2_1',['IMB_ARCH_AVX2',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a701628fddfdd84d56495839f20573de6',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5favx512_2',['IMB_ARCH_AVX512',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1df4f12c5769b52a95f1a4b89a8163c8',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnoaesni_3',['IMB_ARCH_NOAESNI',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a6a5b29422391adbb6806575bf7ccae44',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnone_4',['IMB_ARCH_NONE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a1ff5121849f183accf83fd652b854733',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fnum_5',['IMB_ARCH_NUM',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6ae6536a65a8d0cd53faa325d5ee4dcdda',1,'intel-ipsec-mb.h']]], + ['imb_5farch_5fsse_6',['IMB_ARCH_SSE',['../intel-ipsec-mb_8h.html#ad138a5bf5508a0be5d3d61f018d5ebb6a3c2ccbe898a62cbaed5bad7717fdfa16',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fccm_7',['IMB_AUTH_AES_CCM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a824187b143cb8d4b1778d10bcedbce3e',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_8',['IMB_AUTH_AES_CMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aed85279dbff19be59c16216364375a48',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5f256_9',['IMB_AUTH_AES_CMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a0d0cad9513b19110454c9e4e9c549205',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fcmac_5fbitlen_10',['IMB_AUTH_AES_CMAC_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa9a94b55feed30ce57ecadf17b7c1a83',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_11',['IMB_AUTH_AES_GMAC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa1496e45f00afe54aeeb527089e14a0d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f128_12',['IMB_AUTH_AES_GMAC_128',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a189af960a1558b7ac3289c0892739ea0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f192_13',['IMB_AUTH_AES_GMAC_192',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adfd8db9df85ef929d9e4a38612f07d66',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fgmac_5f256_14',['IMB_AUTH_AES_GMAC_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae6fab01dbaf8b0110ca8ccdce98547af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5faes_5fxcbc_15',['IMB_AUTH_AES_XCBC',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a17deb61684701b7ec37f68bd34b6d454',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_16',['IMB_AUTH_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab737fa5ff7244ddf5024905f6477ae01',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fchacha20_5fpoly1305_5fsgl_17',['IMB_AUTH_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a71119edfbe668d423e910b11c0efb712',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc10_5fiuup_5fdata_18',['IMB_AUTH_CRC10_IUUP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ac27de87b2687b1e2a293544be6d3fab9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc11_5ffp_5fheader_19',['IMB_AUTH_CRC11_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab0f02ac14a03f137a83bb7d6dff539fc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5ffp_5fdata_20',['IMB_AUTH_CRC16_FP_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa78e695260f8fe6bb56d242771624ccc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc16_5fx25_21',['IMB_AUTH_CRC16_X25',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a22e2e766cbec475508af01c6a722c2c3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fa_22',['IMB_AUTH_CRC24_LTE_A',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3a8aa07fe9f70ece22f9351db0f8bb31',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc24_5flte_5fb_23',['IMB_AUTH_CRC24_LTE_B',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a77ccdf9abd97310f6808c8cf12c1a395',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fethernet_5ffcs_24',['IMB_AUTH_CRC32_ETHERNET_FCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a421294f9a35f87e796f96b48257e6dc2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fsctp_25',['IMB_AUTH_CRC32_SCTP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff8b335b5391fc33c80c076590de44',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc32_5fwimax_5fofdma_5fdata_26',['IMB_AUTH_CRC32_WIMAX_OFDMA_DATA',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad3936131d6804e6c6aa3bd497ae675ab',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc6_5fiuup_5fheader_27',['IMB_AUTH_CRC6_IUUP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a88175070e077037332ad5cd5fdece46f',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc7_5ffp_5fheader_28',['IMB_AUTH_CRC7_FP_HEADER',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ab7f1372ade9da040caf195b13f2b79f0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcrc8_5fwimax_5fofdma_5fhcs_29',['IMB_AUTH_CRC8_WIMAX_OFDMA_HCS',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae62fe411b683840a5f25c199b66c7f42',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fcustom_30',['IMB_AUTH_CUSTOM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a70151af8840242c8f5bdb375f2f1135b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fdocsis_5fcrc32_31',['IMB_AUTH_DOCSIS_CRC32',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae006c86697198ca852fc612d376135dc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fgcm_5fsgl_32',['IMB_AUTH_GCM_SGL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ada7d3fc165b6855ae3e83467de9916b2',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fghash_33',['IMB_AUTH_GHASH',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a67297eacd4913bd65a1aae650c3a22e3',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f1_34',['IMB_AUTH_HMAC_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a441cc344587709b2ff2fc26f36ff1e58',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f224_35',['IMB_AUTH_HMAC_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad0ad42ef3cb8cef38a29bcc667e70fca',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f256_36',['IMB_AUTH_HMAC_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a60c1e71108516903aded4534f0f2e995',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f384_37',['IMB_AUTH_HMAC_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ae98f8f8e42b7a026bb2aa2a402a291b9',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fhmac_5fsha_5f512_38',['IMB_AUTH_HMAC_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006ad8f7bc68ad10fe75665692a27ada55f1',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fkasumi_5fuia1_39',['IMB_AUTH_KASUMI_UIA1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a42dc1dc990059b61ef784813e0f8f49a',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fmd5_40',['IMB_AUTH_MD5',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a958fb278e4745955ebc791ba652954fe',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnull_41',['IMB_AUTH_NULL',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a25f47297460775ab891a0abb8240b8e0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fnum_42',['IMB_AUTH_NUM',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a5d34120f8a1c5cde857baf32536868af',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpoly1305_43',['IMB_AUTH_POLY1305',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a93e6c082f7ec5926a087d139a2bd2e56',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fpon_5fcrc_5fbip_44',['IMB_AUTH_PON_CRC_BIP',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a2fcc080535b4dd16502ecd2b01c14048',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f1_45',['IMB_AUTH_SHA_1',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006adcaf4a43c9c114bcdb75ff44ba1bea77',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f224_46',['IMB_AUTH_SHA_224',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a3936068075b4572d859c9b1984312325',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f256_47',['IMB_AUTH_SHA_256',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4962dbdf6fcb8d743e4ab55ce23c4b88',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f384_48',['IMB_AUTH_SHA_384',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006afe74859581d19e70b8667773662081bc',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsha_5f512_49',['IMB_AUTH_SHA_512',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006aa12aca10078adbe0bc2a021190663e2d',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow3g_5fuia2_5fbitlen_50',['IMB_AUTH_SNOW3G_UIA2_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a4eadf3270a81eb2567388843da2fa89b',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fsnow_5fv_5faead_51',['IMB_AUTH_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a53ff331c62b01fb8427e63e18fc433f6',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc256_5feia3_5fbitlen_52',['IMB_AUTH_ZUC256_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a7c535075abfb0878168a00600a762fe0',1,'intel-ipsec-mb.h']]], + ['imb_5fauth_5fzuc_5feia3_5fbitlen_53',['IMB_AUTH_ZUC_EIA3_BITLEN',['../intel-ipsec-mb_8h.html#a83132be2c4a6bf0453550bd1ae720006a188c2297dc5228820e690e8938cd407b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbc_54',['IMB_CIPHER_CBC',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0e2ebfcfd101ab4a39129082b2de517f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcbcs_5f1_5f9_55',['IMB_CIPHER_CBCS_1_9',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca71d87659793f7377d1d8b3315a3559f8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fccm_56',['IMB_CIPHER_CCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca7d3946d77e4364bc45a9e72499bf817f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_57',['IMB_CIPHER_CHACHA20',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca39d8d7ebca424a1c1a65bc04b5f64741',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_58',['IMB_CIPHER_CHACHA20_POLY1305',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caa5c375f97dce274a2446c8abf90aa61a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fchacha20_5fpoly1305_5fsgl_59',['IMB_CIPHER_CHACHA20_POLY1305_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca40926d8b731a401ec3e86ffa2089a1eb',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_60',['IMB_CIPHER_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca5b4126e439eaff0975846425594201ea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcntr_5fbitlen_61',['IMB_CIPHER_CNTR_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca6f69815480531651948b5549fa84023a',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fcustom_62',['IMB_CIPHER_CUSTOM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca4301726c71a4faf412a1a011cc4f0df8',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes_63',['IMB_CIPHER_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca049e85047492494fac0a4ec785707851',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdes3_64',['IMB_CIPHER_DES3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca639b1382de364a2b810f7876189bc64f',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fdes_65',['IMB_CIPHER_DOCSIS_DES',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caf9d62e1ff3a1a8b2ed1dcc515a3ffe1b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fdocsis_5fsec_5fbpi_66',['IMB_CIPHER_DOCSIS_SEC_BPI',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334caed8ba68d7d2d7602407f17305d1d99b7',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fecb_67',['IMB_CIPHER_ECB',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca357dec895f8dfa3371add1feaa001f56',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_68',['IMB_CIPHER_GCM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0f97344e525fef167a92d1a59a0492aa',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fgcm_5fsgl_69',['IMB_CIPHER_GCM_SGL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca48e024bc875c46b714b19a1c19908918',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fkasumi_5fuea1_5fbitlen_70',['IMB_CIPHER_KASUMI_UEA1_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3f7a3edb272955efea015c5bb8572e3b',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnull_71',['IMB_CIPHER_NULL',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca0bebd24d7311dedce100455dd3a8ddf6',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fnum_72',['IMB_CIPHER_NUM',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca78c470d405fb36037e41224b99d23c75',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fpon_5faes_5fcntr_73',['IMB_CIPHER_PON_AES_CNTR',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca896285cd2fe56f73011b908e490cbbea',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow3g_5fuea2_5fbitlen_74',['IMB_CIPHER_SNOW3G_UEA2_BITLEN',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca88c3ff9a6816b3629dc5b4815b47607e',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_75',['IMB_CIPHER_SNOW_V',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cafea56410df85a0b006963e50fba78465',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fsnow_5fv_5faead_76',['IMB_CIPHER_SNOW_V_AEAD',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334cada26994f8467300f862dbf45b5a22409',1,'intel-ipsec-mb.h']]], + ['imb_5fcipher_5fzuc_5feea3_77',['IMB_CIPHER_ZUC_EEA3',['../intel-ipsec-mb_8h.html#a3be1dfb4cbd4bd13dfdb2cf881f8334ca3618bfaab3c19aabc8a93a03ae121efc',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fdecrypt_78',['IMB_DIR_DECRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eea881652011955c227d94a881d2d2c5c59',1,'intel-ipsec-mb.h']]], + ['imb_5fdir_5fencrypt_79',['IMB_DIR_ENCRYPT',['../intel-ipsec-mb_8h.html#abdbf0bd32da801c52993cadb7a49c9eeabed12e936823e8f8b2de70388a2d15ce',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5faad_5flen_80',['IMB_ERR_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0ad3ade3fe22c5887d9bf444ff3923aa',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5flen_81',['IMB_ERR_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafe5af1c17e1caece14b3a5f659a1336d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fauth_5ftag_5flen_82',['IMB_ERR_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6af5bd8fb7690b46afe3d2c1e623808e',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5flen_83',['IMB_ERR_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bab5a2b4c1df7778c3efc236976062ed94',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fciph_5fmode_84',['IMB_ERR_CIPH_MODE',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba61a411feda0e8cf5aff8c58ad2e92bdd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fhash_5falgo_85',['IMB_ERR_HASH_ALGO',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8ac2802661bd9928d0c766129db2a814',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fiv_5flen_86',['IMB_ERR_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baac8500b58709f0bd05ad129337bb6af2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5faad_5flen_87',['IMB_ERR_JOB_AAD_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba31a6b672b95d7d100a97fa6d788c0dd6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5flen_88',['IMB_ERR_JOB_AUTH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69dd17473dc4afb2f04c417fe6882608',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fauth_5ftag_5flen_89',['IMB_ERR_JOB_AUTH_TAG_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1b7e4c43808275e9cd63ca21091e5ab2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fchain_5forder_90',['IMB_ERR_JOB_CHAIN_ORDER',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacaec18a061b4bf792178408d69775950',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5fdir_91',['IMB_ERR_JOB_CIPH_DIR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacbf5d148be9963df1d46400ff26396db',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fciph_5flen_92',['IMB_ERR_JOB_CIPH_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bae1c9e58d4be3fdd7bb37247667472920',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fiv_5flen_93',['IMB_ERR_JOB_IV_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba390f91827f87bbd3233ad6e1f104b2b4',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fkey_5flen_94',['IMB_ERR_JOB_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badfea107da94936d30900f23ad22b5285',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5faad_95',['IMB_ERR_JOB_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3ef3d3180e3052862c796cf89320d60c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_96',['IMB_ERR_JOB_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba503544043c60c2195003c00b18141beb',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fauth_5fkey_97',['IMB_ERR_JOB_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba97fb89f93299f67de86aac919bb83aa9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fdst_98',['IMB_ERR_JOB_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba55b3ddc1a48cb14ad020031b8d5b4927',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fghash_5finit_5ftag_99',['IMB_ERR_JOB_NULL_GHASH_INIT_TAG',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba6c97704719f286b14204ca144bbdc7a9',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fipad_100',['IMB_ERR_JOB_NULL_HMAC_IPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba15f55e87bb42cf5494dd01b9a1c4d712',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fhmac_5fopad_101',['IMB_ERR_JOB_NULL_HMAC_OPAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bacb9286af7533993ab79982b288aab97a',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fiv_102',['IMB_ERR_JOB_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0d35070d6476cde0202deb5531ae09c2',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fkey_103',['IMB_ERR_JOB_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba22402460340d6b34492ece390a7913c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fnext_5fiv_104',['IMB_ERR_JOB_NULL_NEXT_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba983561202b1623b0bd1f691bd6036039',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsgl_5fctx_105',['IMB_ERR_JOB_NULL_SGL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba05f9212af539a22ad65b12a9abc8e5c0',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fsrc_106',['IMB_ERR_JOB_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4bc6a784afa9bb88c5afe4fd1b5f610d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk1_5fexp_107',['IMB_ERR_JOB_NULL_XCBC_K1_EXP',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba69bf9e54fdf84b154284eab071c98804',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk2_108',['IMB_ERR_JOB_NULL_XCBC_K2',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bad7fbd09092821fb8605e11384dcdc428',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fnull_5fxcbc_5fk3_109',['IMB_ERR_JOB_NULL_XCBC_K3',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baf12955208a99c8feeeb580fee940b7c3',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fpon_5fpli_110',['IMB_ERR_JOB_PON_PLI',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bace7e834842ddef5b063c99544d00b303',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fjob_5fsrc_5foffset_111',['IMB_ERR_JOB_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba207979780b5d8472dfef2f5e12998d22',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fkey_5flen_112',['IMB_ERR_KEY_LEN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba8d6b3ea721216bb12fbe40e72a5bd768',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmax_113',['IMB_ERR_MAX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bac773be615f9904fd69e961451fb0fac6',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmin_114',['IMB_ERR_MIN',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3f9d55f356a0caf413d2114a468ba659',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fmissing_5fcpuflags_5finit_5fmgr_115',['IMB_ERR_MISSING_CPUFLAGS_INIT_MGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0babdd0b7bc92e7dcf166d36a9bcd118d60',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fno_5faesni_5femu_116',['IMB_ERR_NO_AESNI_EMU',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba4496bf14e2ddf1133a384f9fbf96d5dc',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5faad_117',['IMB_ERR_NULL_AAD',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0badc7d17d90012d5c14fd8a12d9ea12543',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_118',['IMB_ERR_NULL_AUTH',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1e20954ec8a501093175d640631550c1',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fauth_5fkey_119',['IMB_ERR_NULL_AUTH_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0baa4f5aeab78ed40a4c19389b8f691306c',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fctx_120',['IMB_ERR_NULL_CTX',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba0b57ec134f6e10e11c3fbcf77ede282d',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fdst_121',['IMB_ERR_NULL_DST',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba1a61dcc99a1a9725c89076dec3c02e02',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fexp_5fkey_122',['IMB_ERR_NULL_EXP_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba3b60a570372529486da69c6d2b7be382',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fiv_123',['IMB_ERR_NULL_IV',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0bafcfe882ee9730fd1724b587106f2f419',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fjob_124',['IMB_ERR_NULL_JOB',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba007c436f8e21f317e8f9225c5b009d38',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fkey_125',['IMB_ERR_NULL_KEY',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba63029e3673eef74345ad80867f7ba0ef',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fmbmgr_126',['IMB_ERR_NULL_MBMGR',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba9ba69cc32a2d17e3e7b4d4c982f26abd',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fnull_5fsrc_127',['IMB_ERR_NULL_SRC',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba308819b7cbdadfd6bbee290408880302',1,'intel-ipsec-mb.h']]], + ['imb_5ferr_5fsrc_5foffset_128',['IMB_ERR_SRC_OFFSET',['../intel-ipsec-mb_8h.html#aa12c6dbc4b212887ef528c215f513e0ba20dfbdc8feb8332a9efe51de48fa6e09',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f128_5fbytes_129',['IMB_KEY_128_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccaf3e82db9a8b5b1b229c201735938969d',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f192_5fbytes_130',['IMB_KEY_192_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33cca573d62aa6a04d9ce2b3c34e226846598',1,'intel-ipsec-mb.h']]], + ['imb_5fkey_5f256_5fbytes_131',['IMB_KEY_256_BYTES',['../intel-ipsec-mb_8h.html#a203fa854f71a1cca79f3f3ec251a33ccac139e7555c9b30725359698833eb213a',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fcipher_5fhash_132',['IMB_ORDER_CIPHER_HASH',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5e3ab845752ce3d16e40ea7802ce4a52',1,'intel-ipsec-mb.h']]], + ['imb_5forder_5fhash_5fcipher_133',['IMB_ORDER_HASH_CIPHER',['../intel-ipsec-mb_8h.html#a4a72299215a4838a93cdce6e5a37cfa5a5de8d032616267941d99dad14801044d',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fall_134',['IMB_SGL_ALL',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da2f0678ab7ad9bb30c040e0b3ee2fa408',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fcomplete_135',['IMB_SGL_COMPLETE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3dab7a602903c0fbaad7ffb893cb2547975',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5finit_136',['IMB_SGL_INIT',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3daf6e77a48759ab366cc1160725ff4261e',1,'intel-ipsec-mb.h']]], + ['imb_5fsgl_5fupdate_137',['IMB_SGL_UPDATE',['../intel-ipsec-mb_8h.html#a6b93fe0c0d7cdc520e6b465ddd93ec3da15c5d0aeeb9d5eb4e4c4878c837c5977',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fbeing_5fprocessed_138',['IMB_STATUS_BEING_PROCESSED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaa562b01ade81e0196fc23cd79d534141',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_139',['IMB_STATUS_COMPLETED',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba508a9f195acee09621542ca687ba2341',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fauth_140',['IMB_STATUS_COMPLETED_AUTH',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba79df83a15cdafc2a6c79751457ae9792',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5fcompleted_5fcipher_141',['IMB_STATUS_COMPLETED_CIPHER',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba6fcaccd3853c9e8cf3b402edc3e22089',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5ferror_142',['IMB_STATUS_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba0ef40895610ac6343d171f65be676a1d',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finternal_5ferror_143',['IMB_STATUS_INTERNAL_ERROR',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066aba546465b9d4cfb1f9c98c864bf5ed51fb',1,'intel-ipsec-mb.h']]], + ['imb_5fstatus_5finvalid_5fargs_144',['IMB_STATUS_INVALID_ARGS',['../intel-ipsec-mb_8h.html#afd5f50bd2277b5f92f0ec786c8f066abaee85b6a8bc33e0b39001e9ff538540b7',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/files_0.html b/docs/search/files_0.html new file mode 100644 index 00000000..b416deb0 --- /dev/null +++ b/docs/search/files_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/files_0.js b/docs/search/files_0.js new file mode 100644 index 00000000..6ed22803 --- /dev/null +++ b/docs/search/files_0.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['intel_2dipsec_2dmb_2eh_0',['intel-ipsec-mb.h',['../intel-ipsec-mb_8h.html',1,'']]] +]; diff --git a/docs/search/files_1.html b/docs/search/files_1.html new file mode 100644 index 00000000..b999da2c --- /dev/null +++ b/docs/search/files_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/files_1.js b/docs/search/files_1.js new file mode 100644 index 00000000..98476761 --- /dev/null +++ b/docs/search/files_1.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['readme_2emd_0',['README.md',['../README_8md.html',1,'']]] +]; diff --git a/docs/search/functions_0.html b/docs/search/functions_0.html new file mode 100644 index 00000000..3a7c1549 --- /dev/null +++ b/docs/search/functions_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_0.js b/docs/search/functions_0.js new file mode 100644 index 00000000..0b95ec55 --- /dev/null +++ b/docs/search/functions_0.js @@ -0,0 +1,121 @@ +var searchData= +[ + ['aes_5fcfb_5f128_5fone_5favx_0',['aes_cfb_128_one_avx',['../intel-ipsec-mb_8h.html#a5a330bd4ed945f830c88dd6e1066652d',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5favx2_1',['aes_cfb_128_one_avx2',['../intel-ipsec-mb_8h.html#a7046666312bdba42b05bfbf63d0e1c6a',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5favx512_2',['aes_cfb_128_one_avx512',['../intel-ipsec-mb_8h.html#abb7c6395420af2addd92a3fa46e0fd28',1,'intel-ipsec-mb.h']]], + ['aes_5fcfb_5f128_5fone_5fsse_3',['aes_cfb_128_one_sse',['../intel-ipsec-mb_8h.html#a7bc28362ae2fb8a4d624b8a7a00adebd',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx_4',['aes_cmac_subkey_gen_avx',['../intel-ipsec-mb_8h.html#a6541825066a8d2b8b3c7b6e2c4b28c83',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx2_5',['aes_cmac_subkey_gen_avx2',['../intel-ipsec-mb_8h.html#a6903e1e0dfb8fd75c2f556596f3efa1b',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5favx512_6',['aes_cmac_subkey_gen_avx512',['../intel-ipsec-mb_8h.html#a730f18bd7cdcbbb995e24931307867fc',1,'intel-ipsec-mb.h']]], + ['aes_5fcmac_5fsubkey_5fgen_5fsse_7',['aes_cmac_subkey_gen_sse',['../intel-ipsec-mb_8h.html#a4dafe82f4933670f68e1bf34d55a691d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen2_8',['aes_gcm_dec_128_avx_gen2',['../intel-ipsec-mb_8h.html#a8c9a7e51599dbce89a416e0d8422742b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5favx_5fgen4_9',['aes_gcm_dec_128_avx_gen4',['../intel-ipsec-mb_8h.html#ae1e8e1b4425af520048a9e1fe5689e4d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen2_10',['aes_gcm_dec_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#ac173e4fb252a820edb25ad790065ad39',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5favx_5fgen4_11',['aes_gcm_dec_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a8e2bc4fff9de3c70ac140ff30e1cdce0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5ffinalize_5fsse_12',['aes_gcm_dec_128_finalize_sse',['../intel-ipsec-mb_8h.html#a08603861ac18d1dce7b322b0c3fb69c5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fsse_13',['aes_gcm_dec_128_sse',['../intel-ipsec-mb_8h.html#add2f5aafaf55cd0e0b20b2b62fcb025d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen2_14',['aes_gcm_dec_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#a796e1df6d2aa233a6e28a867c1a2c628',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5favx_5fgen4_15',['aes_gcm_dec_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#a4464684ff4cf4e9cb978409ad350c4d6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f128_5fupdate_5fsse_16',['aes_gcm_dec_128_update_sse',['../intel-ipsec-mb_8h.html#a2216e461e916668b70d547513b0cf26f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen2_17',['aes_gcm_dec_192_avx_gen2',['../intel-ipsec-mb_8h.html#ad45e83e263506684c786b4e7012055ac',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5favx_5fgen4_18',['aes_gcm_dec_192_avx_gen4',['../intel-ipsec-mb_8h.html#a5ccd87fbcfffc238fbcd1fc816eb06ee',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen2_19',['aes_gcm_dec_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#aacc19f04c9ac80d1bf7391661e191798',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5favx_5fgen4_20',['aes_gcm_dec_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#afa136a3c06d45e914d15415dee27f001',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5ffinalize_5fsse_21',['aes_gcm_dec_192_finalize_sse',['../intel-ipsec-mb_8h.html#a0fd357a1794d78ac2168bc7eff0442ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fsse_22',['aes_gcm_dec_192_sse',['../intel-ipsec-mb_8h.html#acaff83c8ca568201805e0ada5a79f6e1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen2_23',['aes_gcm_dec_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#aa81615f7ddfc8e1a794c5e278d22f7fc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5favx_5fgen4_24',['aes_gcm_dec_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a615c26f03ba08c8cde73cb074bf6b363',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f192_5fupdate_5fsse_25',['aes_gcm_dec_192_update_sse',['../intel-ipsec-mb_8h.html#a58455aec5650ecfc6077c69c97031f63',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen2_26',['aes_gcm_dec_256_avx_gen2',['../intel-ipsec-mb_8h.html#a9b79136dc5996baf598e3aeee520871c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5favx_5fgen4_27',['aes_gcm_dec_256_avx_gen4',['../intel-ipsec-mb_8h.html#ac16384799933d4f7b11cb56d9c66f5da',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen2_28',['aes_gcm_dec_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a72f76caa84e2803b2b4a26c81623f89e',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5favx_5fgen4_29',['aes_gcm_dec_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a791be274839adc3d26d9a32aa319dc3b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5ffinalize_5fsse_30',['aes_gcm_dec_256_finalize_sse',['../intel-ipsec-mb_8h.html#adf7aa2e79dce7a9ed79134c086715fd0',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fsse_31',['aes_gcm_dec_256_sse',['../intel-ipsec-mb_8h.html#afd8763c66749e9bc5e4e5eb2ac130e43',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen2_32',['aes_gcm_dec_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a0c31c2e2760edc7b252214214159cd7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5favx_5fgen4_33',['aes_gcm_dec_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#a141303c38f4eed1245b542b6e9473296',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fdec_5f256_5fupdate_5fsse_34',['aes_gcm_dec_256_update_sse',['../intel-ipsec-mb_8h.html#ab1a51ae4b3cf3e8fdb618773d7b24b68',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen2_35',['aes_gcm_enc_128_avx_gen2',['../intel-ipsec-mb_8h.html#af33d7afa861dcedc2c284330a5a66164',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5favx_5fgen4_36',['aes_gcm_enc_128_avx_gen4',['../intel-ipsec-mb_8h.html#aa17c919e873b6cfd7ac990977eef7601',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen2_37',['aes_gcm_enc_128_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a40a83ee6dec0440cd99e5ff600e3803d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5favx_5fgen4_38',['aes_gcm_enc_128_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a004c38b922d25d8631252854426ebf44',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5ffinalize_5fsse_39',['aes_gcm_enc_128_finalize_sse',['../intel-ipsec-mb_8h.html#af63ae2246c6d89b7d6630623948d3886',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fsse_40',['aes_gcm_enc_128_sse',['../intel-ipsec-mb_8h.html#aad9e6c466a5274b974d20330f91add50',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen2_41',['aes_gcm_enc_128_update_avx_gen2',['../intel-ipsec-mb_8h.html#ae3459d38a2ee2e1ce04b9f9f4bf55ea5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5favx_5fgen4_42',['aes_gcm_enc_128_update_avx_gen4',['../intel-ipsec-mb_8h.html#abb5570a161a90391e25b2016bc98381b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f128_5fupdate_5fsse_43',['aes_gcm_enc_128_update_sse',['../intel-ipsec-mb_8h.html#a7b12cabbfebb420022a19124aff78a3f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen2_44',['aes_gcm_enc_192_avx_gen2',['../intel-ipsec-mb_8h.html#a0ff9569b5072896a4119a4c7d3849bfc',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5favx_5fgen4_45',['aes_gcm_enc_192_avx_gen4',['../intel-ipsec-mb_8h.html#a9b462a3c00471e4338f0bae7d5abbda7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen2_46',['aes_gcm_enc_192_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a5177757ae4a75ba41d5e7a8a534edc06',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5favx_5fgen4_47',['aes_gcm_enc_192_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a4f76beaee47ffeeeb6f94a132f898339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5ffinalize_5fsse_48',['aes_gcm_enc_192_finalize_sse',['../intel-ipsec-mb_8h.html#a5dcb5c37d1a5377967888e9b76208f8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fsse_49',['aes_gcm_enc_192_sse',['../intel-ipsec-mb_8h.html#adec2f9778622b3e226f12bb326e75076',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen2_50',['aes_gcm_enc_192_update_avx_gen2',['../intel-ipsec-mb_8h.html#a548fcbb89ff5fb798bf71ab8e4ac88eb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5favx_5fgen4_51',['aes_gcm_enc_192_update_avx_gen4',['../intel-ipsec-mb_8h.html#a174447b05661285b2729805f66bcef81',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f192_5fupdate_5fsse_52',['aes_gcm_enc_192_update_sse',['../intel-ipsec-mb_8h.html#a46c3df131f9e1e3ed1a283349c4072f7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen2_53',['aes_gcm_enc_256_avx_gen2',['../intel-ipsec-mb_8h.html#a45dea093699e6e2dd462b6cafccee672',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5favx_5fgen4_54',['aes_gcm_enc_256_avx_gen4',['../intel-ipsec-mb_8h.html#a52c0913f10dd41fea5fe328037561ff8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen2_55',['aes_gcm_enc_256_finalize_avx_gen2',['../intel-ipsec-mb_8h.html#a7fa5d4f44bb44600fa7b86697ccc385f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5favx_5fgen4_56',['aes_gcm_enc_256_finalize_avx_gen4',['../intel-ipsec-mb_8h.html#a873a75e2175acd3741b5cfc41180fe2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5ffinalize_5fsse_57',['aes_gcm_enc_256_finalize_sse',['../intel-ipsec-mb_8h.html#afcdedeabc7516e7506fa5c81d07f6fd8',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fsse_58',['aes_gcm_enc_256_sse',['../intel-ipsec-mb_8h.html#a460ad1ebcb7df39e39c9aca214d8109c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen2_59',['aes_gcm_enc_256_update_avx_gen2',['../intel-ipsec-mb_8h.html#a351c2bf07dcab7e01570273c9df1e56f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5favx_5fgen4_60',['aes_gcm_enc_256_update_avx_gen4',['../intel-ipsec-mb_8h.html#ad9592051cf26814ea5830826b553c547',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5f256_5fupdate_5fsse_61',['aes_gcm_enc_256_update_sse',['../intel-ipsec-mb_8h.html#ab831bdb2193a5adc54b51285f23ad5ce',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen2_62',['aes_gcm_init_128_avx_gen2',['../intel-ipsec-mb_8h.html#a7a9a24d480dcedac1aa40d53e01b8a76',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5favx_5fgen4_63',['aes_gcm_init_128_avx_gen4',['../intel-ipsec-mb_8h.html#a8d225d08a5cf144208dacb155e192cb6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f128_5fsse_64',['aes_gcm_init_128_sse',['../intel-ipsec-mb_8h.html#aca7ab49c370f8f7cb82f93db199bfd1c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen2_65',['aes_gcm_init_192_avx_gen2',['../intel-ipsec-mb_8h.html#ac24548c2955399fd84b03216ad5b36fb',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5favx_5fgen4_66',['aes_gcm_init_192_avx_gen4',['../intel-ipsec-mb_8h.html#a757cd199a2e993c6610bbcdfc1da1ae4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f192_5fsse_67',['aes_gcm_init_192_sse',['../intel-ipsec-mb_8h.html#a5bbf84135701f572ec2ecb0c0898848b',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen2_68',['aes_gcm_init_256_avx_gen2',['../intel-ipsec-mb_8h.html#a3dd74260f4f34f7a53253c63ad3a6701',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5favx_5fgen4_69',['aes_gcm_init_256_avx_gen4',['../intel-ipsec-mb_8h.html#a99310b73bc9cde6b69b682e72cd0efc1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5f256_5fsse_70',['aes_gcm_init_256_sse',['../intel-ipsec-mb_8h.html#a09949bd5112eec14773dce239ca39033',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen2_71',['aes_gcm_pre_128_avx_gen2',['../intel-ipsec-mb_8h.html#a2f60098fff25e8b71b35e42cc7854339',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5favx_5fgen4_72',['aes_gcm_pre_128_avx_gen4',['../intel-ipsec-mb_8h.html#a3d7b8bc454b44abbeaa8f435280f8cb1',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f128_5fsse_73',['aes_gcm_pre_128_sse',['../intel-ipsec-mb_8h.html#a5d02ac2586befa6cd3fd4921858c33fd',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen2_74',['aes_gcm_pre_192_avx_gen2',['../intel-ipsec-mb_8h.html#a6c9c0b179633aca3ba5c520bee32662f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5favx_5fgen4_75',['aes_gcm_pre_192_avx_gen4',['../intel-ipsec-mb_8h.html#ab71c95df9bd059e1a5e108d2513a0e2c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f192_5fsse_76',['aes_gcm_pre_192_sse',['../intel-ipsec-mb_8h.html#ac0ebc1e7678bde75c25d5c0ec7ab2f7a',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen2_77',['aes_gcm_pre_256_avx_gen2',['../intel-ipsec-mb_8h.html#aaaabf4acdd43f98b6c2d31c8013ebbf4',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5favx_5fgen4_78',['aes_gcm_pre_256_avx_gen4',['../intel-ipsec-mb_8h.html#a082773f106f399ca73afc56583805a8c',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5f256_5fsse_79',['aes_gcm_pre_256_sse',['../intel-ipsec-mb_8h.html#a18fac8ea0f3acabb4d5ffab1152f477f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen2_80',['aes_gcm_precomp_128_avx_gen2',['../intel-ipsec-mb_8h.html#ade16ed4c31baa50f8a10e3c85156a942',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5favx_5fgen4_81',['aes_gcm_precomp_128_avx_gen4',['../intel-ipsec-mb_8h.html#aecf7277999128599c6cf8432b242ba36',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f128_5fsse_82',['aes_gcm_precomp_128_sse',['../intel-ipsec-mb_8h.html#a46253857355ddae8d17bab1403289d0f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen2_83',['aes_gcm_precomp_192_avx_gen2',['../intel-ipsec-mb_8h.html#ab0055e3b7d00cdb578647921207a1de7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5favx_5fgen4_84',['aes_gcm_precomp_192_avx_gen4',['../intel-ipsec-mb_8h.html#a6cb54cf9321dfb95af2eeb812f41fa8f',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f192_5fsse_85',['aes_gcm_precomp_192_sse',['../intel-ipsec-mb_8h.html#a72d93e3249806eee186d465dc04c09e6',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen2_86',['aes_gcm_precomp_256_avx_gen2',['../intel-ipsec-mb_8h.html#a2a883085207ee21caca109ab8250db08',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5favx_5fgen4_87',['aes_gcm_precomp_256_avx_gen4',['../intel-ipsec-mb_8h.html#abef922c52013630ad70273cef095c564',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5f256_5fsse_88',['aes_gcm_precomp_256_sse',['../intel-ipsec-mb_8h.html#a18001897ea80cdce1f4f608b0922f767',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx_89',['aes_keyexp_128_avx',['../intel-ipsec-mb_8h.html#aec7fe23c32ee94de88e5feba8f373a04',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx2_90',['aes_keyexp_128_avx2',['../intel-ipsec-mb_8h.html#a4fc7f88bdb263ae0d45f37af7e4cb930',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5favx512_91',['aes_keyexp_128_avx512',['../intel-ipsec-mb_8h.html#a9b7faaf3e5235d0a3c4b784f1035896d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx_92',['aes_keyexp_128_enc_avx',['../intel-ipsec-mb_8h.html#ac7d7c889a25730d82ddee27bad7f0a63',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx2_93',['aes_keyexp_128_enc_avx2',['../intel-ipsec-mb_8h.html#a84e76bdfb5a69dba7b515dd0b4fb4312',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5favx512_94',['aes_keyexp_128_enc_avx512',['../intel-ipsec-mb_8h.html#a68e6a0e99a2091d3630bf3315fe9d95c',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fenc_5fsse_95',['aes_keyexp_128_enc_sse',['../intel-ipsec-mb_8h.html#a270621a2f73134d8d021b5dd57ad6817',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f128_5fsse_96',['aes_keyexp_128_sse',['../intel-ipsec-mb_8h.html#a32bcda3484b33b7d01124dbfeb9a229b',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx_97',['aes_keyexp_192_avx',['../intel-ipsec-mb_8h.html#a3bc56101b48217094086ae3120b0e086',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx2_98',['aes_keyexp_192_avx2',['../intel-ipsec-mb_8h.html#aba6dd2e633819829740cb7917ba6b1b2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5favx512_99',['aes_keyexp_192_avx512',['../intel-ipsec-mb_8h.html#a1f69c0a43de83f8fba19107a63c07831',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx_100',['aes_keyexp_192_enc_avx',['../intel-ipsec-mb_8h.html#a5350b81ab574c990d6a5b9dc3b203c51',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx2_101',['aes_keyexp_192_enc_avx2',['../intel-ipsec-mb_8h.html#a460853c072adc3a0f431e861d3e5bba1',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5favx512_102',['aes_keyexp_192_enc_avx512',['../intel-ipsec-mb_8h.html#a72807e8300208b22b5d532e274385127',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fenc_5fsse_103',['aes_keyexp_192_enc_sse',['../intel-ipsec-mb_8h.html#a0a843e722f2af7a645340f6a98cfa4c2',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f192_5fsse_104',['aes_keyexp_192_sse',['../intel-ipsec-mb_8h.html#a7681d777bb5f991ab59b5581bc854562',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx_105',['aes_keyexp_256_avx',['../intel-ipsec-mb_8h.html#a5b92ee3873b2e2cd02c9312b6040ee91',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx2_106',['aes_keyexp_256_avx2',['../intel-ipsec-mb_8h.html#af1f440855a00edbf95c124a1e1d18c37',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5favx512_107',['aes_keyexp_256_avx512',['../intel-ipsec-mb_8h.html#a37e0f61656618413fcfd919dfe79ca45',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx_108',['aes_keyexp_256_enc_avx',['../intel-ipsec-mb_8h.html#a22096fdd9b5d32095571f04a838d66a9',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx2_109',['aes_keyexp_256_enc_avx2',['../intel-ipsec-mb_8h.html#a73da5625112ad845a3ec94ee84590f54',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5favx512_110',['aes_keyexp_256_enc_avx512',['../intel-ipsec-mb_8h.html#a9fb47323b270bffc87ad8e77985e827d',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fenc_5fsse_111',['aes_keyexp_256_enc_sse',['../intel-ipsec-mb_8h.html#afa369dee8def2ef620e7f33db86a3356',1,'intel-ipsec-mb.h']]], + ['aes_5fkeyexp_5f256_5fsse_112',['aes_keyexp_256_sse',['../intel-ipsec-mb_8h.html#a5105a8d995cd8e4354c7de7ac94886cb',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx_113',['aes_xcbc_expand_key_avx',['../intel-ipsec-mb_8h.html#afe460e4ff6a7b0c7a6edcc87e9976121',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx2_114',['aes_xcbc_expand_key_avx2',['../intel-ipsec-mb_8h.html#ac3f27672ce30d74f5cdc28f1903f68b1',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5favx512_115',['aes_xcbc_expand_key_avx512',['../intel-ipsec-mb_8h.html#a1308be648182642b8d73ff27d9344564',1,'intel-ipsec-mb.h']]], + ['aes_5fxcbc_5fexpand_5fkey_5fsse_116',['aes_xcbc_expand_key_sse',['../intel-ipsec-mb_8h.html#aefb28e46620cbf65a2af3073b2401e40',1,'intel-ipsec-mb.h']]], + ['alloc_5fmb_5fmgr_117',['alloc_mb_mgr',['../intel-ipsec-mb_8h.html#ab8dc617feb010b0f4cd74fd9e1eabb5e',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_1.html b/docs/search/functions_1.html new file mode 100644 index 00000000..801b6ddf --- /dev/null +++ b/docs/search/functions_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_1.js b/docs/search/functions_1.js new file mode 100644 index 00000000..d54b34ab --- /dev/null +++ b/docs/search/functions_1.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['des_5fkey_5fschedule_0',['des_key_schedule',['../intel-ipsec-mb_8h.html#a9911462fe4689c7420b827bb13fc3440',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_2.html b/docs/search/functions_2.html new file mode 100644 index 00000000..f1d0a602 --- /dev/null +++ b/docs/search/functions_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_2.js b/docs/search/functions_2.js new file mode 100644 index 00000000..2a4dca86 --- /dev/null +++ b/docs/search/functions_2.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['flush_5fjob_5favx_0',['flush_job_avx',['../intel-ipsec-mb_8h.html#abc6b0f1c857f26b834f22b6bd8f51846',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx2_1',['flush_job_avx2',['../intel-ipsec-mb_8h.html#ae242d142065172898129f30e6bc40d8c',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5favx512_2',['flush_job_avx512',['../intel-ipsec-mb_8h.html#ac7aec54f653aeee2554ba980d62fd491',1,'intel-ipsec-mb.h']]], + ['flush_5fjob_5fsse_3',['flush_job_sse',['../intel-ipsec-mb_8h.html#a96b880b8a32b0466f880d4f22d95cc3b',1,'intel-ipsec-mb.h']]], + ['free_5fmb_5fmgr_4',['free_mb_mgr',['../intel-ipsec-mb_8h.html#a238b96cc36c0dc64798a1244b94add8b',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_3.html b/docs/search/functions_3.html new file mode 100644 index 00000000..8bf95fec --- /dev/null +++ b/docs/search/functions_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_3.js b/docs/search/functions_3.js new file mode 100644 index 00000000..f70b47e0 --- /dev/null +++ b/docs/search/functions_3.js @@ -0,0 +1,11 @@ +var searchData= +[ + ['get_5fcompleted_5fjob_5favx_0',['get_completed_job_avx',['../intel-ipsec-mb_8h.html#a9d115c47670330bd01b960f2188169d9',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx2_1',['get_completed_job_avx2',['../intel-ipsec-mb_8h.html#ad85bdbf56038fd72ceda7e50deea586e',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5favx512_2',['get_completed_job_avx512',['../intel-ipsec-mb_8h.html#a693afd528fcc0809cc667b6c38de84dc',1,'intel-ipsec-mb.h']]], + ['get_5fcompleted_5fjob_5fsse_3',['get_completed_job_sse',['../intel-ipsec-mb_8h.html#a267a718472e0c1b6585402e81e44e7e9',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx_4',['get_next_job_avx',['../intel-ipsec-mb_8h.html#a3518b02f06a82e9f83cd0435f3cf25c9',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx2_5',['get_next_job_avx2',['../intel-ipsec-mb_8h.html#a49a3d54979188b3c68bdd621043125f7',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5favx512_6',['get_next_job_avx512',['../intel-ipsec-mb_8h.html#a40c08ba2c387e24f2096b1213c46c812',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5fsse_7',['get_next_job_sse',['../intel-ipsec-mb_8h.html#a055db1f6f16bc17724bdc64d0f313871',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_4.html b/docs/search/functions_4.html new file mode 100644 index 00000000..be730119 --- /dev/null +++ b/docs/search/functions_4.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_4.js b/docs/search/functions_4.js new file mode 100644 index 00000000..8061e62d --- /dev/null +++ b/docs/search/functions_4.js @@ -0,0 +1,16 @@ +var searchData= +[ + ['imb_5fclear_5fmem_0',['imb_clear_mem',['../intel-ipsec-mb_8h.html#aecdd69541b6d2a83d19ca804ef4b78d6',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ferrno_1',['imb_get_errno',['../intel-ipsec-mb_8h.html#a79f60b2bffbe91dac3e4f8b486e7ab10',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5ffeature_5fflags_2',['imb_get_feature_flags',['../intel-ipsec-mb_8h.html#a725703ff6da6b402bde7041a2fabfddf',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fmb_5fmgr_5fsize_3',['imb_get_mb_mgr_size',['../intel-ipsec-mb_8h.html#ac8379ada0d6ec030ebad9828fbea9459',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fstrerror_4',['imb_get_strerror',['../intel-ipsec-mb_8h.html#a61f664448a862fbb02e728ce518ff8e4',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5',['imb_get_version',['../intel-ipsec-mb_8h.html#aee02bcb3a5e5eaee3632086f75c3d78d',1,'intel-ipsec-mb.h']]], + ['imb_5fget_5fversion_5fstr_6',['imb_get_version_str',['../intel-ipsec-mb_8h.html#a4490e7b324236315f1a9e141b39f99f2',1,'intel-ipsec-mb.h']]], + ['imb_5fset_5fpointers_5fmb_5fmgr_7',['imb_set_pointers_mb_mgr',['../intel-ipsec-mb_8h.html#a3a825e903d1f8f8611b98b60fcaa5a81',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fauto_8',['init_mb_mgr_auto',['../intel-ipsec-mb_8h.html#a2cbbd032835837fd0aa83b04f5337638',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx_9',['init_mb_mgr_avx',['../intel-ipsec-mb_8h.html#aee547b1b9e70e12c84d9949317e41c6c',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx2_10',['init_mb_mgr_avx2',['../intel-ipsec-mb_8h.html#a71b6563b6b4c0c504cd9b8302a242cfa',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5favx512_11',['init_mb_mgr_avx512',['../intel-ipsec-mb_8h.html#a98932a3c61d798b0c8a560dcc3dce16f',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5fsse_12',['init_mb_mgr_sse',['../intel-ipsec-mb_8h.html#a2ad1941fe5d77b8bb2d6445679e3b0c5',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_5.html b/docs/search/functions_5.html new file mode 100644 index 00000000..431ac7f8 --- /dev/null +++ b/docs/search/functions_5.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_5.js b/docs/search/functions_5.js new file mode 100644 index 00000000..1926a4b0 --- /dev/null +++ b/docs/search/functions_5.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['kasumi_5ff8_5fiv_5fgen_0',['kasumi_f8_iv_gen',['../intel-ipsec-mb_8h.html#a3876dca52c5c210dd0827b2a472e6d0b',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5fiv_5fgen_1',['kasumi_f9_iv_gen',['../intel-ipsec-mb_8h.html#ac886170fc4e70b467383fdf09f0724cf',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_6.html b/docs/search/functions_6.html new file mode 100644 index 00000000..23e61367 --- /dev/null +++ b/docs/search/functions_6.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_6.js b/docs/search/functions_6.js new file mode 100644 index 00000000..e146d871 --- /dev/null +++ b/docs/search/functions_6.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['md5_5fone_5fblock_5favx_0',['md5_one_block_avx',['../intel-ipsec-mb_8h.html#a7addeb4d0c6ddcdaa031b9e7b4dd35c2',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx2_1',['md5_one_block_avx2',['../intel-ipsec-mb_8h.html#a8b912812c65f4d3b592875cdd233704d',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5favx512_2',['md5_one_block_avx512',['../intel-ipsec-mb_8h.html#ac7f4da2fc0601ad50851e7e43f53e0cf',1,'intel-ipsec-mb.h']]], + ['md5_5fone_5fblock_5fsse_3',['md5_one_block_sse',['../intel-ipsec-mb_8h.html#a9d883a3a629240fe86efedd180294433',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_7.html b/docs/search/functions_7.html new file mode 100644 index 00000000..3bd47127 --- /dev/null +++ b/docs/search/functions_7.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_7.js b/docs/search/functions_7.js new file mode 100644 index 00000000..a280dec1 --- /dev/null +++ b/docs/search/functions_7.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['queue_5fsize_5favx_0',['queue_size_avx',['../intel-ipsec-mb_8h.html#a800c395d326e718a2b520d0bb91cbb89',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx2_1',['queue_size_avx2',['../intel-ipsec-mb_8h.html#a8a540000c08366ad47d95e23ed5b2918',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5favx512_2',['queue_size_avx512',['../intel-ipsec-mb_8h.html#a51c0426a4b57c9e7db2b091692dff472',1,'intel-ipsec-mb.h']]], + ['queue_5fsize_5fsse_3',['queue_size_sse',['../intel-ipsec-mb_8h.html#a80f792744558de173ea97d722c58e5c1',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_8.html b/docs/search/functions_8.html new file mode 100644 index 00000000..0374d612 --- /dev/null +++ b/docs/search/functions_8.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_8.js b/docs/search/functions_8.js new file mode 100644 index 00000000..37fa1d7f --- /dev/null +++ b/docs/search/functions_8.js @@ -0,0 +1,53 @@ +var searchData= +[ + ['sha1_5favx_0',['sha1_avx',['../intel-ipsec-mb_8h.html#a9478828dde5161c2a1b43cb0da0f9f67',1,'intel-ipsec-mb.h']]], + ['sha1_5favx2_1',['sha1_avx2',['../intel-ipsec-mb_8h.html#ac13d60c1ddbd3c770f9d85d59821e520',1,'intel-ipsec-mb.h']]], + ['sha1_5favx512_2',['sha1_avx512',['../intel-ipsec-mb_8h.html#ad9b42865aff95b5bcde6efa4b4e83fe8',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5favx_3',['sha1_one_block_avx',['../intel-ipsec-mb_8h.html#a532582e6d066b9c6b1c2189ba1b36f27',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5favx2_4',['sha1_one_block_avx2',['../intel-ipsec-mb_8h.html#af1a63afff082c9d14bbe22f14bd6b5d9',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5favx512_5',['sha1_one_block_avx512',['../intel-ipsec-mb_8h.html#a745c97ce8263ebc67f4d8070f5c9ee50',1,'intel-ipsec-mb.h']]], + ['sha1_5fone_5fblock_5fsse_6',['sha1_one_block_sse',['../intel-ipsec-mb_8h.html#a99f608f81f6b692eaeea1d68cac25a28',1,'intel-ipsec-mb.h']]], + ['sha1_5fsse_7',['sha1_sse',['../intel-ipsec-mb_8h.html#a37491294dc799c5477e3979e89e0cf92',1,'intel-ipsec-mb.h']]], + ['sha224_5favx_8',['sha224_avx',['../intel-ipsec-mb_8h.html#af0e48b520099db5a76190aa5395c90d3',1,'intel-ipsec-mb.h']]], + ['sha224_5favx2_9',['sha224_avx2',['../intel-ipsec-mb_8h.html#ad5945d85d451ccf9b3ba06c06c10d4e9',1,'intel-ipsec-mb.h']]], + ['sha224_5favx512_10',['sha224_avx512',['../intel-ipsec-mb_8h.html#a51c522a828b36d9c69dd7c357b556fc9',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5favx_11',['sha224_one_block_avx',['../intel-ipsec-mb_8h.html#a76d64c2094522011f741de4ac8a03125',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5favx2_12',['sha224_one_block_avx2',['../intel-ipsec-mb_8h.html#a58916afb9e72f42a1c8c2658e248703a',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5favx512_13',['sha224_one_block_avx512',['../intel-ipsec-mb_8h.html#a16f7bd134f4a667f14261a89ca541126',1,'intel-ipsec-mb.h']]], + ['sha224_5fone_5fblock_5fsse_14',['sha224_one_block_sse',['../intel-ipsec-mb_8h.html#a6a15b62259d7475edd02b11b775b7a38',1,'intel-ipsec-mb.h']]], + ['sha224_5fsse_15',['sha224_sse',['../intel-ipsec-mb_8h.html#a2ef2c1513632e69da882a895a075aa2c',1,'intel-ipsec-mb.h']]], + ['sha256_5favx_16',['sha256_avx',['../intel-ipsec-mb_8h.html#a7976461101f011a77d19f4fa480d5be3',1,'intel-ipsec-mb.h']]], + ['sha256_5favx2_17',['sha256_avx2',['../intel-ipsec-mb_8h.html#aec411a6b29d7adff67e6eaa1b13b3975',1,'intel-ipsec-mb.h']]], + ['sha256_5favx512_18',['sha256_avx512',['../intel-ipsec-mb_8h.html#a1020a70f15b89b7146733d8cbc2bcee8',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5favx_19',['sha256_one_block_avx',['../intel-ipsec-mb_8h.html#a2e481e815eb9c8109460e8a8dac62ce9',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5favx2_20',['sha256_one_block_avx2',['../intel-ipsec-mb_8h.html#a0a3c025cfbe9b6e72e9b9f4805f0ca1f',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5favx512_21',['sha256_one_block_avx512',['../intel-ipsec-mb_8h.html#ac2bfb5dca839523118362b48115d7c18',1,'intel-ipsec-mb.h']]], + ['sha256_5fone_5fblock_5fsse_22',['sha256_one_block_sse',['../intel-ipsec-mb_8h.html#ae5ce647166e6f9f8b1d616584ee5d1d6',1,'intel-ipsec-mb.h']]], + ['sha256_5fsse_23',['sha256_sse',['../intel-ipsec-mb_8h.html#a4d9d41b8311d2649ec1b43d6e6edd398',1,'intel-ipsec-mb.h']]], + ['sha384_5favx_24',['sha384_avx',['../intel-ipsec-mb_8h.html#a7b656d085ad03ff43a6c3a4751bfbf56',1,'intel-ipsec-mb.h']]], + ['sha384_5favx2_25',['sha384_avx2',['../intel-ipsec-mb_8h.html#a389b88f838e79306acea6c8bd157c712',1,'intel-ipsec-mb.h']]], + ['sha384_5favx512_26',['sha384_avx512',['../intel-ipsec-mb_8h.html#a908b9e33df6e434c4bb0c74fa20ceece',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5favx_27',['sha384_one_block_avx',['../intel-ipsec-mb_8h.html#aef8af95a47ec01a8c280802a16d0f0c7',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5favx2_28',['sha384_one_block_avx2',['../intel-ipsec-mb_8h.html#ae5cf8bca3e0f1c7c40c110b88168669e',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5favx512_29',['sha384_one_block_avx512',['../intel-ipsec-mb_8h.html#a39fa05d7d02e8911e3d882f379d8661e',1,'intel-ipsec-mb.h']]], + ['sha384_5fone_5fblock_5fsse_30',['sha384_one_block_sse',['../intel-ipsec-mb_8h.html#aa3bf3d1bd2a5d62035c972fe38827549',1,'intel-ipsec-mb.h']]], + ['sha384_5fsse_31',['sha384_sse',['../intel-ipsec-mb_8h.html#ad63e21495f9b798250f5c68371af4f37',1,'intel-ipsec-mb.h']]], + ['sha512_5favx_32',['sha512_avx',['../intel-ipsec-mb_8h.html#a17c4f20de186e1f04580d04d6551e844',1,'intel-ipsec-mb.h']]], + ['sha512_5favx2_33',['sha512_avx2',['../intel-ipsec-mb_8h.html#afc6b16cbe9cf1f840723aa6be3913131',1,'intel-ipsec-mb.h']]], + ['sha512_5favx512_34',['sha512_avx512',['../intel-ipsec-mb_8h.html#a7fa788fbcd353c13fd1eac251f658f1c',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5favx_35',['sha512_one_block_avx',['../intel-ipsec-mb_8h.html#a7d95ff83cbadc815cbe8c11b0af74b7e',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5favx2_36',['sha512_one_block_avx2',['../intel-ipsec-mb_8h.html#ada14329d2331d02241efaafc3ba26ffd',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5favx512_37',['sha512_one_block_avx512',['../intel-ipsec-mb_8h.html#a5d2e35fbfc14d979d87b5b9a63bf19c9',1,'intel-ipsec-mb.h']]], + ['sha512_5fone_5fblock_5fsse_38',['sha512_one_block_sse',['../intel-ipsec-mb_8h.html#aa5e8c4861339d101c718449cf544d5c4',1,'intel-ipsec-mb.h']]], + ['sha512_5fsse_39',['sha512_sse',['../intel-ipsec-mb_8h.html#a392a99d0622122fe0e3c17611ef276a7',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fiv_5fgen_40',['snow3g_f8_iv_gen',['../intel-ipsec-mb_8h.html#a2cbdb64c0b305cb5af010f9c98f70a43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5fiv_5fgen_41',['snow3g_f9_iv_gen',['../intel-ipsec-mb_8h.html#a93e7a99927bc37fb4cb03f966fa4ebdd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx_42',['submit_job_avx',['../intel-ipsec-mb_8h.html#a950bf3b7a3c63a371b561ea20aaf3a18',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx2_43',['submit_job_avx2',['../intel-ipsec-mb_8h.html#a8913995859e45005f8de57a3456c93cf',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5favx512_44',['submit_job_avx512',['../intel-ipsec-mb_8h.html#a75cc0d090470e47cd8be04451ee31a68',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx_45',['submit_job_nocheck_avx',['../intel-ipsec-mb_8h.html#ac7b339426753d3a7c44625a5ded3170a',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx2_46',['submit_job_nocheck_avx2',['../intel-ipsec-mb_8h.html#a72caaf8162ae6f802f7c1fe570afe42e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5favx512_47',['submit_job_nocheck_avx512',['../intel-ipsec-mb_8h.html#a5a95b9268508d68b9027df98167d22cd',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fnocheck_5fsse_48',['submit_job_nocheck_sse',['../intel-ipsec-mb_8h.html#a35e4c8a3d2eba8a612f021572621f98e',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5fsse_49',['submit_job_sse',['../intel-ipsec-mb_8h.html#a7700e90058a74980bfd7d462a19d32a0',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/functions_9.html b/docs/search/functions_9.html new file mode 100644 index 00000000..51225e83 --- /dev/null +++ b/docs/search/functions_9.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/functions_9.js b/docs/search/functions_9.js new file mode 100644 index 00000000..b0c1e891 --- /dev/null +++ b/docs/search/functions_9.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['zuc_5feea3_5fiv_5fgen_0',['zuc_eea3_iv_gen',['../intel-ipsec-mb_8h.html#a4931dd6b1b91456bfe885e28ada0aaf3',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fiv_5fgen_1',['zuc_eia3_iv_gen',['../intel-ipsec-mb_8h.html#ae3cefd0e992f1390110f0ef9a3aa510f',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/mag_sel.png b/docs/search/mag_sel.png new file mode 100644 index 0000000000000000000000000000000000000000..39c0ed52a25dd9d080ee0d42ae6c6042bdfa04d7 GIT binary patch literal 465 zcmeAS@N?(olHy`uVBq!ia0vp^B0wz6!2%?$TA$hhDVB6cUq=Rpjs4tz5?O(Kg=CK) zUj~NU84L`?eGCi_EEpJ?t}-xGu`@87+QPtK?83kxQ`TapwHK(CDaqU2h2ejD|C#+j z9%q3^WHAE+w=f7ZGR&GI0Tg5}@$_|Nf5gMiEhFgvHvB$N=!mC_V~EE2vzPXI9ZnEo zd+1zHor@dYLod2Y{ z@R$7$Z!PXTbY$|@#T!bMzm?`b<(R`cbw(gxJHzu zB$lLFB^RXvDF!10LknF)BV7aY5JN*NBMU1-b8Q0yD+2>vd*|CI8glbfGSez?Ylunu RoetE%;OXk;vd$@?2>>CYplSdB literal 0 HcmV?d00001 diff --git a/docs/search/mag_sel.svg b/docs/search/mag_sel.svg new file mode 100644 index 00000000..03626f64 --- /dev/null +++ b/docs/search/mag_sel.svg @@ -0,0 +1,74 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + diff --git a/docs/search/nomatches.html b/docs/search/nomatches.html new file mode 100644 index 00000000..2b9360b6 --- /dev/null +++ b/docs/search/nomatches.html @@ -0,0 +1,13 @@ + + + + + + + + +
    +
    No Matches
    +
    + + diff --git a/docs/search/pages_0.html b/docs/search/pages_0.html new file mode 100644 index 00000000..14584eba --- /dev/null +++ b/docs/search/pages_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/pages_0.js b/docs/search/pages_0.js new file mode 100644 index 00000000..3a040329 --- /dev/null +++ b/docs/search/pages_0.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['intel_2dipsec_2dmb_0',['intel-ipsec-mb',['../index.html',1,'']]] +]; diff --git a/docs/search/search.css b/docs/search/search.css new file mode 100644 index 00000000..7f54c5c3 --- /dev/null +++ b/docs/search/search.css @@ -0,0 +1,274 @@ +/*---------------- Search Box */ + +#FSearchBox { + float: left; +} + +#MSearchBox { + white-space : nowrap; + float: none; + margin-top: 0px; + width: 161px; + height: 24px; + z-index: 102; + right: 10px; + display: inline; + position: absolute; +} + +#MSearchBox .left +{ + display:block; + position:absolute; + left:10px; + width:20px; + height:19px; + background:url('search_l.png') no-repeat; + background-position:right; +} + +#MSearchSelect { + display:block; + position:absolute; + width:20px; + height:19px; +} + +.left #MSearchSelect { + left:4px; +} + +.right #MSearchSelect { + right:5px; +} + +#MSearchField { + display:block; + position:absolute; + height:19px; + background:url('search_m.png') repeat-x; + border:none; + width:111px; + margin-left:20px; + padding-left:4px; + color: #909090; + outline: none; + font: 9pt Arial, Verdana, sans-serif; + -webkit-border-radius: 0px; +} + +#FSearchBox #MSearchField { + margin-left:15px; +} + +#MSearchBox .right { + display:block; + position:absolute; + left:141px; + width:20px; + height:19px; + background:url('search_r.png') no-repeat; + background-position:left; +} + +#MSearchClose { + display: none; + position: absolute; + top: 4px; + background : none; + border: none; + margin: 0px 4px 0px 0px; + padding: 0px 0px; + outline: none; +} + +.left #MSearchClose { + left: 6px; +} + +.right #MSearchClose { + right: 2px; +} + +.MSearchBoxActive #MSearchField { + color: #000000; +} + +/*---------------- Search filter selection */ + +#MSearchSelectWindow { + display: none; + position: absolute; + left: 0; top: 0; + border: 1px solid #90A5CE; + background-color: #F9FAFC; + z-index: 10001; + padding-top: 4px; + padding-bottom: 4px; + -moz-border-radius: 4px; + -webkit-border-top-left-radius: 4px; + -webkit-border-top-right-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + -webkit-box-shadow: 5px 5px 5px rgba(0, 0, 0, 0.15); +} + +.SelectItem { + font: 8pt Arial, Verdana, sans-serif; + padding-left: 2px; + padding-right: 12px; + border: 0px; +} + +span.SelectionMark { + margin-right: 4px; + font-family: monospace; + outline-style: none; + text-decoration: none; +} + +a.SelectItem { + display: block; + outline-style: none; + color: #000000; + text-decoration: none; + padding-left: 6px; + padding-right: 12px; +} + +a.SelectItem:focus, +a.SelectItem:active { + color: #000000; + outline-style: none; + text-decoration: none; +} + +a.SelectItem:hover { + color: #FFFFFF; + background-color: #3D578C; + outline-style: none; + text-decoration: none; + cursor: pointer; + display: block; +} + +/*---------------- Search results window */ + +iframe#MSearchResults { + /*width: 60ex;*/ + height: 15em; +} + +#MSearchResultsWindow { + display: none; + position: absolute; + left: 0; top: 0; + border: 1px solid #000; + background-color: #EEF1F7; + z-index:10000; +} + +/* ----------------------------------- */ + + +#SRIndex { + clear:both; + padding-bottom: 15px; +} + +.SREntry { + font-size: 10pt; + padding-left: 1ex; +} + +.SRPage .SREntry { + font-size: 8pt; + padding: 1px 5px; +} + +body.SRPage { + margin: 5px 2px; +} + +.SRChildren { + padding-left: 3ex; padding-bottom: .5em +} + +.SRPage .SRChildren { + display: none; +} + +.SRSymbol { + font-weight: bold; + color: #425E97; + font-family: Arial, Verdana, sans-serif; + text-decoration: none; + outline: none; +} + +a.SRScope { + display: block; + color: #425E97; + font-family: Arial, Verdana, sans-serif; + text-decoration: none; + outline: none; +} + +a.SRSymbol:focus, a.SRSymbol:active, +a.SRScope:focus, a.SRScope:active { + text-decoration: underline; +} + +span.SRScope { + padding-left: 4px; + font-family: Arial, Verdana, sans-serif; +} + +.SRPage .SRStatus { + padding: 2px 5px; + font-size: 8pt; + font-style: italic; + font-family: Arial, Verdana, sans-serif; +} + +.SRResult { + display: none; +} + +div.searchresults { + margin-left: 10px; + margin-right: 10px; +} + +/*---------------- External search page results */ + +.searchresult { + background-color: #F0F3F8; +} + +.pages b { + color: white; + padding: 5px 5px 3px 5px; + background-image: url("../tab_a.png"); + background-repeat: repeat-x; + text-shadow: 0 1px 1px #000000; +} + +.pages { + line-height: 17px; + margin-left: 4px; + text-decoration: none; +} + +.hl { + font-weight: bold; +} + +#searchresults { + margin-bottom: 20px; +} + +.searchpages { + margin-top: 10px; +} + diff --git a/docs/search/search.js b/docs/search/search.js new file mode 100644 index 00000000..4c674e0e --- /dev/null +++ b/docs/search/search.js @@ -0,0 +1,793 @@ +/* + @licstart The following is the entire license notice for the JavaScript code in this file. + + The MIT License (MIT) + + Copyright (C) 1997-2020 by Dimitri van Heesch + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, publish, distribute, + sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING + BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + @licend The above is the entire license notice for the JavaScript code in this file + */ +function convertToId(search) +{ + var result = ''; + for (i=0;i do a search + { + this.Search(); + } + } + + this.OnSearchSelectKey = function(evt) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==40 && this.searchIndex0) // Up + { + this.searchIndex--; + this.OnSelectItem(this.searchIndex); + } + else if (e.keyCode==13 || e.keyCode==27) + { + this.OnSelectItem(this.searchIndex); + this.CloseSelectionWindow(); + this.DOMSearchField().focus(); + } + return false; + } + + // --------- Actions + + // Closes the results window. + this.CloseResultsWindow = function() + { + this.DOMPopupSearchResultsWindow().style.display = 'none'; + this.DOMSearchClose().style.display = 'none'; + this.Activate(false); + } + + this.CloseSelectionWindow = function() + { + this.DOMSearchSelectWindow().style.display = 'none'; + } + + // Performs a search. + this.Search = function() + { + this.keyTimeout = 0; + + // strip leading whitespace + var searchValue = this.DOMSearchField().value.replace(/^ +/, ""); + + var code = searchValue.toLowerCase().charCodeAt(0); + var idxChar = searchValue.substr(0, 1).toLowerCase(); + if ( 0xD800 <= code && code <= 0xDBFF && searchValue > 1) // surrogate pair + { + idxChar = searchValue.substr(0, 2); + } + + var resultsPage; + var resultsPageWithSearch; + var hasResultsPage; + + var idx = indexSectionsWithContent[this.searchIndex].indexOf(idxChar); + if (idx!=-1) + { + var hexCode=idx.toString(16); + resultsPage = this.resultsPath + '/' + indexSectionNames[this.searchIndex] + '_' + hexCode + this.extension; + resultsPageWithSearch = resultsPage+'?'+escape(searchValue); + hasResultsPage = true; + } + else // nothing available for this search term + { + resultsPage = this.resultsPath + '/nomatches' + this.extension; + resultsPageWithSearch = resultsPage; + hasResultsPage = false; + } + + window.frames.MSearchResults.location = resultsPageWithSearch; + var domPopupSearchResultsWindow = this.DOMPopupSearchResultsWindow(); + + if (domPopupSearchResultsWindow.style.display!='block') + { + var domSearchBox = this.DOMSearchBox(); + this.DOMSearchClose().style.display = 'inline-block'; + var domPopupSearchResults = this.DOMPopupSearchResults(); + var left = getXPos(domSearchBox) + 150; // domSearchBox.offsetWidth; + var top = getYPos(domSearchBox) + 20; // domSearchBox.offsetHeight + 1; + domPopupSearchResultsWindow.style.display = 'block'; + left -= domPopupSearchResults.offsetWidth; + var maxWidth = document.body.clientWidth; + var width = 400; + if (left<10) left=10; + if (width+left+8>maxWidth) width=maxWidth-left-8; + domPopupSearchResultsWindow.style.top = top + 'px'; + domPopupSearchResultsWindow.style.left = left + 'px'; + domPopupSearchResultsWindow.style.width = width + 'px'; + } + + this.lastSearchValue = searchValue; + this.lastResultsPage = resultsPage; + } + + // -------- Activation Functions + + // Activates or deactivates the search panel, resetting things to + // their default values if necessary. + this.Activate = function(isActive) + { + if (isActive || // open it + this.DOMPopupSearchResultsWindow().style.display == 'block' + ) + { + this.DOMSearchBox().className = 'MSearchBoxActive'; + this.searchActive = true; + } + else if (!isActive) // directly remove the panel + { + this.DOMSearchBox().className = 'MSearchBoxInactive'; + this.searchActive = false; + this.lastSearchValue = '' + this.lastResultsPage = ''; + } + } +} + +// ----------------------------------------------------------------------- + +// The class that handles everything on the search results page. +function SearchResults(name) +{ + // The number of matches from the last run of . + this.lastMatchCount = 0; + this.lastKey = 0; + this.repeatOn = false; + + // Toggles the visibility of the passed element ID. + this.FindChildElement = function(id) + { + var parentElement = document.getElementById(id); + var element = parentElement.firstChild; + + while (element && element!=parentElement) + { + if (element.nodeName.toLowerCase() == 'div' && element.className == 'SRChildren') + { + return element; + } + + if (element.nodeName.toLowerCase() == 'div' && element.hasChildNodes()) + { + element = element.firstChild; + } + else if (element.nextSibling) + { + element = element.nextSibling; + } + else + { + do + { + element = element.parentNode; + } + while (element && element!=parentElement && !element.nextSibling); + + if (element && element!=parentElement) + { + element = element.nextSibling; + } + } + } + } + + this.Toggle = function(id) + { + var element = this.FindChildElement(id); + if (element) + { + if (element.style.display == 'block') + { + element.style.display = 'none'; + } + else + { + element.style.display = 'block'; + } + } + } + + // Searches for the passed string. If there is no parameter, + // it takes it from the URL query. + // + // Always returns true, since other documents may try to call it + // and that may or may not be possible. + this.Search = function(search) + { + if (!search) // get search word from URL + { + search = window.location.search; + search = search.substring(1); // Remove the leading '?' + search = unescape(search); + } + + search = search.replace(/^ +/, ""); // strip leading spaces + search = search.replace(/ +$/, ""); // strip trailing spaces + search = search.toLowerCase(); + search = convertToId(search); + + var resultRows = document.getElementsByTagName("div"); + var matches = 0; + + var i = 0; + while (i < resultRows.length) + { + var row = resultRows.item(i); + if (row.className == "SRResult") + { + var rowMatchName = row.id.toLowerCase(); + rowMatchName = rowMatchName.replace(/^sr\d*_/, ''); // strip 'sr123_' + + if (search.length<=rowMatchName.length && + rowMatchName.substr(0, search.length)==search) + { + row.style.display = 'block'; + matches++; + } + else + { + row.style.display = 'none'; + } + } + i++; + } + document.getElementById("Searching").style.display='none'; + if (matches == 0) // no results + { + document.getElementById("NoMatches").style.display='block'; + } + else // at least one result + { + document.getElementById("NoMatches").style.display='none'; + } + this.lastMatchCount = matches; + return true; + } + + // return the first item with index index or higher that is visible + this.NavNext = function(index) + { + var focusItem; + while (1) + { + var focusName = 'Item'+index; + focusItem = document.getElementById(focusName); + if (focusItem && focusItem.parentNode.parentNode.style.display=='block') + { + break; + } + else if (!focusItem) // last element + { + break; + } + focusItem=null; + index++; + } + return focusItem; + } + + this.NavPrev = function(index) + { + var focusItem; + while (1) + { + var focusName = 'Item'+index; + focusItem = document.getElementById(focusName); + if (focusItem && focusItem.parentNode.parentNode.style.display=='block') + { + break; + } + else if (!focusItem) // last element + { + break; + } + focusItem=null; + index--; + } + return focusItem; + } + + this.ProcessKeys = function(e) + { + if (e.type == "keydown") + { + this.repeatOn = false; + this.lastKey = e.keyCode; + } + else if (e.type == "keypress") + { + if (!this.repeatOn) + { + if (this.lastKey) this.repeatOn = true; + return false; // ignore first keypress after keydown + } + } + else if (e.type == "keyup") + { + this.lastKey = 0; + this.repeatOn = false; + } + return this.lastKey!=0; + } + + this.Nav = function(evt,itemIndex) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==13) return true; + if (!this.ProcessKeys(e)) return false; + + if (this.lastKey==38) // Up + { + var newIndex = itemIndex-1; + var focusItem = this.NavPrev(newIndex); + if (focusItem) + { + var child = this.FindChildElement(focusItem.parentNode.parentNode.id); + if (child && child.style.display == 'block') // children visible + { + var n=0; + var tmpElem; + while (1) // search for last child + { + tmpElem = document.getElementById('Item'+newIndex+'_c'+n); + if (tmpElem) + { + focusItem = tmpElem; + } + else // found it! + { + break; + } + n++; + } + } + } + if (focusItem) + { + focusItem.focus(); + } + else // return focus to search field + { + parent.document.getElementById("MSearchField").focus(); + } + } + else if (this.lastKey==40) // Down + { + var newIndex = itemIndex+1; + var focusItem; + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem && elem.style.display == 'block') // children visible + { + focusItem = document.getElementById('Item'+itemIndex+'_c0'); + } + if (!focusItem) focusItem = this.NavNext(newIndex); + if (focusItem) focusItem.focus(); + } + else if (this.lastKey==39) // Right + { + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem) elem.style.display = 'block'; + } + else if (this.lastKey==37) // Left + { + var item = document.getElementById('Item'+itemIndex); + var elem = this.FindChildElement(item.parentNode.parentNode.id); + if (elem) elem.style.display = 'none'; + } + else if (this.lastKey==27) // Escape + { + parent.searchBox.CloseResultsWindow(); + parent.document.getElementById("MSearchField").focus(); + } + else if (this.lastKey==13) // Enter + { + return true; + } + return false; + } + + this.NavChild = function(evt,itemIndex,childIndex) + { + var e = (evt) ? evt : window.event; // for IE + if (e.keyCode==13) return true; + if (!this.ProcessKeys(e)) return false; + + if (this.lastKey==38) // Up + { + if (childIndex>0) + { + var newIndex = childIndex-1; + document.getElementById('Item'+itemIndex+'_c'+newIndex).focus(); + } + else // already at first child, jump to parent + { + document.getElementById('Item'+itemIndex).focus(); + } + } + else if (this.lastKey==40) // Down + { + var newIndex = childIndex+1; + var elem = document.getElementById('Item'+itemIndex+'_c'+newIndex); + if (!elem) // last child, jump to parent next parent + { + elem = this.NavNext(itemIndex+1); + } + if (elem) + { + elem.focus(); + } + } + else if (this.lastKey==27) // Escape + { + parent.searchBox.CloseResultsWindow(); + parent.document.getElementById("MSearchField").focus(); + } + else if (this.lastKey==13) // Enter + { + return true; + } + return false; + } +} + +function setKeyActions(elem,action) +{ + elem.setAttribute('onkeydown',action); + elem.setAttribute('onkeypress',action); + elem.setAttribute('onkeyup',action); +} + +function setClassAttr(elem,attr) +{ + elem.setAttribute('class',attr); + elem.setAttribute('className',attr); +} + +function createResults() +{ + var results = document.getElementById("SRResults"); + for (var e=0; e(R!W8j_r#qQ#gnr4kAxdU#F0+OBry$Z+ z_0PMi;P|#{d%mw(dnw=jM%@$onTJa%@6Nm3`;2S#nwtVFJI#`U@2Q@@JCCctagvF- z8H=anvo~dTmJ2YA%wA6IHRv%{vxvUm|R)kgZeo zmX%Zb;mpflGZdXCTAgit`||AFzkI#z&(3d4(htA?U2FOL4WF6wY&TB#n3n*I4+hl| z*NBpo#FA92vEu822WQ%mvv4FO#qs` BFGc_W literal 0 HcmV?d00001 diff --git a/docs/search/search_r.png b/docs/search/search_r.png new file mode 100644 index 0000000000000000000000000000000000000000..1af5d21ee13e070d7600f1c4657fde843b953a69 GIT binary patch literal 553 zcmeAS@N?(olHy`uVBq!ia0vp^LO?9c!2%@BXHTsJQY`6?zK#qG8~eHcB(ehe3dtTp zz6=bxGZ+|(`xqD=STHa&U1eaXVrO7DwS|Gf*oA>XrmV$GYcEhOQT(QLuS{~ooZ2P@v=Xc@RKW@Irliv8_;wroU0*)0O?temdsA~70jrdux+`@W7 z-N(<(C)L?hOO?KV{>8(jC{hpKsws)#Fh zvsO>IB+gb@b+rGWaO&!a9Z{!U+fV*s7TS>fdt&j$L%^U@Epd$~Nl7e8wMs5Z1yT$~ z28I^8hDN#u<{^fLRz?<9hUVG^237_Jy7tbuQ8eV{r(~v8;?@w8^gA7>fx*+&&t;uc GLK6VEQpiUD literal 0 HcmV?d00001 diff --git a/docs/search/searchdata.js b/docs/search/searchdata.js new file mode 100644 index 00000000..2769faa3 --- /dev/null +++ b/docs/search/searchdata.js @@ -0,0 +1,42 @@ +var indexSectionsWithContent = +{ + 0: "_acdefghijklmnopqrsuvxz", + 1: "cgiks", + 2: "ir", + 3: "adfgikmqsz", + 4: "_acdefghijklmnopqrsuvxz", + 5: "acdfghikqsxz", + 6: "i", + 7: "i", + 8: "_dik", + 9: "i" +}; + +var indexSectionNames = +{ + 0: "all", + 1: "classes", + 2: "files", + 3: "functions", + 4: "variables", + 5: "typedefs", + 6: "enums", + 7: "enumvalues", + 8: "defines", + 9: "pages" +}; + +var indexSectionLabels = +{ + 0: "All", + 1: "Data Structures", + 2: "Files", + 3: "Functions", + 4: "Variables", + 5: "Typedefs", + 6: "Enumerations", + 7: "Enumerator", + 8: "Macros", + 9: "Pages" +}; + diff --git a/docs/search/typedefs_0.html b/docs/search/typedefs_0.html new file mode 100644 index 00000000..83d96e48 --- /dev/null +++ b/docs/search/typedefs_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_0.js b/docs/search/typedefs_0.js new file mode 100644 index 00000000..be4fc8b6 --- /dev/null +++ b/docs/search/typedefs_0.js @@ -0,0 +1,15 @@ +var searchData= +[ + ['aes_5fcfb_5ft_0',['aes_cfb_t',['../intel-ipsec-mb_8h.html#aeb837fda81cdf56ffcfa4e68b5877a35',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ffinalize_5ft_1',['aes_gcm_enc_dec_finalize_t',['../intel-ipsec-mb_8h.html#a8c85430db7a8b695a2836dcb96653bc5',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fiv_5ft_2',['aes_gcm_enc_dec_iv_t',['../intel-ipsec-mb_8h.html#ac5bfb31d18fdc55451cd325c4af032c7',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5ft_3',['aes_gcm_enc_dec_t',['../intel-ipsec-mb_8h.html#a71fb9f25bcab533a98f3d1c5cc2c7f48',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fenc_5fdec_5fupdate_5ft_4',['aes_gcm_enc_dec_update_t',['../intel-ipsec-mb_8h.html#af701adcd98fd95084d76e0cb0f926fef',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5ft_5',['aes_gcm_init_t',['../intel-ipsec-mb_8h.html#adeecd4bac067493d8465e8d7b6b7dd6d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5finit_5fvar_5fiv_5ft_6',['aes_gcm_init_var_iv_t',['../intel-ipsec-mb_8h.html#a65f909001359ad986640155f8756a11d',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fpre_5ft_7',['aes_gcm_pre_t',['../intel-ipsec-mb_8h.html#a11e10a3ac7a08801dc68949fb0f428af',1,'intel-ipsec-mb.h']]], + ['aes_5fgcm_5fprecomp_5ft_8',['aes_gcm_precomp_t',['../intel-ipsec-mb_8h.html#ab50906d68121cb8e6edf2a3493a60c5c',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5ffinalize_5ft_9',['aes_gmac_finalize_t',['../intel-ipsec-mb_8h.html#af3f75897a0ec8b3f22c4e2b01f7f48b7',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5finit_5ft_10',['aes_gmac_init_t',['../intel-ipsec-mb_8h.html#a31987251c968435d720e8656ae4621a0',1,'intel-ipsec-mb.h']]], + ['aes_5fgmac_5fupdate_5ft_11',['aes_gmac_update_t',['../intel-ipsec-mb_8h.html#aba0ac59064802c6d0a182a5164568c18',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_1.html b/docs/search/typedefs_1.html new file mode 100644 index 00000000..a252bc79 --- /dev/null +++ b/docs/search/typedefs_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_1.js b/docs/search/typedefs_1.js new file mode 100644 index 00000000..1a0f289d --- /dev/null +++ b/docs/search/typedefs_1.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['chacha_5fpoly_5fenc_5fdec_5fupdate_5ft_0',['chacha_poly_enc_dec_update_t',['../intel-ipsec-mb_8h.html#a437de868b8dba4f9b878d99a82f80548',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5ffinalize_5ft_1',['chacha_poly_finalize_t',['../intel-ipsec-mb_8h.html#a1a96f063bf1bd07de158985b57cff4a0',1,'intel-ipsec-mb.h']]], + ['chacha_5fpoly_5finit_5ft_2',['chacha_poly_init_t',['../intel-ipsec-mb_8h.html#a4b0b9ffa331f8d675d24911639f0374f',1,'intel-ipsec-mb.h']]], + ['cmac_5fsubkey_5fgen_5ft_3',['cmac_subkey_gen_t',['../intel-ipsec-mb_8h.html#aab07996d622d682adc1db8286e633cb3',1,'intel-ipsec-mb.h']]], + ['crc32_5ffn_5ft_4',['crc32_fn_t',['../intel-ipsec-mb_8h.html#a56dd197c301ad27dc9ec0862721cb73d',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_2.html b/docs/search/typedefs_2.html new file mode 100644 index 00000000..a88f0d5f --- /dev/null +++ b/docs/search/typedefs_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_2.js b/docs/search/typedefs_2.js new file mode 100644 index 00000000..2a780d05 --- /dev/null +++ b/docs/search/typedefs_2.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['des_5fkeysched_5ft_0',['des_keysched_t',['../intel-ipsec-mb_8h.html#a60a93569304a978c798bb8c4a4f7f52f',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_3.html b/docs/search/typedefs_3.html new file mode 100644 index 00000000..32c8c2d9 --- /dev/null +++ b/docs/search/typedefs_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_3.js b/docs/search/typedefs_3.js new file mode 100644 index 00000000..13fff02d --- /dev/null +++ b/docs/search/typedefs_3.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['flush_5fjob_5ft_0',['flush_job_t',['../intel-ipsec-mb_8h.html#ae64d0f677e42b8f8d50289113cf5e01e',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_4.html b/docs/search/typedefs_4.html new file mode 100644 index 00000000..4af58694 --- /dev/null +++ b/docs/search/typedefs_4.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_4.js b/docs/search/typedefs_4.js new file mode 100644 index 00000000..a07b3f26 --- /dev/null +++ b/docs/search/typedefs_4.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['get_5fcompleted_5fjob_5ft_0',['get_completed_job_t',['../intel-ipsec-mb_8h.html#ac8cb9a545cdef14f60f36fe8888e6e7c',1,'intel-ipsec-mb.h']]], + ['get_5fnext_5fjob_5ft_1',['get_next_job_t',['../intel-ipsec-mb_8h.html#a5a533d852e3b65e9e5978fec37da43b5',1,'intel-ipsec-mb.h']]], + ['ghash_5ft_2',['ghash_t',['../intel-ipsec-mb_8h.html#a12e0c05f0ecec8d75d4b288a7ec11c84',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_5.html b/docs/search/typedefs_5.html new file mode 100644 index 00000000..6b3e1980 --- /dev/null +++ b/docs/search/typedefs_5.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_5.js b/docs/search/typedefs_5.js new file mode 100644 index 00000000..0b7f484d --- /dev/null +++ b/docs/search/typedefs_5.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['hash_5ffn_5ft_0',['hash_fn_t',['../intel-ipsec-mb_8h.html#afacacc8b74c05c80a0ee255c68637490',1,'intel-ipsec-mb.h']]], + ['hash_5fone_5fblock_5ft_1',['hash_one_block_t',['../intel-ipsec-mb_8h.html#aade58eb541a45650ed7d875dc2be97fa',1,'intel-ipsec-mb.h']]], + ['hec_5f32_5ft_2',['hec_32_t',['../intel-ipsec-mb_8h.html#aa34b11bf4be97c764b283aedc0589575',1,'intel-ipsec-mb.h']]], + ['hec_5f64_5ft_3',['hec_64_t',['../intel-ipsec-mb_8h.html#aa9afd4ee91f0d82729afcf5f1cdcb808',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_6.html b/docs/search/typedefs_6.html new file mode 100644 index 00000000..1b776917 --- /dev/null +++ b/docs/search/typedefs_6.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_6.js b/docs/search/typedefs_6.js new file mode 100644 index 00000000..5b71a92d --- /dev/null +++ b/docs/search/typedefs_6.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['imb_5fjob_0',['IMB_JOB',['../intel-ipsec-mb_8h.html#aed2387ba36d72c3d148fcb9d2802ccf5',1,'intel-ipsec-mb.h']]], + ['imb_5fmgr_1',['IMB_MGR',['../intel-ipsec-mb_8h.html#a6d466496b4adfea3f9bc0881f11fe551',1,'intel-ipsec-mb.h']]], + ['init_5fmb_5fmgr_5ft_2',['init_mb_mgr_t',['../intel-ipsec-mb_8h.html#af0c38ca0a5aac81d8db7e0cc2811426b',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_7.html b/docs/search/typedefs_7.html new file mode 100644 index 00000000..3482be4f --- /dev/null +++ b/docs/search/typedefs_7.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_7.js b/docs/search/typedefs_7.js new file mode 100644 index 00000000..f475d1fc --- /dev/null +++ b/docs/search/typedefs_7.js @@ -0,0 +1,16 @@ +var searchData= +[ + ['kasumi_5ff8_5f1_5fbuffer_5fbit_5ft_0',['kasumi_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a81afa744fb5669fe37cf0dce2193d8a7',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f1_5fbuffer_5ft_1',['kasumi_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a89fe975f0388b71765be494d04b034f4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f2_5fbuffer_5ft_2',['kasumi_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#af1eb074f5da469fa655d1404be8701dd',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f3_5fbuffer_5ft_3',['kasumi_f8_3_buffer_t',['../intel-ipsec-mb_8h.html#a53ecac7e009fcbeee603f975a45d3012',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5f4_5fbuffer_5ft_4',['kasumi_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#accf08a616e22a2c56e6a37d56f38b354',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff8_5fn_5fbuffer_5ft_5',['kasumi_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a51bbde186cdd437b6f9448e19a58e8a1',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5ft_6',['kasumi_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#a8b02114f003b5fe0e1ccbfdc13ac55e4',1,'intel-ipsec-mb.h']]], + ['kasumi_5ff9_5f1_5fbuffer_5fuser_5ft_7',['kasumi_f9_1_buffer_user_t',['../intel-ipsec-mb_8h.html#ac17147d04602964bea59322ff31ebf59',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_5ft_8',['kasumi_init_f8_key_sched_t',['../intel-ipsec-mb_8h.html#a7bec62666456d8962676961cf187ac80',1,'intel-ipsec-mb.h']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_5ft_9',['kasumi_init_f9_key_sched_t',['../intel-ipsec-mb_8h.html#a8b30db8462aca1c039c3fd20646f342c',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5fsize_5ft_10',['kasumi_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad7444c36024a63025e7682c16946a00a',1,'intel-ipsec-mb.h']]], + ['kasumi_5fkey_5fsched_5ft_11',['kasumi_key_sched_t',['../intel-ipsec-mb_8h.html#a82d6866ebcf1bd46b684fe473b4b73f2',1,'intel-ipsec-mb.h']]], + ['keyexp_5ft_12',['keyexp_t',['../intel-ipsec-mb_8h.html#a60de89a391c8d6d9c51c742a87effb6f',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_8.html b/docs/search/typedefs_8.html new file mode 100644 index 00000000..bae717dd --- /dev/null +++ b/docs/search/typedefs_8.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_8.js b/docs/search/typedefs_8.js new file mode 100644 index 00000000..e6432589 --- /dev/null +++ b/docs/search/typedefs_8.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['queue_5fsize_5ft_0',['queue_size_t',['../intel-ipsec-mb_8h.html#a9884ae92b598dd504f7d0ef8be5051d6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_9.html b/docs/search/typedefs_9.html new file mode 100644 index 00000000..9e84f8ea --- /dev/null +++ b/docs/search/typedefs_9.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_9.js b/docs/search/typedefs_9.js new file mode 100644 index 00000000..14176751 --- /dev/null +++ b/docs/search/typedefs_9.js @@ -0,0 +1,19 @@ +var searchData= +[ + ['snow3g_5ff8_5f1_5fbuffer_5fbit_5ft_0',['snow3g_f8_1_buffer_bit_t',['../intel-ipsec-mb_8h.html#a7bd00b31d12260305accf0d37b60c958',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f1_5fbuffer_5ft_1',['snow3g_f8_1_buffer_t',['../intel-ipsec-mb_8h.html#a281315fb8b441d837ebed541b3a23380',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f2_5fbuffer_5ft_2',['snow3g_f8_2_buffer_t',['../intel-ipsec-mb_8h.html#a4e8912e7e55a2ac891f85c19e7f29328',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f4_5fbuffer_5ft_3',['snow3g_f8_4_buffer_t',['../intel-ipsec-mb_8h.html#ae13b1154dcc13ca562e81471659d6099',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_5ft_4',['snow3g_f8_8_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a109b2a72d05bbe05a373ed5cf3c1f201',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5f8_5fbuffer_5ft_5',['snow3g_f8_8_buffer_t',['../intel-ipsec-mb_8h.html#a6155f472c6794e88289235752d83154a',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_5ft_6',['snow3g_f8_n_buffer_multikey_t',['../intel-ipsec-mb_8h.html#a4049b601f9da6542e25d903efe524bc1',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff8_5fn_5fbuffer_5ft_7',['snow3g_f8_n_buffer_t',['../intel-ipsec-mb_8h.html#a6a4a1da950207c9ed04f3f18ae1b7e43',1,'intel-ipsec-mb.h']]], + ['snow3g_5ff9_5f1_5fbuffer_5ft_8',['snow3g_f9_1_buffer_t',['../intel-ipsec-mb_8h.html#ae7a48386f7190b4e46cd113eee413733',1,'intel-ipsec-mb.h']]], + ['snow3g_5finit_5fkey_5fsched_5ft_9',['snow3g_init_key_sched_t',['../intel-ipsec-mb_8h.html#adfa182361f1c179d75129b4e46841af5',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fsched_5fsize_5ft_10',['snow3g_key_sched_size_t',['../intel-ipsec-mb_8h.html#ad78a4f1c0d6de808f7046c0fbcd98d8f',1,'intel-ipsec-mb.h']]], + ['snow3g_5fkey_5fschedule_5ft_11',['snow3g_key_schedule_t',['../intel-ipsec-mb_8h.html#ae50b48832b41b4b468cf933b44849213',1,'intel-ipsec-mb.h']]], + ['submit_5fburst_5ft_12',['submit_burst_t',['../intel-ipsec-mb_8h.html#a0cf6acb86385bc984ec5f916fe6e1066',1,'intel-ipsec-mb.h']]], + ['submit_5fcipher_5fburst_5ft_13',['submit_cipher_burst_t',['../intel-ipsec-mb_8h.html#ac4bd89ab3cb1341eb4f30c4c501a9009',1,'intel-ipsec-mb.h']]], + ['submit_5fhash_5fburst_5ft_14',['submit_hash_burst_t',['../intel-ipsec-mb_8h.html#a4f6fca648737cec9ee869786e9c240b5',1,'intel-ipsec-mb.h']]], + ['submit_5fjob_5ft_15',['submit_job_t',['../intel-ipsec-mb_8h.html#a41297d789b434157e746e338f0af96ec',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_a.html b/docs/search/typedefs_a.html new file mode 100644 index 00000000..7feb897e --- /dev/null +++ b/docs/search/typedefs_a.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_a.js b/docs/search/typedefs_a.js new file mode 100644 index 00000000..2626c381 --- /dev/null +++ b/docs/search/typedefs_a.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['xcbc_5fkeyexp_5ft_0',['xcbc_keyexp_t',['../intel-ipsec-mb_8h.html#abb36f71b9fb4d928c2ad9fa462b80ae6',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/typedefs_b.html b/docs/search/typedefs_b.html new file mode 100644 index 00000000..1a68048f --- /dev/null +++ b/docs/search/typedefs_b.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/typedefs_b.js b/docs/search/typedefs_b.js new file mode 100644 index 00000000..958c4594 --- /dev/null +++ b/docs/search/typedefs_b.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['zuc_5feea3_5f1_5fbuffer_5ft_0',['zuc_eea3_1_buffer_t',['../intel-ipsec-mb_8h.html#a0a3032f0698c42bd486d053e90c44fdd',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5f4_5fbuffer_5ft_1',['zuc_eea3_4_buffer_t',['../intel-ipsec-mb_8h.html#a7df2e3f063333d677ef7668626126c4d',1,'intel-ipsec-mb.h']]], + ['zuc_5feea3_5fn_5fbuffer_5ft_2',['zuc_eea3_n_buffer_t',['../intel-ipsec-mb_8h.html#a303f2dd11b1193938596b4ec69edf8cc',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5f1_5fbuffer_5ft_3',['zuc_eia3_1_buffer_t',['../intel-ipsec-mb_8h.html#a95ed5580686f708ba2313179cd6064a8',1,'intel-ipsec-mb.h']]], + ['zuc_5feia3_5fn_5fbuffer_5ft_4',['zuc_eia3_n_buffer_t',['../intel-ipsec-mb_8h.html#ada45082bb04dd727c0deec5c4fc96fed',1,'intel-ipsec-mb.h']]] +]; diff --git a/docs/search/variables_0.html b/docs/search/variables_0.html new file mode 100644 index 00000000..cc10fc25 --- /dev/null +++ b/docs/search/variables_0.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_0.js b/docs/search/variables_0.js new file mode 100644 index 00000000..b571fe4d --- /dev/null +++ b/docs/search/variables_0.js @@ -0,0 +1,15 @@ +var searchData= +[ + ['_5fhashed_5fauth_5fkey_5fxor_5fipad_0',['_hashed_auth_key_xor_ipad',['../structIMB__JOB.html#a2e74bbeeed2c70f97e6b30a0a12aec0e',1,'IMB_JOB']]], + ['_5fhashed_5fauth_5fkey_5fxor_5fopad_1',['_hashed_auth_key_xor_opad',['../structIMB__JOB.html#a684b751d0a275a28d345a3d1051204ef',1,'IMB_JOB']]], + ['_5finit_5ftag_2',['_init_tag',['../structIMB__JOB.html#a1e2fc881ce9c84173168df1dbda851f8',1,'IMB_JOB']]], + ['_5fiv_3',['_iv',['../structIMB__JOB.html#ae571147592c332cea0d90a36e133e6a5',1,'IMB_JOB::_iv()'],['../structIMB__JOB.html#a4415e6851e770734c5ec1b46a828ce55',1,'IMB_JOB::_iv()']]], + ['_5fiv23_4',['_iv23',['../structIMB__JOB.html#a2bd093b1609dad73bbef2a728995d12f',1,'IMB_JOB']]], + ['_5fk1_5fexpanded_5',['_k1_expanded',['../structIMB__JOB.html#a76d55a988c71e40c577c6f8f5d0496de',1,'IMB_JOB']]], + ['_5fk2_6',['_k2',['../structIMB__JOB.html#adc47d4af15c88dcb7abf01bb07e79e63',1,'IMB_JOB']]], + ['_5fk3_7',['_k3',['../structIMB__JOB.html#afa62af4d5f1ecbb80f5216ec298fcfe8',1,'IMB_JOB']]], + ['_5fkey_8',['_key',['../structIMB__JOB.html#a7ba1fdbcc6ed2e3840990d2e34dac8e8',1,'IMB_JOB::_key()'],['../structIMB__JOB.html#a7a62dc4b2ef35231a93ce035561c4153',1,'IMB_JOB::_key()'],['../structIMB__JOB.html#a3b3f616759b854553a351e6b44e08471',1,'IMB_JOB::_key()']]], + ['_5fkey_5fexpanded_9',['_key_expanded',['../structIMB__JOB.html#a95cc77e0c079f43dec1a46a2b75422a0',1,'IMB_JOB']]], + ['_5fskey1_10',['_skey1',['../structIMB__JOB.html#ab64e8f6bf8dfaf56305b9023103ea5e2',1,'IMB_JOB']]], + ['_5fskey2_11',['_skey2',['../structIMB__JOB.html#a8e8143e461eab3b2b04333c5f1c17a82',1,'IMB_JOB']]] +]; diff --git a/docs/search/variables_1.html b/docs/search/variables_1.html new file mode 100644 index 00000000..360f8fe4 --- /dev/null +++ b/docs/search/variables_1.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_1.js b/docs/search/variables_1.js new file mode 100644 index 00000000..b29834dc --- /dev/null +++ b/docs/search/variables_1.js @@ -0,0 +1,21 @@ +var searchData= +[ + ['aad_0',['aad',['../structIMB__JOB.html#a4c943fcac7f5c4f5d6d030547840071e',1,'IMB_JOB']]], + ['aad_5fhash_1',['aad_hash',['../structgcm__context__data.html#a98c22f35eb0b4d368e2f454b347ed3b9',1,'gcm_context_data']]], + ['aad_5flen_2',['aad_len',['../structchacha20__poly1305__context__data.html#a7b5edcebc9a6bec3776daf6d13bf950c',1,'chacha20_poly1305_context_data']]], + ['aad_5flen_5fin_5fbytes_3',['aad_len_in_bytes',['../structIMB__JOB.html#aa607a780bc2c8509270625c7520a11f1',1,'IMB_JOB']]], + ['aad_5flength_4',['aad_length',['../structgcm__context__data.html#ab1dadabcbd2002738ec7db7da332e04f',1,'gcm_context_data']]], + ['aes128_5fcbcs_5fooo_5',['aes128_cbcs_ooo',['../structIMB__MGR.html#a9c72f3341a9a4837595338d06c3eda5c',1,'IMB_MGR']]], + ['aes128_5fcfb_5fone_6',['aes128_cfb_one',['../structIMB__MGR.html#abd468de03563a40da0b21b0885f44cbe',1,'IMB_MGR']]], + ['aes128_5fooo_7',['aes128_ooo',['../structIMB__MGR.html#a97fd05a4057fc898f647955a7ba8501b',1,'IMB_MGR']]], + ['aes192_5fooo_8',['aes192_ooo',['../structIMB__MGR.html#afe48187946dff59f6c69103501691921',1,'IMB_MGR']]], + ['aes256_5fccm_5fooo_9',['aes256_ccm_ooo',['../structIMB__MGR.html#a04c28bfe763544b68dac98e23aa0a8d9',1,'IMB_MGR']]], + ['aes256_5fcmac_5fooo_10',['aes256_cmac_ooo',['../structIMB__MGR.html#acbd18bab8176262daab2bd808ca30616',1,'IMB_MGR']]], + ['aes256_5fooo_11',['aes256_ooo',['../structIMB__MGR.html#a6abbf129001cb44003a10105115e52b7',1,'IMB_MGR']]], + ['aes_5fccm_5fooo_12',['aes_ccm_ooo',['../structIMB__MGR.html#a3eeeead99056e46d8cb1ed316f587aaf',1,'IMB_MGR']]], + ['aes_5fcmac_5fooo_13',['aes_cmac_ooo',['../structIMB__MGR.html#a0bd2acac3a6c3f90a998ed53a7d28e57',1,'IMB_MGR']]], + ['aes_5fxcbc_5fooo_14',['aes_xcbc_ooo',['../structIMB__MGR.html#ad554a25793a904ae6cf7ca87250adbd3',1,'IMB_MGR']]], + ['auth_5ftag_5foutput_15',['auth_tag_output',['../structIMB__JOB.html#ac37080ddb0baf56c9589e65960430b32',1,'IMB_JOB']]], + ['auth_5ftag_5foutput_5flen_5fin_5fbytes_16',['auth_tag_output_len_in_bytes',['../structIMB__JOB.html#a750df4fa7330e2387138409d0eb50e0f',1,'IMB_JOB']]], + ['avx2_5favx512_17',['avx2_avx512',['../structgcm__key__data.html#a3f231fdc1d521cd15c111667e4aea818',1,'gcm_key_data']]] +]; diff --git a/docs/search/variables_10.html b/docs/search/variables_10.html new file mode 100644 index 00000000..a90f1bc7 --- /dev/null +++ b/docs/search/variables_10.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_10.js b/docs/search/variables_10.js new file mode 100644 index 00000000..015406a3 --- /dev/null +++ b/docs/search/variables_10.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['queue_5fsize_0',['queue_size',['../structIMB__MGR.html#a5a35c9c2a1c8eb5b4aa239a3b04f4fbf',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_11.html b/docs/search/variables_11.html new file mode 100644 index 00000000..7691db3e --- /dev/null +++ b/docs/search/variables_11.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_11.js b/docs/search/variables_11.js new file mode 100644 index 00000000..0e7759b9 --- /dev/null +++ b/docs/search/variables_11.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['remain_5fct_5fbytes_0',['remain_ct_bytes',['../structchacha20__poly1305__context__data.html#a74e971fe76c8b946e30e8f0c8cce111a',1,'chacha20_poly1305_context_data']]], + ['remain_5fks_5fbytes_1',['remain_ks_bytes',['../structchacha20__poly1305__context__data.html#a84693b96048f6130a749b633c65e2105',1,'chacha20_poly1305_context_data']]], + ['reserved_2',['reserved',['../structIMB__JOB.html#a9d5e67861edf38dfa23160b650c7caa9',1,'IMB_JOB::reserved()'],['../structIMB__MGR.html#a136697eb37d5a06eeada0f338abd8b51',1,'IMB_MGR::reserved()']]] +]; diff --git a/docs/search/variables_12.html b/docs/search/variables_12.html new file mode 100644 index 00000000..7007ee24 --- /dev/null +++ b/docs/search/variables_12.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_12.js b/docs/search/variables_12.js new file mode 100644 index 00000000..3c0f2c8f --- /dev/null +++ b/docs/search/variables_12.js @@ -0,0 +1,49 @@ +var searchData= +[ + ['sgl_5fio_5fsegs_0',['sgl_io_segs',['../structIMB__JOB.html#a2462f9efb16c31ae615df7493dddb343',1,'IMB_JOB']]], + ['sgl_5fstate_1',['sgl_state',['../structIMB__JOB.html#a3d7e363be004999d02db464c721e9751',1,'IMB_JOB']]], + ['sha1_2',['sha1',['../structIMB__MGR.html#a82aaedc800096aaa55062c56131e0b13',1,'IMB_MGR']]], + ['sha1_5fone_5fblock_3',['sha1_one_block',['../structIMB__MGR.html#a80a26be9a12f561a68b70d65e096cde9',1,'IMB_MGR']]], + ['sha224_4',['sha224',['../structIMB__MGR.html#a52e26b8805b28d1298ac48e5762412b5',1,'IMB_MGR']]], + ['sha224_5fone_5fblock_5',['sha224_one_block',['../structIMB__MGR.html#aacf8209094d12ceceed96d4d0fae193c',1,'IMB_MGR']]], + ['sha256_6',['sha256',['../structIMB__MGR.html#a8d978764a953e220b3c799990bae490b',1,'IMB_MGR']]], + ['sha256_5fone_5fblock_7',['sha256_one_block',['../structIMB__MGR.html#ade4a72c39980f0bba790b70529b60158',1,'IMB_MGR']]], + ['sha384_8',['sha384',['../structIMB__MGR.html#a9ffcce0ba9aef3cd68bdd7f6e9e3e2a0',1,'IMB_MGR']]], + ['sha384_5fone_5fblock_9',['sha384_one_block',['../structIMB__MGR.html#a92b7ff145f0016dd46eabf9271ef03f1',1,'IMB_MGR']]], + ['sha512_10',['sha512',['../structIMB__MGR.html#a62ebdd5e2b5a59c9e648511caf56f3f1',1,'IMB_MGR']]], + ['sha512_5fone_5fblock_11',['sha512_one_block',['../structIMB__MGR.html#a88f1060a2adb8098fc085fc2b812addc',1,'IMB_MGR']]], + ['sha_5f1_5fooo_12',['sha_1_ooo',['../structIMB__MGR.html#acdb53e812f3e4a307a497eeab28c4256',1,'IMB_MGR']]], + ['sha_5f224_5fooo_13',['sha_224_ooo',['../structIMB__MGR.html#aa174bf6846067a48f3f47ee5ebd74877',1,'IMB_MGR']]], + ['sha_5f256_5fooo_14',['sha_256_ooo',['../structIMB__MGR.html#a37fc5ccae7af099a8517566a20751265',1,'IMB_MGR']]], + ['sha_5f384_5fooo_15',['sha_384_ooo',['../structIMB__MGR.html#a51a499114f84eed88cb48bd3b461bd1c',1,'IMB_MGR']]], + ['sha_5f512_5fooo_16',['sha_512_ooo',['../structIMB__MGR.html#a6e21e034cd011b695bac5a711be65da1',1,'IMB_MGR']]], + ['shifted_5fhkey_17',['shifted_hkey',['../structgcm__key__data.html#a73b61a77635c0357b36b83aaf69d1237',1,'gcm_key_data']]], + ['shifted_5fhkey_5fk_18',['shifted_hkey_k',['../structgcm__key__data.html#a570a82f076ceb5da6719d1a554584820',1,'gcm_key_data']]], + ['sk16_19',['sk16',['../structkasumi__key__sched__s.html#a7e69de411b81753c10776442bbe03100',1,'kasumi_key_sched_s']]], + ['snow3g_5ff8_5f1_5fbuffer_20',['snow3g_f8_1_buffer',['../structIMB__MGR.html#ab70cbec09a61b5adeae37260c10234ce',1,'IMB_MGR']]], + ['snow3g_5ff8_5f1_5fbuffer_5fbit_21',['snow3g_f8_1_buffer_bit',['../structIMB__MGR.html#ad85fdcb101391b0a2488a3244d5f442d',1,'IMB_MGR']]], + ['snow3g_5ff8_5f2_5fbuffer_22',['snow3g_f8_2_buffer',['../structIMB__MGR.html#a25f46509b2f8926902a73e2210d697fd',1,'IMB_MGR']]], + ['snow3g_5ff8_5f4_5fbuffer_23',['snow3g_f8_4_buffer',['../structIMB__MGR.html#a194a0f15a7345ee369780dbdeb4f52f6',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_24',['snow3g_f8_8_buffer',['../structIMB__MGR.html#a620d2d75bb50784aa0000b458197afaf',1,'IMB_MGR']]], + ['snow3g_5ff8_5f8_5fbuffer_5fmultikey_25',['snow3g_f8_8_buffer_multikey',['../structIMB__MGR.html#a898a4c56e1a746a328529c3665c5682e',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_26',['snow3g_f8_n_buffer',['../structIMB__MGR.html#a829c4cf1157dfc3fcb829df31e408e9d',1,'IMB_MGR']]], + ['snow3g_5ff8_5fn_5fbuffer_5fmultikey_27',['snow3g_f8_n_buffer_multikey',['../structIMB__MGR.html#a63a2a5f74e7cc2fc637896bdaeb6b262',1,'IMB_MGR']]], + ['snow3g_5ff9_5f1_5fbuffer_28',['snow3g_f9_1_buffer',['../structIMB__MGR.html#a9a4d2148b6e8b4426b2946e9fe695461',1,'IMB_MGR']]], + ['snow3g_5finit_5fkey_5fsched_29',['snow3g_init_key_sched',['../structIMB__MGR.html#a38ff0528815be4d92481948664366ca9',1,'IMB_MGR']]], + ['snow3g_5fkey_5fsched_5fsize_30',['snow3g_key_sched_size',['../structIMB__MGR.html#a5e0002330399f643e8f0ca0c1b96a7a0',1,'IMB_MGR']]], + ['snow3g_5fuea2_5fooo_31',['snow3g_uea2_ooo',['../structIMB__MGR.html#ab0193bf46978026a82803c25e11acbfb',1,'IMB_MGR']]], + ['snow3g_5fuia2_32',['SNOW3G_UIA2',['../structIMB__JOB.html#a2a48f64b9344d0b6ca6775a4fdbe21b7',1,'IMB_JOB']]], + ['snow3g_5fuia2_5fooo_33',['snow3g_uia2_ooo',['../structIMB__MGR.html#aa7d4cfe53e0d70c22d3a274b8cd218af',1,'IMB_MGR']]], + ['snow_5fv_5faead_34',['SNOW_V_AEAD',['../structIMB__JOB.html#abf4a3f2db05c6583de5a99d9590ddca1',1,'IMB_JOB']]], + ['src_35',['src',['../structIMB__JOB.html#af83d63eb71a4f26b4915562bea554f28',1,'IMB_JOB']]], + ['sse_5favx_36',['sse_avx',['../structgcm__key__data.html#a908988e25607cd3e092bfcea9964deb1',1,'gcm_key_data']]], + ['status_37',['status',['../structIMB__JOB.html#ad4e08611eee5188cb84cb2dc337ef9f0',1,'IMB_JOB']]], + ['submit_5fburst_38',['submit_burst',['../structIMB__MGR.html#a6a790b2aa04138fce78c3347e285a31b',1,'IMB_MGR']]], + ['submit_5fburst_5fnocheck_39',['submit_burst_nocheck',['../structIMB__MGR.html#ad16230a6a671adbd597675abbc0f0a87',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_40',['submit_cipher_burst',['../structIMB__MGR.html#a8b52f1b3ca92eff48b945929a0eab839',1,'IMB_MGR']]], + ['submit_5fcipher_5fburst_5fnocheck_41',['submit_cipher_burst_nocheck',['../structIMB__MGR.html#acb2ee84d30247bb257635063db3ae308',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_42',['submit_hash_burst',['../structIMB__MGR.html#a20a3d4d1e40aed26d83db5395b7ccc83',1,'IMB_MGR']]], + ['submit_5fhash_5fburst_5fnocheck_43',['submit_hash_burst_nocheck',['../structIMB__MGR.html#a809782eb9dcbd264442bcd9edd2f4c2f',1,'IMB_MGR']]], + ['submit_5fjob_44',['submit_job',['../structIMB__MGR.html#ab143534d96e0d585b19dfc19553e929b',1,'IMB_MGR']]], + ['submit_5fjob_5fnocheck_45',['submit_job_nocheck',['../structIMB__MGR.html#aab7350eeecc56b1ef45606b83519a72c',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_13.html b/docs/search/variables_13.html new file mode 100644 index 00000000..157da6fb --- /dev/null +++ b/docs/search/variables_13.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_13.js b/docs/search/variables_13.js new file mode 100644 index 00000000..6cf3c83e --- /dev/null +++ b/docs/search/variables_13.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['u_0',['u',['../structIMB__JOB.html#a5e6d8702069bf88ce81cd750df66bcbc',1,'IMB_JOB']]], + ['used_5farch_1',['used_arch',['../structIMB__MGR.html#a44b6c95f8ce611828b6acacdb459cdc9',1,'IMB_MGR']]], + ['user_5fdata_2',['user_data',['../structIMB__JOB.html#abd56907c9485830817f7b30870726aeb',1,'IMB_JOB']]], + ['user_5fdata2_3',['user_data2',['../structIMB__JOB.html#a4e68c8cf14552db625e26477650dfb8b',1,'IMB_JOB']]] +]; diff --git a/docs/search/variables_14.html b/docs/search/variables_14.html new file mode 100644 index 00000000..58ed5809 --- /dev/null +++ b/docs/search/variables_14.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_14.js b/docs/search/variables_14.js new file mode 100644 index 00000000..835924a7 --- /dev/null +++ b/docs/search/variables_14.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['vaes_5favx512_0',['vaes_avx512',['../structgcm__key__data.html#a7ad44e105b23e41b7f79530b6370e509',1,'gcm_key_data']]] +]; diff --git a/docs/search/variables_15.html b/docs/search/variables_15.html new file mode 100644 index 00000000..cddde1b2 --- /dev/null +++ b/docs/search/variables_15.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_15.js b/docs/search/variables_15.js new file mode 100644 index 00000000..bef3a62f --- /dev/null +++ b/docs/search/variables_15.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['xcbc_0',['XCBC',['../structIMB__JOB.html#a3508ac7d49477dffb93b6b9cfbf0ebf3',1,'IMB_JOB']]], + ['xcbc_5fkeyexp_1',['xcbc_keyexp',['../structIMB__MGR.html#ae575c2350f5fca9a182c541795bb7d4d',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_16.html b/docs/search/variables_16.html new file mode 100644 index 00000000..88e62b25 --- /dev/null +++ b/docs/search/variables_16.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_16.js b/docs/search/variables_16.js new file mode 100644 index 00000000..787e2b78 --- /dev/null +++ b/docs/search/variables_16.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['zuc256_5feea3_5fooo_0',['zuc256_eea3_ooo',['../structIMB__MGR.html#a6d05b0a20196be514182920710dce2a8',1,'IMB_MGR']]], + ['zuc256_5feia3_5fooo_1',['zuc256_eia3_ooo',['../structIMB__MGR.html#a5f8744a8d64273d3f4cbcd9ad41ce3d5',1,'IMB_MGR']]], + ['zuc_5feea3_5fooo_2',['zuc_eea3_ooo',['../structIMB__MGR.html#a26d22c5ff963b9c7ffc8af19292bce5c',1,'IMB_MGR']]], + ['zuc_5feia3_3',['ZUC_EIA3',['../structIMB__JOB.html#ab42f79355b9173c63aca9b0b577a2b61',1,'IMB_JOB']]], + ['zuc_5feia3_5fooo_4',['zuc_eia3_ooo',['../structIMB__MGR.html#a910f747aa1a0e6a46f87b0dfd2189219',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_2.html b/docs/search/variables_2.html new file mode 100644 index 00000000..407ac957 --- /dev/null +++ b/docs/search/variables_2.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_2.js b/docs/search/variables_2.js new file mode 100644 index 00000000..5f1cd934 --- /dev/null +++ b/docs/search/variables_2.js @@ -0,0 +1,35 @@ +var searchData= +[ + ['cbcs_0',['CBCS',['../structIMB__JOB.html#a832c73263792bf8895cf028035730c4e',1,'IMB_JOB']]], + ['ccm_1',['CCM',['../structIMB__JOB.html#ad43a9c9149ec16b8e9164771ec38d379',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_2',['CHACHA20_POLY1305',['../structIMB__JOB.html#ace537bcd7b871fddd4514b7fd4f678a3',1,'IMB_JOB']]], + ['chacha20_5fpoly1305_5fdec_5fupdate_3',['chacha20_poly1305_dec_update',['../structIMB__MGR.html#a715aede4176564ac475e42c8a377bd8d',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5fenc_5fupdate_4',['chacha20_poly1305_enc_update',['../structIMB__MGR.html#ab2c87c40f9844795e3987536c602ce03',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5ffinalize_5',['chacha20_poly1305_finalize',['../structIMB__MGR.html#ab18b3fcba2aa252bb31f66093bc62abb',1,'IMB_MGR']]], + ['chacha20_5fpoly1305_5finit_6',['chacha20_poly1305_init',['../structIMB__MGR.html#a67b96b1b26fa25391f9f4b83a7a4b6c0',1,'IMB_MGR']]], + ['chain_5forder_7',['chain_order',['../structIMB__JOB.html#a41e9fa248ebb040ce1e074a2444b55d2',1,'IMB_JOB']]], + ['cipher_5fdirection_8',['cipher_direction',['../structIMB__JOB.html#a1449c51b88ccdb2f855525b89c05eb84',1,'IMB_JOB']]], + ['cipher_5ffields_9',['cipher_fields',['../structIMB__JOB.html#a006bf024dd3e809fad23c3df813c6fb2',1,'IMB_JOB']]], + ['cipher_5ffunc_10',['cipher_func',['../structIMB__JOB.html#a652eb3eee14e59be703bf28e69ca0671',1,'IMB_JOB']]], + ['cipher_5fmode_11',['cipher_mode',['../structIMB__JOB.html#a78113e875fdb681e775fd32ee67c7597',1,'IMB_JOB']]], + ['cipher_5fstart_5foffset_5fin_5fbits_12',['cipher_start_offset_in_bits',['../structIMB__JOB.html#a60a01da0056367a6f0a326265e4da4fb',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbits_13',['cipher_start_src_offset_in_bits',['../structIMB__JOB.html#af5c226d1c875c4e30a8f7bd7c03f05b1',1,'IMB_JOB']]], + ['cipher_5fstart_5fsrc_5foffset_5fin_5fbytes_14',['cipher_start_src_offset_in_bytes',['../structIMB__JOB.html#adda0b9bd22d445402bb9866e423707c4',1,'IMB_JOB']]], + ['cmac_15',['CMAC',['../structIMB__JOB.html#a499c1e90c9004ab7bb50fac98b6a9048',1,'IMB_JOB']]], + ['cmac_5fsubkey_5fgen_5f128_16',['cmac_subkey_gen_128',['../structIMB__MGR.html#aa3404b7811d57821ea574994e0c88995',1,'IMB_MGR']]], + ['cmac_5fsubkey_5fgen_5f256_17',['cmac_subkey_gen_256',['../structIMB__MGR.html#abd500d2cc03f6470ac6988785bebdf76',1,'IMB_MGR']]], + ['crc10_5fiuup_5fdata_18',['crc10_iuup_data',['../structIMB__MGR.html#a7d0979deda4e2ff7ac202c82611ce94c',1,'IMB_MGR']]], + ['crc11_5ffp_5fheader_19',['crc11_fp_header',['../structIMB__MGR.html#a4b901916c2f262f1c873e4cec7b7f474',1,'IMB_MGR']]], + ['crc16_5ffp_5fdata_20',['crc16_fp_data',['../structIMB__MGR.html#ad566ac43002c2ba8ecb93048d5edc41b',1,'IMB_MGR']]], + ['crc16_5fx25_21',['crc16_x25',['../structIMB__MGR.html#a52a7ab10d6434820e9b31d0e772ea0bc',1,'IMB_MGR']]], + ['crc24_5flte_5fa_22',['crc24_lte_a',['../structIMB__MGR.html#a058cfc60f2fdd68678b36f4f038270ac',1,'IMB_MGR']]], + ['crc24_5flte_5fb_23',['crc24_lte_b',['../structIMB__MGR.html#abe8406f3582d3540edb770e153b74c3e',1,'IMB_MGR']]], + ['crc32_5fethernet_5ffcs_24',['crc32_ethernet_fcs',['../structIMB__MGR.html#a8483109fbe42f8b7fa6e7f53e016d5ec',1,'IMB_MGR']]], + ['crc32_5fsctp_25',['crc32_sctp',['../structIMB__MGR.html#a4aed66c9e73808a65a062e1a5b0f66aa',1,'IMB_MGR']]], + ['crc32_5fwimax_5fofdma_5fdata_26',['crc32_wimax_ofdma_data',['../structIMB__MGR.html#ad08e44bc4edc54c556ea2245f848b974',1,'IMB_MGR']]], + ['crc6_5fiuup_5fheader_27',['crc6_iuup_header',['../structIMB__MGR.html#aa610dc7226673020be410762b995b725',1,'IMB_MGR']]], + ['crc7_5ffp_5fheader_28',['crc7_fp_header',['../structIMB__MGR.html#ac819ddcc968604186983d58f275bd24b',1,'IMB_MGR']]], + ['crc8_5fwimax_5fofdma_5fhcs_29',['crc8_wimax_ofdma_hcs',['../structIMB__MGR.html#a5e5be572c46ed01ac11fc95738373e2a',1,'IMB_MGR']]], + ['ctx_30',['ctx',['../structIMB__JOB.html#a053ceeb65e611c48a97a3e10d03d47eb',1,'IMB_JOB::ctx()'],['../structIMB__JOB.html#a4abdf3311d861979dc39669b3b7f1ed8',1,'IMB_JOB::ctx()']]], + ['current_5fcounter_31',['current_counter',['../structgcm__context__data.html#a2a454a5e44422aaec135dc7807aa55a4',1,'gcm_context_data']]] +]; diff --git a/docs/search/variables_3.html b/docs/search/variables_3.html new file mode 100644 index 00000000..cded9928 --- /dev/null +++ b/docs/search/variables_3.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_3.js b/docs/search/variables_3.js new file mode 100644 index 00000000..8accaf45 --- /dev/null +++ b/docs/search/variables_3.js @@ -0,0 +1,16 @@ +var searchData= +[ + ['dec_5fkeys_0',['dec_keys',['../structIMB__JOB.html#af78a3df5a9f79f329c8d1095b9ecea05',1,'IMB_JOB']]], + ['des3_5fdec_5fooo_1',['des3_dec_ooo',['../structIMB__MGR.html#a2ab69fdbef6a9a0574fcbb5ea9a7da3b',1,'IMB_MGR']]], + ['des3_5fenc_5fooo_2',['des3_enc_ooo',['../structIMB__MGR.html#a87f7566634911b4dba48a19a0c3a4993',1,'IMB_MGR']]], + ['des_5fdec_5fooo_3',['des_dec_ooo',['../structIMB__MGR.html#af03e401b002e2e415c8a927c46e1517b',1,'IMB_MGR']]], + ['des_5fenc_5fooo_4',['des_enc_ooo',['../structIMB__MGR.html#a54ec50411917f1aa9016db268ac638c1',1,'IMB_MGR']]], + ['des_5fkey_5fsched_5',['des_key_sched',['../structIMB__MGR.html#aae9afac5c2c1bf7b27d187d10778a1ee',1,'IMB_MGR']]], + ['docsis128_5fcrc32_5fsec_5fooo_6',['docsis128_crc32_sec_ooo',['../structIMB__MGR.html#acf438b50cb95a4fff1276b6666d22ad4',1,'IMB_MGR']]], + ['docsis128_5fsec_5fooo_7',['docsis128_sec_ooo',['../structIMB__MGR.html#a93cb5904608e179c49917f80c64fabd1',1,'IMB_MGR']]], + ['docsis256_5fcrc32_5fsec_5fooo_8',['docsis256_crc32_sec_ooo',['../structIMB__MGR.html#af6531598b8dd0131a11e0cf95854605d',1,'IMB_MGR']]], + ['docsis256_5fsec_5fooo_9',['docsis256_sec_ooo',['../structIMB__MGR.html#acb6e3b70e16194274e2eaefdff8f9e89',1,'IMB_MGR']]], + ['docsis_5fdes_5fdec_5fooo_10',['docsis_des_dec_ooo',['../structIMB__MGR.html#a3d5173cea3d4932576f0a5b3abbc1f40',1,'IMB_MGR']]], + ['docsis_5fdes_5fenc_5fooo_11',['docsis_des_enc_ooo',['../structIMB__MGR.html#a937fb0d3b113d203ee1a4beaf5b5b98f',1,'IMB_MGR']]], + ['dst_12',['dst',['../structIMB__JOB.html#ac1f7c363e7709cf1a7de1364aa8625e2',1,'IMB_JOB']]] +]; diff --git a/docs/search/variables_4.html b/docs/search/variables_4.html new file mode 100644 index 00000000..bae8b4aa --- /dev/null +++ b/docs/search/variables_4.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_4.js b/docs/search/variables_4.js new file mode 100644 index 00000000..086cbee1 --- /dev/null +++ b/docs/search/variables_4.js @@ -0,0 +1,12 @@ +var searchData= +[ + ['earliest_5fjob_0',['earliest_job',['../structIMB__MGR.html#ac5cd33ac3bf985dc31ccc716fb3a7747',1,'IMB_MGR']]], + ['eea3_5f1_5fbuffer_1',['eea3_1_buffer',['../structIMB__MGR.html#aeff54b370fbe495ce96370811d280140',1,'IMB_MGR']]], + ['eea3_5f4_5fbuffer_2',['eea3_4_buffer',['../structIMB__MGR.html#ac5ad1bf8ecdcfc51eaf681fb353c9621',1,'IMB_MGR']]], + ['eea3_5fn_5fbuffer_3',['eea3_n_buffer',['../structIMB__MGR.html#a67d8a55ba87742b8cf282d23ed9e3ef5',1,'IMB_MGR']]], + ['eia3_5f1_5fbuffer_4',['eia3_1_buffer',['../structIMB__MGR.html#a3ed58bdf37ab6d5f0d28271882d52327',1,'IMB_MGR']]], + ['eia3_5fn_5fbuffer_5',['eia3_n_buffer',['../structIMB__MGR.html#ab515c5aec39b1a9a31454b945e5ccd35',1,'IMB_MGR']]], + ['enc_5fkeys_6',['enc_keys',['../structIMB__JOB.html#ac656e2d98f36bd5547cc9fa297c1de62',1,'IMB_JOB']]], + ['end_5fooo_7',['end_ooo',['../structIMB__MGR.html#a6e1ab772182b97f931af3d4685785353',1,'IMB_MGR']]], + ['expanded_5fkeys_8',['expanded_keys',['../structgcm__key__data.html#aa5bda4f55b8393be151cfe28a9dc1ca9',1,'gcm_key_data']]] +]; diff --git a/docs/search/variables_5.html b/docs/search/variables_5.html new file mode 100644 index 00000000..24972338 --- /dev/null +++ b/docs/search/variables_5.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_5.js b/docs/search/variables_5.js new file mode 100644 index 00000000..0a5afefa --- /dev/null +++ b/docs/search/variables_5.js @@ -0,0 +1,14 @@ +var searchData= +[ + ['f8_5f1_5fbuffer_0',['f8_1_buffer',['../structIMB__MGR.html#af854e664cef84cb8be4495a1af5e9cb5',1,'IMB_MGR']]], + ['f8_5f1_5fbuffer_5fbit_1',['f8_1_buffer_bit',['../structIMB__MGR.html#aacf6ce3f508fef115dcd2dc415ca7379',1,'IMB_MGR']]], + ['f8_5f2_5fbuffer_2',['f8_2_buffer',['../structIMB__MGR.html#ac733fc4c5ee6e80e8e71b20a9bb2dc6e',1,'IMB_MGR']]], + ['f8_5f3_5fbuffer_3',['f8_3_buffer',['../structIMB__MGR.html#a929b9c50bc98bf37daec7a1603f10118',1,'IMB_MGR']]], + ['f8_5f4_5fbuffer_4',['f8_4_buffer',['../structIMB__MGR.html#af0b1c5d3b269b252a75a55df8bdf472c',1,'IMB_MGR']]], + ['f8_5fn_5fbuffer_5',['f8_n_buffer',['../structIMB__MGR.html#ae0f2316d96e46b64d08e3cf73a93b39d',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_6',['f9_1_buffer',['../structIMB__MGR.html#afafb95a00fe6633fb6901855928b08bb',1,'IMB_MGR']]], + ['f9_5f1_5fbuffer_5fuser_7',['f9_1_buffer_user',['../structIMB__MGR.html#a4fbdac066c9d0fbd1d586a2e5b2319c3',1,'IMB_MGR']]], + ['features_8',['features',['../structIMB__MGR.html#af877c7669265775604857c6b81063bdb',1,'IMB_MGR']]], + ['flags_9',['flags',['../structIMB__MGR.html#ae1a596c9c9d6ca1292c6116575021c14',1,'IMB_MGR']]], + ['flush_5fjob_10',['flush_job',['../structIMB__MGR.html#a225e3b35eef8e18fd09c429fbb2f70b5',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_6.html b/docs/search/variables_6.html new file mode 100644 index 00000000..8633e44e --- /dev/null +++ b/docs/search/variables_6.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_6.js b/docs/search/variables_6.js new file mode 100644 index 00000000..8719ff9d --- /dev/null +++ b/docs/search/variables_6.js @@ -0,0 +1,50 @@ +var searchData= +[ + ['gcm_0',['GCM',['../structIMB__JOB.html#ac9fc7aef831cfc1ca598a55b250dd858',1,'IMB_JOB']]], + ['gcm128_5fdec_1',['gcm128_dec',['../structIMB__MGR.html#a2ce2f6e6d61953b24a5859279534b02a',1,'IMB_MGR']]], + ['gcm128_5fdec_5ffinalize_2',['gcm128_dec_finalize',['../structIMB__MGR.html#af95fa831fb24629e19613e8594e5d32a',1,'IMB_MGR']]], + ['gcm128_5fdec_5fupdate_3',['gcm128_dec_update',['../structIMB__MGR.html#a28ddfed1a807fedb9c77c99736278dcb',1,'IMB_MGR']]], + ['gcm128_5fenc_4',['gcm128_enc',['../structIMB__MGR.html#a9f8b9687fbb17f7a78fc4ba79e2749fd',1,'IMB_MGR']]], + ['gcm128_5fenc_5ffinalize_5',['gcm128_enc_finalize',['../structIMB__MGR.html#a8efc0cf95907805289a297c024ce0cac',1,'IMB_MGR']]], + ['gcm128_5fenc_5fupdate_6',['gcm128_enc_update',['../structIMB__MGR.html#ad2c64d9d5f9ffd65d13e73205cb803cd',1,'IMB_MGR']]], + ['gcm128_5finit_7',['gcm128_init',['../structIMB__MGR.html#ad8c004899d29974a3f4c034370422c31',1,'IMB_MGR']]], + ['gcm128_5finit_5fvar_5fiv_8',['gcm128_init_var_iv',['../structIMB__MGR.html#aebd9eab76cd056add559e6bea328dbf0',1,'IMB_MGR']]], + ['gcm128_5fpre_9',['gcm128_pre',['../structIMB__MGR.html#a207c78e01e4aa1a35de1bbd2cc0a6ee2',1,'IMB_MGR']]], + ['gcm128_5fprecomp_10',['gcm128_precomp',['../structIMB__MGR.html#a69e4dd044703cd3aa9c4c36e04b2a701',1,'IMB_MGR']]], + ['gcm192_5fdec_11',['gcm192_dec',['../structIMB__MGR.html#adff2cf46e7cbb5e2926f348623d8ad2a',1,'IMB_MGR']]], + ['gcm192_5fdec_5ffinalize_12',['gcm192_dec_finalize',['../structIMB__MGR.html#a4d7879172344be144b92be68b2e2a81a',1,'IMB_MGR']]], + ['gcm192_5fdec_5fupdate_13',['gcm192_dec_update',['../structIMB__MGR.html#a66fa94f9c07061f3d93b78ed505931e3',1,'IMB_MGR']]], + ['gcm192_5fenc_14',['gcm192_enc',['../structIMB__MGR.html#aaaeb1676e0cf19e3558d10e537f71bda',1,'IMB_MGR']]], + ['gcm192_5fenc_5ffinalize_15',['gcm192_enc_finalize',['../structIMB__MGR.html#a91ad9ce7a4b3c9ceeffa9dd712013915',1,'IMB_MGR']]], + ['gcm192_5fenc_5fupdate_16',['gcm192_enc_update',['../structIMB__MGR.html#a3b5c3407cc4595420dbe43e98090e558',1,'IMB_MGR']]], + ['gcm192_5finit_17',['gcm192_init',['../structIMB__MGR.html#a006315a7045996d451c7dfd03b96a076',1,'IMB_MGR']]], + ['gcm192_5finit_5fvar_5fiv_18',['gcm192_init_var_iv',['../structIMB__MGR.html#a7bfa7b17c0fb36dc832138a86948991f',1,'IMB_MGR']]], + ['gcm192_5fpre_19',['gcm192_pre',['../structIMB__MGR.html#a149b463ecf68ee78d39c33e183ed8997',1,'IMB_MGR']]], + ['gcm192_5fprecomp_20',['gcm192_precomp',['../structIMB__MGR.html#abec8bdb319ca132028d5b787bf60a895',1,'IMB_MGR']]], + ['gcm256_5fdec_21',['gcm256_dec',['../structIMB__MGR.html#acfdfb511771cac71e032c5bf1c9273b7',1,'IMB_MGR']]], + ['gcm256_5fdec_5ffinalize_22',['gcm256_dec_finalize',['../structIMB__MGR.html#a5c2e4a1296e33f5c9eaff2067a6432d4',1,'IMB_MGR']]], + ['gcm256_5fdec_5fupdate_23',['gcm256_dec_update',['../structIMB__MGR.html#a99fcb1d18fa45044063c6c288b0fc769',1,'IMB_MGR']]], + ['gcm256_5fenc_24',['gcm256_enc',['../structIMB__MGR.html#a79767326692a8d4faabe5ab435dc5df8',1,'IMB_MGR']]], + ['gcm256_5fenc_5ffinalize_25',['gcm256_enc_finalize',['../structIMB__MGR.html#a9764d27efef2491a86d517f38dd9b6ef',1,'IMB_MGR']]], + ['gcm256_5fenc_5fupdate_26',['gcm256_enc_update',['../structIMB__MGR.html#ac131ceec0dcb0d08097b76c8abe9e024',1,'IMB_MGR']]], + ['gcm256_5finit_27',['gcm256_init',['../structIMB__MGR.html#a8987d7d491628d9d80f12820468ac541',1,'IMB_MGR']]], + ['gcm256_5finit_5fvar_5fiv_28',['gcm256_init_var_iv',['../structIMB__MGR.html#afa275308dab467d0c68dc921bd68d1a4',1,'IMB_MGR']]], + ['gcm256_5fpre_29',['gcm256_pre',['../structIMB__MGR.html#a44b0b9d218043be965a84c29ced2584b',1,'IMB_MGR']]], + ['gcm256_5fprecomp_30',['gcm256_precomp',['../structIMB__MGR.html#a1ea15abc11b618ceb438c7f282643c52',1,'IMB_MGR']]], + ['get_5fcompleted_5fjob_31',['get_completed_job',['../structIMB__MGR.html#a99b7111c8fc59161e0bf291d13b1f195',1,'IMB_MGR']]], + ['get_5fnext_5fjob_32',['get_next_job',['../structIMB__MGR.html#a35939892cd168eb7ce9c6c29c5401709',1,'IMB_MGR']]], + ['ghash_33',['ghash',['../structIMB__MGR.html#afc7d7ae10e8e6f37b9ec894059818fad',1,'IMB_MGR']]], + ['ghash_34',['GHASH',['../structIMB__JOB.html#af24c379b5414a5d3ec290954b98b1c72',1,'IMB_JOB']]], + ['ghash_5fkeys_35',['ghash_keys',['../structgcm__key__data.html#af5adb4225d569050207cfcfe00fe273a',1,'gcm_key_data']]], + ['ghash_5fpre_36',['ghash_pre',['../structIMB__MGR.html#af6a1c323981b8c3cfe09f94cd9dd58e2',1,'IMB_MGR']]], + ['gmac_37',['GMAC',['../structIMB__JOB.html#a07832e4571ec56f377dffa477b7aafe0',1,'IMB_JOB']]], + ['gmac128_5ffinalize_38',['gmac128_finalize',['../structIMB__MGR.html#a00ce9a8422a961166ce9855d4815cc2e',1,'IMB_MGR']]], + ['gmac128_5finit_39',['gmac128_init',['../structIMB__MGR.html#a1098c38784f6c6e4e3a3380f8be8ecc1',1,'IMB_MGR']]], + ['gmac128_5fupdate_40',['gmac128_update',['../structIMB__MGR.html#a90aad4f47448a1959b7cf747438d7ce9',1,'IMB_MGR']]], + ['gmac192_5ffinalize_41',['gmac192_finalize',['../structIMB__MGR.html#a000b24d7fbb9b57ed6bd4147d225e015',1,'IMB_MGR']]], + ['gmac192_5finit_42',['gmac192_init',['../structIMB__MGR.html#a3dd58a9d9b4fb6cf85fa26175ffac9a6',1,'IMB_MGR']]], + ['gmac192_5fupdate_43',['gmac192_update',['../structIMB__MGR.html#a5220f0d243291ae5a2f43fbc93d18725',1,'IMB_MGR']]], + ['gmac256_5ffinalize_44',['gmac256_finalize',['../structIMB__MGR.html#a3bb98c9a9c8545f363b74627870c7038',1,'IMB_MGR']]], + ['gmac256_5finit_45',['gmac256_init',['../structIMB__MGR.html#aca7d6388863eb76b60651e7b16c41b82',1,'IMB_MGR']]], + ['gmac256_5fupdate_46',['gmac256_update',['../structIMB__MGR.html#a0720a1fc9c0652c6b3aa7dfab070f73d',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_7.html b/docs/search/variables_7.html new file mode 100644 index 00000000..f122cc9c --- /dev/null +++ b/docs/search/variables_7.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_7.js b/docs/search/variables_7.js new file mode 100644 index 00000000..518ad064 --- /dev/null +++ b/docs/search/variables_7.js @@ -0,0 +1,18 @@ +var searchData= +[ + ['hash_0',['hash',['../structchacha20__poly1305__context__data.html#ae12447b2fd2e9f8a6f09f443bd303887',1,'chacha20_poly1305_context_data']]], + ['hash_5falg_1',['hash_alg',['../structIMB__JOB.html#aa4d9b62d3353150aba88aec5d8d08853',1,'IMB_JOB']]], + ['hash_5ffunc_2',['hash_func',['../structIMB__JOB.html#a62883bf8707d09816753832bef5d2902',1,'IMB_JOB']]], + ['hash_5flen_3',['hash_len',['../structchacha20__poly1305__context__data.html#ade4568d041695b318df773787872127f',1,'chacha20_poly1305_context_data']]], + ['hash_5fstart_5fsrc_5foffset_5fin_5fbytes_4',['hash_start_src_offset_in_bytes',['../structIMB__JOB.html#af7cdc3d086f6e2b12d21c1383b2bd152',1,'IMB_JOB']]], + ['hec_5f32_5',['hec_32',['../structIMB__MGR.html#a0863c9f64126aba5cb47628493dae582',1,'IMB_MGR']]], + ['hec_5f64_6',['hec_64',['../structIMB__MGR.html#a4128ee02f4cbef5aa1e6b0bb5fd46097',1,'IMB_MGR']]], + ['high_7',['high',['../structimb__uint128__t.html#aa65815c9b383e888758ae21d0c6159c9',1,'imb_uint128_t']]], + ['hmac_8',['HMAC',['../structIMB__JOB.html#aa84f5eefd378ab65ebe0b302ccb0df92',1,'IMB_JOB']]], + ['hmac_5fmd5_5fooo_9',['hmac_md5_ooo',['../structIMB__MGR.html#a855d14df536b22ceb175dc495ced891c',1,'IMB_MGR']]], + ['hmac_5fsha_5f1_5fooo_10',['hmac_sha_1_ooo',['../structIMB__MGR.html#a49be1d77b96d338c08200dce82e452d3',1,'IMB_MGR']]], + ['hmac_5fsha_5f224_5fooo_11',['hmac_sha_224_ooo',['../structIMB__MGR.html#ab9226cf37e41c2a13ae6003305654193',1,'IMB_MGR']]], + ['hmac_5fsha_5f256_5fooo_12',['hmac_sha_256_ooo',['../structIMB__MGR.html#a633807f7eaae35ada650a2c22f1366be',1,'IMB_MGR']]], + ['hmac_5fsha_5f384_5fooo_13',['hmac_sha_384_ooo',['../structIMB__MGR.html#afd72d6af1d8ff605e8aaaf984f0700ed',1,'IMB_MGR']]], + ['hmac_5fsha_5f512_5fooo_14',['hmac_sha_512_ooo',['../structIMB__MGR.html#af0cf9149495b0805a4f6f511c0fbff5b',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_8.html b/docs/search/variables_8.html new file mode 100644 index 00000000..2635803a --- /dev/null +++ b/docs/search/variables_8.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_8.js b/docs/search/variables_8.js new file mode 100644 index 00000000..90c319c5 --- /dev/null +++ b/docs/search/variables_8.js @@ -0,0 +1,9 @@ +var searchData= +[ + ['imb_5ferrno_0',['imb_errno',['../structIMB__MGR.html#a0ab7a3dfdfd8442f591699e2310df11f',1,'IMB_MGR']]], + ['in_1',['in',['../structIMB__SGL__IOV.html#a56c5b0a033d949869cee2d03f1df3129',1,'IMB_SGL_IOV']]], + ['in_5flength_2',['in_length',['../structgcm__context__data.html#a0d9e72f85aba9363fcb4f2d757342aa2',1,'gcm_context_data']]], + ['iv_3',['IV',['../structchacha20__poly1305__context__data.html#ac0d1657fc57308900691369362c56658',1,'chacha20_poly1305_context_data']]], + ['iv_4',['iv',['../structIMB__JOB.html#a960fdd62c1c10d2d3cc0a765c0a6531d',1,'IMB_JOB']]], + ['iv_5flen_5fin_5fbytes_5',['iv_len_in_bytes',['../structIMB__JOB.html#a25b4ce8aec20f64cfc393b245d62a09e',1,'IMB_JOB']]] +]; diff --git a/docs/search/variables_9.html b/docs/search/variables_9.html new file mode 100644 index 00000000..1c226963 --- /dev/null +++ b/docs/search/variables_9.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_9.js b/docs/search/variables_9.js new file mode 100644 index 00000000..f0cb16fe --- /dev/null +++ b/docs/search/variables_9.js @@ -0,0 +1,4 @@ +var searchData= +[ + ['jobs_0',['jobs',['../structIMB__MGR.html#ab342224dce812fb13a9e935e7a7b0617',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_a.html b/docs/search/variables_a.html new file mode 100644 index 00000000..d68b486b --- /dev/null +++ b/docs/search/variables_a.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_a.js b/docs/search/variables_a.js new file mode 100644 index 00000000..7432bad6 --- /dev/null +++ b/docs/search/variables_a.js @@ -0,0 +1,12 @@ +var searchData= +[ + ['k_0',['k',['../structsnow3g__key__schedule__s.html#ae0ca8cd2581836c0297b34eea1effb12',1,'snow3g_key_schedule_s']]], + ['kasumi_5finit_5ff8_5fkey_5fsched_1',['kasumi_init_f8_key_sched',['../structIMB__MGR.html#ab32d67f27cb6f06980080e09db133b24',1,'IMB_MGR']]], + ['kasumi_5finit_5ff9_5fkey_5fsched_2',['kasumi_init_f9_key_sched',['../structIMB__MGR.html#a58ad87104b08ba5b56fc65928677a639',1,'IMB_MGR']]], + ['kasumi_5fkey_5fsched_5fsize_3',['kasumi_key_sched_size',['../structIMB__MGR.html#abd0c855a3e43df48f5f21686a095f358',1,'IMB_MGR']]], + ['kasumi_5fuia1_4',['KASUMI_UIA1',['../structIMB__JOB.html#a11f452fba9d5cad2b2cdb929b7f244e9',1,'IMB_JOB']]], + ['key_5flen_5fin_5fbytes_5',['key_len_in_bytes',['../structIMB__JOB.html#abdd8a58a37e8d2daa1c2d333b826a65c',1,'IMB_JOB']]], + ['keyexp_5f128_6',['keyexp_128',['../structIMB__MGR.html#a7c97c5361b90abe325db55530b6f198e',1,'IMB_MGR']]], + ['keyexp_5f192_7',['keyexp_192',['../structIMB__MGR.html#a2bd7444a318de06b5f53178abf9c37ff',1,'IMB_MGR']]], + ['keyexp_5f256_8',['keyexp_256',['../structIMB__MGR.html#a431e99ddb3837a32bedb6cd426054bcc',1,'IMB_MGR']]] +]; diff --git a/docs/search/variables_b.html b/docs/search/variables_b.html new file mode 100644 index 00000000..c4891b45 --- /dev/null +++ b/docs/search/variables_b.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_b.js b/docs/search/variables_b.js new file mode 100644 index 00000000..85f6540f --- /dev/null +++ b/docs/search/variables_b.js @@ -0,0 +1,7 @@ +var searchData= +[ + ['last_5fblock_5fcount_0',['last_block_count',['../structchacha20__poly1305__context__data.html#add03511d1f95376b3ac64a7916c27368',1,'chacha20_poly1305_context_data']]], + ['last_5fks_1',['last_ks',['../structchacha20__poly1305__context__data.html#afc74af4af99449f0b9f01338ae723c1a',1,'chacha20_poly1305_context_data']]], + ['len_2',['len',['../structIMB__SGL__IOV.html#a3a3662456b91011eb0585667492acc88',1,'IMB_SGL_IOV']]], + ['low_3',['low',['../structimb__uint128__t.html#a30e60cc86acb6eafaa4079c0e46ba622',1,'imb_uint128_t']]] +]; diff --git a/docs/search/variables_c.html b/docs/search/variables_c.html new file mode 100644 index 00000000..c51d2cb7 --- /dev/null +++ b/docs/search/variables_c.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_c.js b/docs/search/variables_c.js new file mode 100644 index 00000000..da2f4762 --- /dev/null +++ b/docs/search/variables_c.js @@ -0,0 +1,9 @@ +var searchData= +[ + ['md5_5fone_5fblock_0',['md5_one_block',['../structIMB__MGR.html#a54b5f7b625ce226ebd46a2a19e9bc724',1,'IMB_MGR']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbits_1',['msg_len_to_cipher_in_bits',['../structIMB__JOB.html#a425d57ff565bb2ff76e2b0351b00c751',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fcipher_5fin_5fbytes_2',['msg_len_to_cipher_in_bytes',['../structIMB__JOB.html#a9c2e008fdb76df46a33300f692f92265',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbits_3',['msg_len_to_hash_in_bits',['../structIMB__JOB.html#ae45d78dcef87042ffa1daf951f715d42',1,'IMB_JOB']]], + ['msg_5flen_5fto_5fhash_5fin_5fbytes_4',['msg_len_to_hash_in_bytes',['../structIMB__JOB.html#a40caf02dcd7e45dd91cfae33159ca26e',1,'IMB_JOB']]], + ['msk16_5',['msk16',['../structkasumi__key__sched__s.html#a091988e7975015d654ccb6904c4fdf39',1,'kasumi_key_sched_s']]] +]; diff --git a/docs/search/variables_d.html b/docs/search/variables_d.html new file mode 100644 index 00000000..fcaee0d3 --- /dev/null +++ b/docs/search/variables_d.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_d.js b/docs/search/variables_d.js new file mode 100644 index 00000000..0c972f70 --- /dev/null +++ b/docs/search/variables_d.js @@ -0,0 +1,6 @@ +var searchData= +[ + ['next_5fiv_0',['next_iv',['../structIMB__JOB.html#a5d473d9cb01f8e2818a6f5f0c9483bd7',1,'IMB_JOB']]], + ['next_5fjob_1',['next_job',['../structIMB__MGR.html#a106ebd60d876e56bfc6bcc03d20acce4',1,'IMB_MGR']]], + ['num_5fsgl_5fio_5fsegs_2',['num_sgl_io_segs',['../structIMB__JOB.html#a1d0a6baa8d8f7d9714fe86fe7085b8bb',1,'IMB_JOB']]] +]; diff --git a/docs/search/variables_e.html b/docs/search/variables_e.html new file mode 100644 index 00000000..e4b18c66 --- /dev/null +++ b/docs/search/variables_e.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_e.js b/docs/search/variables_e.js new file mode 100644 index 00000000..3f451b03 --- /dev/null +++ b/docs/search/variables_e.js @@ -0,0 +1,5 @@ +var searchData= +[ + ['orig_5fiv_0',['orig_IV',['../structgcm__context__data.html#a84bac4eabc12b81d26898b6fe6728a63',1,'gcm_context_data']]], + ['out_1',['out',['../structIMB__SGL__IOV.html#a54b8da76fb8a400f9f7972bdd97f6f53',1,'IMB_SGL_IOV']]] +]; diff --git a/docs/search/variables_f.html b/docs/search/variables_f.html new file mode 100644 index 00000000..55d1471e --- /dev/null +++ b/docs/search/variables_f.html @@ -0,0 +1,37 @@ + + + + + + + + + + +
    +
    Loading...
    +
    + +
    Searching...
    +
    No Matches
    + +
    + + diff --git a/docs/search/variables_f.js b/docs/search/variables_f.js new file mode 100644 index 00000000..b274ce7e --- /dev/null +++ b/docs/search/variables_f.js @@ -0,0 +1,8 @@ +var searchData= +[ + ['partial_5fblock_5fenc_5fkey_0',['partial_block_enc_key',['../structgcm__context__data.html#a2e52ea863c681afe1d3370a987e252f0',1,'gcm_context_data']]], + ['partial_5fblock_5flength_1',['partial_block_length',['../structgcm__context__data.html#a97411247efd14ab01e7e0da6587bfa0c',1,'gcm_context_data']]], + ['poly1305_2',['POLY1305',['../structIMB__JOB.html#a15f9c1a5ab25dca6e3a550ff00ad2419',1,'IMB_JOB']]], + ['poly_5fkey_3',['poly_key',['../structchacha20__poly1305__context__data.html#a9ac4fafe2ca37395264f728c8e99a9b1',1,'chacha20_poly1305_context_data']]], + ['poly_5fscratch_4',['poly_scratch',['../structchacha20__poly1305__context__data.html#a908d80ce2c35792099dbc967818b9abd',1,'chacha20_poly1305_context_data']]] +]; diff --git a/docs/splitbar.png b/docs/splitbar.png new file mode 100644 index 0000000000000000000000000000000000000000..fe895f2c58179b471a22d8320b39a4bd7312ec8e GIT binary patch literal 314 zcmeAS@N?(olHy`uVBq!ia0vp^Yzz!63>-{AmhX=Jf(#6djGiuzAr*{o?=JLmPLyc> z_*`QK&+BH@jWrYJ7>r6%keRM@)Qyv8R=enp0jiI>aWlGyB58O zFVR20d+y`K7vDw(hJF3;>dD*3-?v=<8M)@x|EEGLnJsniYK!2U1 Y!`|5biEc?d1`HDhPgg&ebxsLQ02F6;9RL6T literal 0 HcmV?d00001 diff --git a/docs/structIMB__JOB.html b/docs/structIMB__JOB.html new file mode 100644 index 00000000..4c55e237 --- /dev/null +++ b/docs/structIMB__JOB.html @@ -0,0 +1,1383 @@ + + + + + + + +intel-ipsec-mb: IMB_JOB Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    IMB_JOB Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    +Data Fields

    const void * enc_keys
     
    const void * dec_keys
     
    uint64_t key_len_in_bytes
     
    union {
       const uint8_t *   src
     
       const struct IMB_SGL_IOV *   sgl_io_segs
     
    }; 
     
    union {
       uint8_t *   dst
     
       uint64_t   num_sgl_io_segs
     
    }; 
     
    union {
       uint64_t   cipher_start_src_offset_in_bytes
     
       uint64_t   cipher_start_src_offset_in_bits
     
       uint64_t   cipher_start_offset_in_bits
     
    }; 
     
    union {
       uint64_t   msg_len_to_cipher_in_bytes
     
       uint64_t   msg_len_to_cipher_in_bits
     
    }; 
     
    uint64_t hash_start_src_offset_in_bytes
     
    union {
       uint64_t   msg_len_to_hash_in_bytes
     
       uint64_t   msg_len_to_hash_in_bits
     
    }; 
     
    const uint8_t * iv
     
    uint64_t iv_len_in_bytes
     
    uint8_t * auth_tag_output
     
    uint64_t auth_tag_output_len_in_bytes
     
    union {
       struct _HMAC_specific_fields {
          const uint8_t *   _hashed_auth_key_xor_ipad
     
          const uint8_t *   _hashed_auth_key_xor_opad
     
       }   HMAC
     
       struct _AES_XCBC_specific_fields {
          const uint32_t *   _k1_expanded
     
          const uint8_t *   _k2
     
          const uint8_t *   _k3
     
       }   XCBC
     
       struct _AES_CCM_specific_fields {
          const void *   aad
     
          uint64_t   aad_len_in_bytes
     
       }   CCM
     
       struct _AES_CMAC_specific_fields {
          const void *   _key_expanded
     
          const void *   _skey1
     
          const void *   _skey2
     
       }   CMAC
     
       struct _AES_GCM_specific_fields {
          const void *   aad
     
          uint64_t   aad_len_in_bytes
     
          struct gcm_context_data *   ctx
     
       }   GCM
     
       struct _ZUC_EIA3_specific_fields {
          const uint8_t *   _key
     
          const uint8_t *   _iv
     
          const uint8_t *   _iv23
     
       }   ZUC_EIA3
     
       struct _SNOW3G_UIA2_specific_fields {
          const void *   _key
     
          const void *   _iv
     
       }   SNOW3G_UIA2
     
       struct _KASUMI_UIA1_specific_fields {
          const void *   _key
     
       }   KASUMI_UIA1
     
       struct _AES_GMAC_specific_fields {
          const struct gcm_key_data *   _key
     
          const void *   _iv
     
          uint64_t   iv_len_in_bytes
     
       }   GMAC
     
       struct _GHASH_specific_fields {
          const struct gcm_key_data *   _key
     
          const void *   _init_tag
     
       }   GHASH
     
       struct _POLY1305_specific_fields {
          const void *   _key
     
       }   POLY1305
     
       struct _CHACHA20_POLY1305_specific_fields {
          const void *   aad
     
          uint64_t   aad_len_in_bytes
     
          struct chacha20_poly1305_context_data *   ctx
     
       }   CHACHA20_POLY1305
     
       struct _SNOW_V_AEAD_specific_fields {
          const void *   aad
     
          uint64_t   aad_len_in_bytes
     
          void *   reserved
     
       }   SNOW_V_AEAD
     
    } u
     
    IMB_STATUS status
     
    IMB_CIPHER_MODE cipher_mode
     
    IMB_CIPHER_DIRECTION cipher_direction
     
    IMB_HASH_ALG hash_alg
     
    IMB_CHAIN_ORDER chain_order
     
    void * user_data
     
    void * user_data2
     
    int(* cipher_func )(struct IMB_JOB *)
     
    int(* hash_func )(struct IMB_JOB *)
     
    IMB_SGL_STATE sgl_state
     
    union {
       struct _CBCS_specific_fields {
          void *   next_iv
     
       }   CBCS
     
    } cipher_fields
     
    +

    Detailed Description

    +

    Job structure.

    +

    For AES, enc_keys and dec_keys are expected to point to expanded keys structure.

      +
    • AES-CTR, AES-ECB and AES-CCM, only enc_keys is used
      • +
    • DOCSIS (AES-CBC + AES-CFB), both pointers are used enc_keys has to be set always for the partial block
      • +
    +

    For DES, enc_keys and dec_keys are expected to point to DES key schedule.

      +
    • same key schedule used for enc and dec operations
      • +
    +

    For 3DES, enc_keys and dec_keys are expected to point to an array of 3 pointers for the corresponding 3 key schedules.

      +
    • same key schedule used for enc and dec operations
      • +
    +

    Field Documentation

    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::@1
    +
    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::@3
    +
    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::@5
    +
    +

    Offset into input buffer to start ciphering

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::@7
    +
    +

    Length of message to cipher

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::@9
    +
    +

    Length of message to hash

    + +
    +
    + +

    ◆ _hashed_auth_key_xor_ipad

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_hashed_auth_key_xor_ipad
    +
    +

    Hashed result of HMAC key xor'd with ipad (0x36).

    + +
    +
    + +

    ◆ _hashed_auth_key_xor_opad

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_hashed_auth_key_xor_opad
    +
    +

    Hashed result of HMAC key xor'd with opad (0x5c).

    + +
    +
    + +

    ◆ _init_tag

    + +
    +
    + + + + +
    const void* IMB_JOB::_init_tag
    +
    +

    initial tag value

    + +
    +
    + +

    ◆ _iv [1/2]

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_iv
    +
    +

    Authentication 25-byte IV (16-byte aligned)

    + +
    +
    + +

    ◆ _iv [2/2]

    + +
    +
    + + + + +
    const void* IMB_JOB::_iv
    +
    +

    Authentication IV (16-byte aligned)

    +

    Authentication IV

    + +
    +
    + +

    ◆ _iv23

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_iv23
    +
    +

    Authentication 23-byte IV (16-byte aligned)

    + +
    +
    + +

    ◆ _k1_expanded

    + +
    +
    + + + + +
    const uint32_t* IMB_JOB::_k1_expanded
    +
    +

    k1 expanded key pointer (16-byte aligned)

    + +
    +
    + +

    ◆ _k2

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_k2
    +
    +

    k2 expanded key pointer (16-byte aligned)

    + +
    +
    + +

    ◆ _k3

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_k3
    +
    +

    k3 expanded key pointer (16-byte aligned)

    + +
    +
    + +

    ◆ _key [1/3]

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::_key
    +
    +

    Authentication key (16-byte aligned)

    + +
    +
    + +

    ◆ _key [2/3]

    + +
    +
    + + + + +
    const void* IMB_JOB::_key
    +
    +

    Authentication key (16-byte aligned)

    +

    Poly1305 key

    + +
    +
    + +

    ◆ _key [3/3]

    + +
    +
    + + + + +
    const struct gcm_key_data* IMB_JOB::_key
    +
    +

    Authentication key

    +

    Expanded GHASH key

    + +
    +
    + +

    ◆ _key_expanded

    + +
    +
    + + + + +
    const void* IMB_JOB::_key_expanded
    +
    +

    Expanded key (16-byte aligned)

    + +
    +
    + +

    ◆ _skey1

    + +
    +
    + + + + +
    const void* IMB_JOB::_skey1
    +
    +

    S key 1 (16-byte aligned)

    + +
    +
    + +

    ◆ _skey2

    + +
    +
    + + + + +
    const void* IMB_JOB::_skey2
    +
    +

    S key 2 (16-byte aligned)

    + +
    +
    + +

    ◆ aad

    + +
    +
    + + + + +
    const void* IMB_JOB::aad
    +
    +

    Additional Authentication Data (AAD)

    + +
    +
    + +

    ◆ aad_len_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::aad_len_in_bytes
    +
    +

    Length of AAD

    + +
    +
    + +

    ◆ auth_tag_output

    + +
    +
    + + + + +
    uint8_t* IMB_JOB::auth_tag_output
    +
    +

    Authentication tag output

    + +
    +
    + +

    ◆ auth_tag_output_len_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::auth_tag_output_len_in_bytes
    +
    +

    Authentication tag output length in bytes

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_CBCS_specific_fields IMB_JOB::CBCS
    +
    +

    CBCS specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_AES_CCM_specific_fields IMB_JOB::CCM
    +
    +

    AES-CCM specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_CHACHA20_POLY1305_specific_fields IMB_JOB::CHACHA20_POLY1305
    +
    +

    Chacha20-Poly1305 specific fields

    + +
    +
    + +

    ◆ chain_order

    + +
    +
    + + + + +
    IMB_CHAIN_ORDER IMB_JOB::chain_order
    +
    +

    Chain order (IMB_ORDER_CIPHER_HASH / IMB_ORDER_HASH_CIPHER).

    + +
    +
    + +

    ◆ cipher_direction

    + +
    +
    + + + + +
    IMB_CIPHER_DIRECTION IMB_JOB::cipher_direction
    +
    +

    Cipher direction

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::cipher_fields
    +
    +

    Cipher algorithm-specific fields

    + +
    +
    + +

    ◆ cipher_func

    + +
    +
    + + + + +
    int(* IMB_JOB::cipher_func) (struct IMB_JOB *)
    +
    +

    Customer cipher function

    + +
    +
    + +

    ◆ cipher_mode

    + +
    +
    + + + + +
    IMB_CIPHER_MODE IMB_JOB::cipher_mode
    +
    +

    Cipher mode

    + +
    +
    + +

    ◆ cipher_start_offset_in_bits

    + +
    +
    + + + + +
    uint64_t IMB_JOB::cipher_start_offset_in_bits
    +
    +

    Offset into input buffer to start ciphering (in bits)

    + +
    +
    + +

    ◆ cipher_start_src_offset_in_bits

    + +
    +
    + + + + +
    uint64_t IMB_JOB::cipher_start_src_offset_in_bits
    +
    +

    Offset into input buffer to start ciphering (in bits)

    + +
    +
    + +

    ◆ cipher_start_src_offset_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::cipher_start_src_offset_in_bytes
    +
    +

    Offset into input buffer to start ciphering (in bytes)

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_AES_CMAC_specific_fields IMB_JOB::CMAC
    +
    +

    AES-CMAC specific fields

    + +
    +
    + +

    ◆ ctx [1/2]

    + +
    +
    + + + + +
    struct gcm_context_data* IMB_JOB::ctx
    +
    +

    AES-GCM context (for SGL only)

    + +
    +
    + +

    ◆ ctx [2/2]

    + +
    +
    + + + + +
    struct chacha20_poly1305_context_data* IMB_JOB::ctx
    +
    +

    Chacha20-Poly1305 context (for SGL only)

    + +
    +
    + +

    ◆ dec_keys

    + +
    +
    + + + + +
    const void* IMB_JOB::dec_keys
    +
    +

    Decryption key pointer

    + +
    +
    + +

    ◆ dst

    + +
    +
    + + + + +
    uint8_t* IMB_JOB::dst
    +
    +

    Output buffer. May be ciphertext or plaintext. In-place ciphering allowed, i.e. dst = src.

    + +
    +
    + +

    ◆ enc_keys

    + +
    +
    + + + + +
    const void* IMB_JOB::enc_keys
    +
    +

    Encryption key pointer

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_AES_GCM_specific_fields IMB_JOB::GCM
    +
    +

    AES-GCM specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_GHASH_specific_fields IMB_JOB::GHASH
    +
    +

    GHASH specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_AES_GMAC_specific_fields IMB_JOB::GMAC
    +
    +

    AES-GMAC specific fields

    + +
    +
    + +

    ◆ hash_alg

    + +
    +
    + + + + +
    IMB_HASH_ALG IMB_JOB::hash_alg
    +
    +

    Hashing algorithm

    + +
    +
    + +

    ◆ hash_func

    + +
    +
    + + + + +
    int(* IMB_JOB::hash_func) (struct IMB_JOB *)
    +
    +

    Customer hash function

    + +
    +
    + +

    ◆ hash_start_src_offset_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::hash_start_src_offset_in_bytes
    +
    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_HMAC_specific_fields IMB_JOB::HMAC
    +
    +

    HMAC specific fields

    + +
    +
    + +

    ◆ iv

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::iv
    +
    +

    Initialization Vector (IV)

    + +
    +
    + +

    ◆ iv_len_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::iv_len_in_bytes
    +
    +

    IV length in bytes

    +

    Authentication IV length in bytes

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_KASUMI_UIA1_specific_fields IMB_JOB::KASUMI_UIA1
    +
    +

    KASUMI-UIA2 specific fields

    + +
    +
    + +

    ◆ key_len_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::key_len_in_bytes
    +
    +

    Key length in bytes

    + +
    +
    + +

    ◆ msg_len_to_cipher_in_bits

    + +
    +
    + + + + +
    uint64_t IMB_JOB::msg_len_to_cipher_in_bits
    +
    +

    Length of message to cipher (in bits)

    + +
    +
    + +

    ◆ msg_len_to_cipher_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::msg_len_to_cipher_in_bytes
    +
    +

    Length of message to cipher (in bytes)

    + +
    +
    + +

    ◆ msg_len_to_hash_in_bits

    + +
    +
    + + + + +
    uint64_t IMB_JOB::msg_len_to_hash_in_bits
    +
    +

    Length of message to hash (in bits)

    + +
    +
    + +

    ◆ msg_len_to_hash_in_bytes

    + +
    +
    + + + + +
    uint64_t IMB_JOB::msg_len_to_hash_in_bytes
    +
    +

    Length of message to hash (in bytes)

    + +
    +
    + +

    ◆ next_iv

    + +
    +
    + + + + +
    void* IMB_JOB::next_iv
    +
    +

    Pointer to next IV (last ciphertext block)

    + +
    +
    + +

    ◆ num_sgl_io_segs

    + +
    +
    + + + + +
    uint64_t IMB_JOB::num_sgl_io_segs
    +
    +

    Number of input/output SGL segments

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_POLY1305_specific_fields IMB_JOB::POLY1305
    +
    +

    Poly1305 specific fields

    + +
    +
    + +

    ◆ reserved

    + +
    +
    + + + + +
    void* IMB_JOB::reserved
    +
    +

    Reserved bytes

    + +
    +
    + +

    ◆ sgl_io_segs

    + +
    +
    + + + + +
    const struct IMB_SGL_IOV* IMB_JOB::sgl_io_segs
    +
    +

    Pointer to array of input/output SGL segments

    + +
    +
    + +

    ◆ sgl_state

    + +
    +
    + + + + +
    IMB_SGL_STATE IMB_JOB::sgl_state
    +
    +

    SGL state (IMB_SGL_INIT/IMB_SGL_UPDATE/IMB_SGL_COMPLETE/ IMB_SGL_ALL)

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_SNOW3G_UIA2_specific_fields IMB_JOB::SNOW3G_UIA2
    +
    +

    SNOW3G-UIA2 specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_SNOW_V_AEAD_specific_fields IMB_JOB::SNOW_V_AEAD
    +
    +

    SNOW-V AEAD specific fields

    + +
    +
    + +

    ◆ src

    + +
    +
    + + + + +
    const uint8_t* IMB_JOB::src
    +
    +

    Input buffer. May be ciphertext or plaintext. In-place ciphering allowed.

    + +
    +
    + +

    ◆ status

    + +
    +
    + + + + +
    IMB_STATUS IMB_JOB::status
    +
    +

    Job status

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } IMB_JOB::u
    +
    +

    Hash algorithm-specific fields

    + +
    +
    + +

    ◆ user_data

    + +
    +
    + + + + +
    void* IMB_JOB::user_data
    +
    +

    Pointer 1 to user data

    + +
    +
    + +

    ◆ user_data2

    + +
    +
    + + + + +
    void* IMB_JOB::user_data2
    +
    +

    Pointer 2 to user data

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_AES_XCBC_specific_fields IMB_JOB::XCBC
    +
    +

    AES-XCBC specific fields

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } ::_ZUC_EIA3_specific_fields IMB_JOB::ZUC_EIA3
    +
    +

    ZUC-EIA3 specific fields

    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structIMB__JOB.js b/docs/structIMB__JOB.js new file mode 100644 index 00000000..18508840 --- /dev/null +++ b/docs/structIMB__JOB.js @@ -0,0 +1,69 @@ +var structIMB__JOB = +[ + [ "_hashed_auth_key_xor_ipad", "structIMB__JOB.html#a2e74bbeeed2c70f97e6b30a0a12aec0e", null ], + [ "_hashed_auth_key_xor_opad", "structIMB__JOB.html#a684b751d0a275a28d345a3d1051204ef", null ], + [ "_init_tag", "structIMB__JOB.html#a1e2fc881ce9c84173168df1dbda851f8", null ], + [ "_iv", "structIMB__JOB.html#ae571147592c332cea0d90a36e133e6a5", null ], + [ "_iv", "structIMB__JOB.html#a4415e6851e770734c5ec1b46a828ce55", null ], + [ "_iv23", "structIMB__JOB.html#a2bd093b1609dad73bbef2a728995d12f", null ], + [ "_k1_expanded", "structIMB__JOB.html#a76d55a988c71e40c577c6f8f5d0496de", null ], + [ "_k2", "structIMB__JOB.html#adc47d4af15c88dcb7abf01bb07e79e63", null ], + [ "_k3", "structIMB__JOB.html#afa62af4d5f1ecbb80f5216ec298fcfe8", null ], + [ "_key", "structIMB__JOB.html#a7ba1fdbcc6ed2e3840990d2e34dac8e8", null ], + [ "_key", "structIMB__JOB.html#a7a62dc4b2ef35231a93ce035561c4153", null ], + [ "_key", "structIMB__JOB.html#a3b3f616759b854553a351e6b44e08471", null ], + [ "_key_expanded", "structIMB__JOB.html#a95cc77e0c079f43dec1a46a2b75422a0", null ], + [ "_skey1", "structIMB__JOB.html#ab64e8f6bf8dfaf56305b9023103ea5e2", null ], + [ "_skey2", "structIMB__JOB.html#a8e8143e461eab3b2b04333c5f1c17a82", null ], + [ "aad", "structIMB__JOB.html#a4c943fcac7f5c4f5d6d030547840071e", null ], + [ "aad_len_in_bytes", "structIMB__JOB.html#aa607a780bc2c8509270625c7520a11f1", null ], + [ "auth_tag_output", "structIMB__JOB.html#ac37080ddb0baf56c9589e65960430b32", null ], + [ "auth_tag_output_len_in_bytes", "structIMB__JOB.html#a750df4fa7330e2387138409d0eb50e0f", null ], + [ "CBCS", "structIMB__JOB.html#a832c73263792bf8895cf028035730c4e", null ], + [ "CCM", "structIMB__JOB.html#ad43a9c9149ec16b8e9164771ec38d379", null ], + [ "CHACHA20_POLY1305", "structIMB__JOB.html#ace537bcd7b871fddd4514b7fd4f678a3", null ], + [ "chain_order", "structIMB__JOB.html#a41e9fa248ebb040ce1e074a2444b55d2", null ], + [ "cipher_direction", "structIMB__JOB.html#a1449c51b88ccdb2f855525b89c05eb84", null ], + [ "cipher_fields", "structIMB__JOB.html#a006bf024dd3e809fad23c3df813c6fb2", null ], + [ "cipher_func", "structIMB__JOB.html#a652eb3eee14e59be703bf28e69ca0671", null ], + [ "cipher_mode", "structIMB__JOB.html#a78113e875fdb681e775fd32ee67c7597", null ], + [ "cipher_start_offset_in_bits", "structIMB__JOB.html#a60a01da0056367a6f0a326265e4da4fb", null ], + [ "cipher_start_src_offset_in_bits", "structIMB__JOB.html#af5c226d1c875c4e30a8f7bd7c03f05b1", null ], + [ "cipher_start_src_offset_in_bytes", "structIMB__JOB.html#adda0b9bd22d445402bb9866e423707c4", null ], + [ "CMAC", "structIMB__JOB.html#a499c1e90c9004ab7bb50fac98b6a9048", null ], + [ "ctx", "structIMB__JOB.html#a053ceeb65e611c48a97a3e10d03d47eb", null ], + [ "ctx", "structIMB__JOB.html#a4abdf3311d861979dc39669b3b7f1ed8", null ], + [ "dec_keys", "structIMB__JOB.html#af78a3df5a9f79f329c8d1095b9ecea05", null ], + [ "dst", "structIMB__JOB.html#ac1f7c363e7709cf1a7de1364aa8625e2", null ], + [ "enc_keys", "structIMB__JOB.html#ac656e2d98f36bd5547cc9fa297c1de62", null ], + [ "GCM", "structIMB__JOB.html#ac9fc7aef831cfc1ca598a55b250dd858", null ], + [ "GHASH", "structIMB__JOB.html#af24c379b5414a5d3ec290954b98b1c72", null ], + [ "GMAC", "structIMB__JOB.html#a07832e4571ec56f377dffa477b7aafe0", null ], + [ "hash_alg", "structIMB__JOB.html#aa4d9b62d3353150aba88aec5d8d08853", null ], + [ "hash_func", "structIMB__JOB.html#a62883bf8707d09816753832bef5d2902", null ], + [ "hash_start_src_offset_in_bytes", "structIMB__JOB.html#af7cdc3d086f6e2b12d21c1383b2bd152", null ], + [ "HMAC", "structIMB__JOB.html#aa84f5eefd378ab65ebe0b302ccb0df92", null ], + [ "iv", "structIMB__JOB.html#a960fdd62c1c10d2d3cc0a765c0a6531d", null ], + [ "iv_len_in_bytes", "structIMB__JOB.html#a25b4ce8aec20f64cfc393b245d62a09e", null ], + [ "KASUMI_UIA1", "structIMB__JOB.html#a11f452fba9d5cad2b2cdb929b7f244e9", null ], + [ "key_len_in_bytes", "structIMB__JOB.html#abdd8a58a37e8d2daa1c2d333b826a65c", null ], + [ "msg_len_to_cipher_in_bits", "structIMB__JOB.html#a425d57ff565bb2ff76e2b0351b00c751", null ], + [ "msg_len_to_cipher_in_bytes", "structIMB__JOB.html#a9c2e008fdb76df46a33300f692f92265", null ], + [ "msg_len_to_hash_in_bits", "structIMB__JOB.html#ae45d78dcef87042ffa1daf951f715d42", null ], + [ "msg_len_to_hash_in_bytes", "structIMB__JOB.html#a40caf02dcd7e45dd91cfae33159ca26e", null ], + [ "next_iv", "structIMB__JOB.html#a5d473d9cb01f8e2818a6f5f0c9483bd7", null ], + [ "num_sgl_io_segs", "structIMB__JOB.html#a1d0a6baa8d8f7d9714fe86fe7085b8bb", null ], + [ "POLY1305", "structIMB__JOB.html#a15f9c1a5ab25dca6e3a550ff00ad2419", null ], + [ "reserved", "structIMB__JOB.html#a9d5e67861edf38dfa23160b650c7caa9", null ], + [ "sgl_io_segs", "structIMB__JOB.html#a2462f9efb16c31ae615df7493dddb343", null ], + [ "sgl_state", "structIMB__JOB.html#a3d7e363be004999d02db464c721e9751", null ], + [ "SNOW3G_UIA2", "structIMB__JOB.html#a2a48f64b9344d0b6ca6775a4fdbe21b7", null ], + [ "SNOW_V_AEAD", "structIMB__JOB.html#abf4a3f2db05c6583de5a99d9590ddca1", null ], + [ "src", "structIMB__JOB.html#af83d63eb71a4f26b4915562bea554f28", null ], + [ "status", "structIMB__JOB.html#ad4e08611eee5188cb84cb2dc337ef9f0", null ], + [ "u", "structIMB__JOB.html#a5e6d8702069bf88ce81cd750df66bcbc", null ], + [ "user_data", "structIMB__JOB.html#abd56907c9485830817f7b30870726aeb", null ], + [ "user_data2", "structIMB__JOB.html#a4e68c8cf14552db625e26477650dfb8b", null ], + [ "XCBC", "structIMB__JOB.html#a3508ac7d49477dffb93b6b9cfbf0ebf3", null ], + [ "ZUC_EIA3", "structIMB__JOB.html#ab42f79355b9173c63aca9b0b577a2b61", null ] +]; \ No newline at end of file diff --git a/docs/structIMB__MGR.html b/docs/structIMB__MGR.html new file mode 100644 index 00000000..5c11889d --- /dev/null +++ b/docs/structIMB__MGR.html @@ -0,0 +1,2720 @@ + + + + + + + +intel-ipsec-mb: IMB_MGR Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    IMB_MGR Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

    +Data Fields

    uint64_t flags
     
    uint64_t features
     
    uint64_t reserved [5]
     
    uint32_t used_arch
     
    int imb_errno
     
    get_next_job_t get_next_job
     
    submit_job_t submit_job
     
    submit_job_t submit_job_nocheck
     
    get_completed_job_t get_completed_job
     
    flush_job_t flush_job
     
    queue_size_t queue_size
     
    keyexp_t keyexp_128
     
    keyexp_t keyexp_192
     
    keyexp_t keyexp_256
     
    cmac_subkey_gen_t cmac_subkey_gen_128
     
    xcbc_keyexp_t xcbc_keyexp
     
    des_keysched_t des_key_sched
     
    hash_one_block_t sha1_one_block
     
    hash_one_block_t sha224_one_block
     
    hash_one_block_t sha256_one_block
     
    hash_one_block_t sha384_one_block
     
    hash_one_block_t sha512_one_block
     
    hash_one_block_t md5_one_block
     
    hash_fn_t sha1
     
    hash_fn_t sha224
     
    hash_fn_t sha256
     
    hash_fn_t sha384
     
    hash_fn_t sha512
     
    aes_cfb_t aes128_cfb_one
     
    aes_gcm_enc_dec_t gcm128_enc
     
    aes_gcm_enc_dec_t gcm192_enc
     
    aes_gcm_enc_dec_t gcm256_enc
     
    aes_gcm_enc_dec_t gcm128_dec
     
    aes_gcm_enc_dec_t gcm192_dec
     
    aes_gcm_enc_dec_t gcm256_dec
     
    aes_gcm_init_t gcm128_init
     
    aes_gcm_init_t gcm192_init
     
    aes_gcm_init_t gcm256_init
     
    aes_gcm_enc_dec_update_t gcm128_enc_update
     
    aes_gcm_enc_dec_update_t gcm192_enc_update
     
    aes_gcm_enc_dec_update_t gcm256_enc_update
     
    aes_gcm_enc_dec_update_t gcm128_dec_update
     
    aes_gcm_enc_dec_update_t gcm192_dec_update
     
    aes_gcm_enc_dec_update_t gcm256_dec_update
     
    aes_gcm_enc_dec_finalize_t gcm128_enc_finalize
     
    aes_gcm_enc_dec_finalize_t gcm192_enc_finalize
     
    aes_gcm_enc_dec_finalize_t gcm256_enc_finalize
     
    aes_gcm_enc_dec_finalize_t gcm128_dec_finalize
     
    aes_gcm_enc_dec_finalize_t gcm192_dec_finalize
     
    aes_gcm_enc_dec_finalize_t gcm256_dec_finalize
     
    aes_gcm_precomp_t gcm128_precomp
     
    aes_gcm_precomp_t gcm192_precomp
     
    aes_gcm_precomp_t gcm256_precomp
     
    aes_gcm_pre_t gcm128_pre
     
    aes_gcm_pre_t gcm192_pre
     
    aes_gcm_pre_t gcm256_pre
     
    zuc_eea3_1_buffer_t eea3_1_buffer
     
    zuc_eea3_4_buffer_t eea3_4_buffer
     
    zuc_eea3_n_buffer_t eea3_n_buffer
     
    zuc_eia3_1_buffer_t eia3_1_buffer
     
    kasumi_f8_1_buffer_t f8_1_buffer
     
    kasumi_f8_1_buffer_bit_t f8_1_buffer_bit
     
    kasumi_f8_2_buffer_t f8_2_buffer
     
    kasumi_f8_3_buffer_t f8_3_buffer
     
    kasumi_f8_4_buffer_t f8_4_buffer
     
    kasumi_f8_n_buffer_t f8_n_buffer
     
    kasumi_f9_1_buffer_t f9_1_buffer
     
    kasumi_f9_1_buffer_user_t f9_1_buffer_user
     
    kasumi_init_f8_key_sched_t kasumi_init_f8_key_sched
     
    kasumi_init_f9_key_sched_t kasumi_init_f9_key_sched
     
    kasumi_key_sched_size_t kasumi_key_sched_size
     
    snow3g_f8_1_buffer_bit_t snow3g_f8_1_buffer_bit
     
    snow3g_f8_1_buffer_t snow3g_f8_1_buffer
     
    snow3g_f8_2_buffer_t snow3g_f8_2_buffer
     
    snow3g_f8_4_buffer_t snow3g_f8_4_buffer
     
    snow3g_f8_8_buffer_t snow3g_f8_8_buffer
     
    snow3g_f8_n_buffer_t snow3g_f8_n_buffer
     
    snow3g_f8_8_buffer_multikey_t snow3g_f8_8_buffer_multikey
     
    snow3g_f8_n_buffer_multikey_t snow3g_f8_n_buffer_multikey
     
    snow3g_f9_1_buffer_t snow3g_f9_1_buffer
     
    snow3g_init_key_sched_t snow3g_init_key_sched
     
    snow3g_key_sched_size_t snow3g_key_sched_size
     
    ghash_t ghash
     
    zuc_eia3_n_buffer_t eia3_n_buffer
     
    aes_gcm_init_var_iv_t gcm128_init_var_iv
     
    aes_gcm_init_var_iv_t gcm192_init_var_iv
     
    aes_gcm_init_var_iv_t gcm256_init_var_iv
     
    aes_gmac_init_t gmac128_init
     
    aes_gmac_init_t gmac192_init
     
    aes_gmac_init_t gmac256_init
     
    aes_gmac_update_t gmac128_update
     
    aes_gmac_update_t gmac192_update
     
    aes_gmac_update_t gmac256_update
     
    aes_gmac_finalize_t gmac128_finalize
     
    aes_gmac_finalize_t gmac192_finalize
     
    aes_gmac_finalize_t gmac256_finalize
     
    hec_32_t hec_32
     
    hec_64_t hec_64
     
    cmac_subkey_gen_t cmac_subkey_gen_256
     
    aes_gcm_pre_t ghash_pre
     
    crc32_fn_t crc32_ethernet_fcs
     
    crc32_fn_t crc16_x25
     
    crc32_fn_t crc32_sctp
     
    crc32_fn_t crc24_lte_a
     
    crc32_fn_t crc24_lte_b
     
    crc32_fn_t crc16_fp_data
     
    crc32_fn_t crc11_fp_header
     
    crc32_fn_t crc7_fp_header
     
    crc32_fn_t crc10_iuup_data
     
    crc32_fn_t crc6_iuup_header
     
    crc32_fn_t crc32_wimax_ofdma_data
     
    crc32_fn_t crc8_wimax_ofdma_hcs
     
    chacha_poly_init_t chacha20_poly1305_init
     
    chacha_poly_enc_dec_update_t chacha20_poly1305_enc_update
     
    chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update
     
    chacha_poly_finalize_t chacha20_poly1305_finalize
     
    submit_burst_t submit_burst
     
    submit_burst_t submit_burst_nocheck
     
    submit_cipher_burst_t submit_cipher_burst
     
    submit_cipher_burst_t submit_cipher_burst_nocheck
     
    submit_hash_burst_t submit_hash_burst
     
    submit_hash_burst_t submit_hash_burst_nocheck
     
    int earliest_job
     
    int next_job
     
    IMB_JOB jobs [IMB_MAX_JOBS]
     
    void * aes128_ooo
     
    void * aes192_ooo
     
    void * aes256_ooo
     
    void * docsis128_sec_ooo
     
    void * docsis128_crc32_sec_ooo
     
    void * docsis256_sec_ooo
     
    void * docsis256_crc32_sec_ooo
     
    void * des_enc_ooo
     
    void * des_dec_ooo
     
    void * des3_enc_ooo
     
    void * des3_dec_ooo
     
    void * docsis_des_enc_ooo
     
    void * docsis_des_dec_ooo
     
    void * hmac_sha_1_ooo
     
    void * hmac_sha_224_ooo
     
    void * hmac_sha_256_ooo
     
    void * hmac_sha_384_ooo
     
    void * hmac_sha_512_ooo
     
    void * hmac_md5_ooo
     
    void * aes_xcbc_ooo
     
    void * aes_ccm_ooo
     
    void * aes_cmac_ooo
     
    void * zuc_eea3_ooo
     
    void * zuc_eia3_ooo
     
    void * aes128_cbcs_ooo
     
    void * zuc256_eea3_ooo
     
    void * zuc256_eia3_ooo
     
    void * aes256_ccm_ooo
     
    void * aes256_cmac_ooo
     
    void * snow3g_uea2_ooo
     
    void * snow3g_uia2_ooo
     
    void * sha_1_ooo
     
    void * sha_224_ooo
     
    void * sha_256_ooo
     
    void * sha_384_ooo
     
    void * sha_512_ooo
     
    void * end_ooo
     
    +

    Field Documentation

    + +

    ◆ aes128_cbcs_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes128_cbcs_ooo
    +
    + +
    +
    + +

    ◆ aes128_cfb_one

    + +
    +
    + + + + +
    aes_cfb_t IMB_MGR::aes128_cfb_one
    +
    + +
    +
    + +

    ◆ aes128_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes128_ooo
    +
    + +
    +
    + +

    ◆ aes192_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes192_ooo
    +
    + +
    +
    + +

    ◆ aes256_ccm_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes256_ccm_ooo
    +
    + +
    +
    + +

    ◆ aes256_cmac_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes256_cmac_ooo
    +
    + +
    +
    + +

    ◆ aes256_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes256_ooo
    +
    + +
    +
    + +

    ◆ aes_ccm_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes_ccm_ooo
    +
    + +
    +
    + +

    ◆ aes_cmac_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes_cmac_ooo
    +
    + +
    +
    + +

    ◆ aes_xcbc_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::aes_xcbc_ooo
    +
    + +
    +
    + +

    ◆ chacha20_poly1305_dec_update

    + +
    +
    + + + + +
    chacha_poly_enc_dec_update_t IMB_MGR::chacha20_poly1305_dec_update
    +
    + +
    +
    + +

    ◆ chacha20_poly1305_enc_update

    + +
    +
    + + + + +
    chacha_poly_enc_dec_update_t IMB_MGR::chacha20_poly1305_enc_update
    +
    + +
    +
    + +

    ◆ chacha20_poly1305_finalize

    + +
    +
    + + + + +
    chacha_poly_finalize_t IMB_MGR::chacha20_poly1305_finalize
    +
    + +
    +
    + +

    ◆ chacha20_poly1305_init

    + +
    +
    + + + + +
    chacha_poly_init_t IMB_MGR::chacha20_poly1305_init
    +
    + +
    +
    + +

    ◆ cmac_subkey_gen_128

    + +
    +
    + + + + +
    cmac_subkey_gen_t IMB_MGR::cmac_subkey_gen_128
    +
    + +
    +
    + +

    ◆ cmac_subkey_gen_256

    + +
    +
    + + + + +
    cmac_subkey_gen_t IMB_MGR::cmac_subkey_gen_256
    +
    + +
    +
    + +

    ◆ crc10_iuup_data

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc10_iuup_data
    +
    + +
    +
    + +

    ◆ crc11_fp_header

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc11_fp_header
    +
    + +
    +
    + +

    ◆ crc16_fp_data

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc16_fp_data
    +
    + +
    +
    + +

    ◆ crc16_x25

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc16_x25
    +
    + +
    +
    + +

    ◆ crc24_lte_a

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc24_lte_a
    +
    + +
    +
    + +

    ◆ crc24_lte_b

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc24_lte_b
    +
    + +
    +
    + +

    ◆ crc32_ethernet_fcs

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc32_ethernet_fcs
    +
    + +
    +
    + +

    ◆ crc32_sctp

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc32_sctp
    +
    + +
    +
    + +

    ◆ crc32_wimax_ofdma_data

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc32_wimax_ofdma_data
    +
    + +
    +
    + +

    ◆ crc6_iuup_header

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc6_iuup_header
    +
    + +
    +
    + +

    ◆ crc7_fp_header

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc7_fp_header
    +
    + +
    +
    + +

    ◆ crc8_wimax_ofdma_hcs

    + +
    +
    + + + + +
    crc32_fn_t IMB_MGR::crc8_wimax_ofdma_hcs
    +
    + +
    +
    + +

    ◆ des3_dec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::des3_dec_ooo
    +
    + +
    +
    + +

    ◆ des3_enc_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::des3_enc_ooo
    +
    + +
    +
    + +

    ◆ des_dec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::des_dec_ooo
    +
    + +
    +
    + +

    ◆ des_enc_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::des_enc_ooo
    +
    + +
    +
    + +

    ◆ des_key_sched

    + +
    +
    + + + + +
    des_keysched_t IMB_MGR::des_key_sched
    +
    + +
    +
    + +

    ◆ docsis128_crc32_sec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis128_crc32_sec_ooo
    +
    + +
    +
    + +

    ◆ docsis128_sec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis128_sec_ooo
    +
    + +
    +
    + +

    ◆ docsis256_crc32_sec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis256_crc32_sec_ooo
    +
    + +
    +
    + +

    ◆ docsis256_sec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis256_sec_ooo
    +
    + +
    +
    + +

    ◆ docsis_des_dec_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis_des_dec_ooo
    +
    + +
    +
    + +

    ◆ docsis_des_enc_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::docsis_des_enc_ooo
    +
    + +
    +
    + +

    ◆ earliest_job

    + +
    +
    + + + + +
    int IMB_MGR::earliest_job
    +
    +

    byte offset, -1 if none

    + +
    +
    + +

    ◆ eea3_1_buffer

    + +
    +
    + + + + +
    zuc_eea3_1_buffer_t IMB_MGR::eea3_1_buffer
    +
    + +
    +
    + +

    ◆ eea3_4_buffer

    + +
    +
    + + + + +
    zuc_eea3_4_buffer_t IMB_MGR::eea3_4_buffer
    +
    + +
    +
    + +

    ◆ eea3_n_buffer

    + +
    +
    + + + + +
    zuc_eea3_n_buffer_t IMB_MGR::eea3_n_buffer
    +
    + +
    +
    + +

    ◆ eia3_1_buffer

    + +
    +
    + + + + +
    zuc_eia3_1_buffer_t IMB_MGR::eia3_1_buffer
    +
    + +
    +
    + +

    ◆ eia3_n_buffer

    + +
    +
    + + + + +
    zuc_eia3_n_buffer_t IMB_MGR::eia3_n_buffer
    +
    + +
    +
    + +

    ◆ end_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::end_ooo
    +
    + +
    +
    + +

    ◆ f8_1_buffer

    + +
    +
    + + + + +
    kasumi_f8_1_buffer_t IMB_MGR::f8_1_buffer
    +
    + +
    +
    + +

    ◆ f8_1_buffer_bit

    + +
    +
    + + + + +
    kasumi_f8_1_buffer_bit_t IMB_MGR::f8_1_buffer_bit
    +
    + +
    +
    + +

    ◆ f8_2_buffer

    + +
    +
    + + + + +
    kasumi_f8_2_buffer_t IMB_MGR::f8_2_buffer
    +
    + +
    +
    + +

    ◆ f8_3_buffer

    + +
    +
    + + + + +
    kasumi_f8_3_buffer_t IMB_MGR::f8_3_buffer
    +
    + +
    +
    + +

    ◆ f8_4_buffer

    + +
    +
    + + + + +
    kasumi_f8_4_buffer_t IMB_MGR::f8_4_buffer
    +
    + +
    +
    + +

    ◆ f8_n_buffer

    + +
    +
    + + + + +
    kasumi_f8_n_buffer_t IMB_MGR::f8_n_buffer
    +
    + +
    +
    + +

    ◆ f9_1_buffer

    + +
    +
    + + + + +
    kasumi_f9_1_buffer_t IMB_MGR::f9_1_buffer
    +
    + +
    +
    + +

    ◆ f9_1_buffer_user

    + +
    +
    + + + + +
    kasumi_f9_1_buffer_user_t IMB_MGR::f9_1_buffer_user
    +
    + +
    +
    + +

    ◆ features

    + +
    +
    + + + + +
    uint64_t IMB_MGR::features
    +
    +

    reflects features of multi-buffer instance

    + +
    +
    + +

    ◆ flags

    + +
    +
    + + + + +
    uint64_t IMB_MGR::flags
    +
    +

    passed to alloc_mb_mgr()

    + +
    +
    + +

    ◆ flush_job

    + +
    +
    + + + + +
    flush_job_t IMB_MGR::flush_job
    +
    + +
    +
    + +

    ◆ gcm128_dec

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm128_dec
    +
    + +
    +
    + +

    ◆ gcm128_dec_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm128_dec_finalize
    +
    + +
    +
    + +

    ◆ gcm128_dec_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm128_dec_update
    +
    + +
    +
    + +

    ◆ gcm128_enc

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm128_enc
    +
    + +
    +
    + +

    ◆ gcm128_enc_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm128_enc_finalize
    +
    + +
    +
    + +

    ◆ gcm128_enc_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm128_enc_update
    +
    + +
    +
    + +

    ◆ gcm128_init

    + +
    +
    + + + + +
    aes_gcm_init_t IMB_MGR::gcm128_init
    +
    + +
    +
    + +

    ◆ gcm128_init_var_iv

    + +
    +
    + + + + +
    aes_gcm_init_var_iv_t IMB_MGR::gcm128_init_var_iv
    +
    + +
    +
    + +

    ◆ gcm128_pre

    + +
    +
    + + + + +
    aes_gcm_pre_t IMB_MGR::gcm128_pre
    +
    + +
    +
    + +

    ◆ gcm128_precomp

    + +
    +
    + + + + +
    aes_gcm_precomp_t IMB_MGR::gcm128_precomp
    +
    + +
    +
    + +

    ◆ gcm192_dec

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm192_dec
    +
    + +
    +
    + +

    ◆ gcm192_dec_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm192_dec_finalize
    +
    + +
    +
    + +

    ◆ gcm192_dec_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm192_dec_update
    +
    + +
    +
    + +

    ◆ gcm192_enc

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm192_enc
    +
    + +
    +
    + +

    ◆ gcm192_enc_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm192_enc_finalize
    +
    + +
    +
    + +

    ◆ gcm192_enc_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm192_enc_update
    +
    + +
    +
    + +

    ◆ gcm192_init

    + +
    +
    + + + + +
    aes_gcm_init_t IMB_MGR::gcm192_init
    +
    + +
    +
    + +

    ◆ gcm192_init_var_iv

    + +
    +
    + + + + +
    aes_gcm_init_var_iv_t IMB_MGR::gcm192_init_var_iv
    +
    + +
    +
    + +

    ◆ gcm192_pre

    + +
    +
    + + + + +
    aes_gcm_pre_t IMB_MGR::gcm192_pre
    +
    + +
    +
    + +

    ◆ gcm192_precomp

    + +
    +
    + + + + +
    aes_gcm_precomp_t IMB_MGR::gcm192_precomp
    +
    + +
    +
    + +

    ◆ gcm256_dec

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm256_dec
    +
    + +
    +
    + +

    ◆ gcm256_dec_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm256_dec_finalize
    +
    + +
    +
    + +

    ◆ gcm256_dec_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm256_dec_update
    +
    + +
    +
    + +

    ◆ gcm256_enc

    + +
    +
    + + + + +
    aes_gcm_enc_dec_t IMB_MGR::gcm256_enc
    +
    + +
    +
    + +

    ◆ gcm256_enc_finalize

    + +
    +
    + + + + +
    aes_gcm_enc_dec_finalize_t IMB_MGR::gcm256_enc_finalize
    +
    + +
    +
    + +

    ◆ gcm256_enc_update

    + +
    +
    + + + + +
    aes_gcm_enc_dec_update_t IMB_MGR::gcm256_enc_update
    +
    + +
    +
    + +

    ◆ gcm256_init

    + +
    +
    + + + + +
    aes_gcm_init_t IMB_MGR::gcm256_init
    +
    + +
    +
    + +

    ◆ gcm256_init_var_iv

    + +
    +
    + + + + +
    aes_gcm_init_var_iv_t IMB_MGR::gcm256_init_var_iv
    +
    + +
    +
    + +

    ◆ gcm256_pre

    + +
    +
    + + + + +
    aes_gcm_pre_t IMB_MGR::gcm256_pre
    +
    + +
    +
    + +

    ◆ gcm256_precomp

    + +
    +
    + + + + +
    aes_gcm_precomp_t IMB_MGR::gcm256_precomp
    +
    + +
    +
    + +

    ◆ get_completed_job

    + +
    +
    + + + + +
    get_completed_job_t IMB_MGR::get_completed_job
    +
    + +
    +
    + +

    ◆ get_next_job

    + +
    +
    + + + + +
    get_next_job_t IMB_MGR::get_next_job
    +
    +

    ARCH handlers / API Careful as changes here can break ABI compatibility (always include function pointers at the end of the list, before "earliest_job")

    + +
    +
    + +

    ◆ ghash

    + +
    +
    + + + + +
    ghash_t IMB_MGR::ghash
    +
    + +
    +
    + +

    ◆ ghash_pre

    + +
    +
    + + + + +
    aes_gcm_pre_t IMB_MGR::ghash_pre
    +
    + +
    +
    + +

    ◆ gmac128_finalize

    + +
    +
    + + + + +
    aes_gmac_finalize_t IMB_MGR::gmac128_finalize
    +
    + +
    +
    + +

    ◆ gmac128_init

    + +
    +
    + + + + +
    aes_gmac_init_t IMB_MGR::gmac128_init
    +
    + +
    +
    + +

    ◆ gmac128_update

    + +
    +
    + + + + +
    aes_gmac_update_t IMB_MGR::gmac128_update
    +
    + +
    +
    + +

    ◆ gmac192_finalize

    + +
    +
    + + + + +
    aes_gmac_finalize_t IMB_MGR::gmac192_finalize
    +
    + +
    +
    + +

    ◆ gmac192_init

    + +
    +
    + + + + +
    aes_gmac_init_t IMB_MGR::gmac192_init
    +
    + +
    +
    + +

    ◆ gmac192_update

    + +
    +
    + + + + +
    aes_gmac_update_t IMB_MGR::gmac192_update
    +
    + +
    +
    + +

    ◆ gmac256_finalize

    + +
    +
    + + + + +
    aes_gmac_finalize_t IMB_MGR::gmac256_finalize
    +
    + +
    +
    + +

    ◆ gmac256_init

    + +
    +
    + + + + +
    aes_gmac_init_t IMB_MGR::gmac256_init
    +
    + +
    +
    + +

    ◆ gmac256_update

    + +
    +
    + + + + +
    aes_gmac_update_t IMB_MGR::gmac256_update
    +
    + +
    +
    + +

    ◆ hec_32

    + +
    +
    + + + + +
    hec_32_t IMB_MGR::hec_32
    +
    + +
    +
    + +

    ◆ hec_64

    + +
    +
    + + + + +
    hec_64_t IMB_MGR::hec_64
    +
    + +
    +
    + +

    ◆ hmac_md5_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_md5_ooo
    +
    + +
    +
    + +

    ◆ hmac_sha_1_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_sha_1_ooo
    +
    + +
    +
    + +

    ◆ hmac_sha_224_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_sha_224_ooo
    +
    + +
    +
    + +

    ◆ hmac_sha_256_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_sha_256_ooo
    +
    + +
    +
    + +

    ◆ hmac_sha_384_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_sha_384_ooo
    +
    + +
    +
    + +

    ◆ hmac_sha_512_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::hmac_sha_512_ooo
    +
    + +
    +
    + +

    ◆ imb_errno

    + +
    +
    + + + + +
    int IMB_MGR::imb_errno
    +
    +

    per mb_mgr error status

    + +
    +
    + +

    ◆ jobs

    + +
    +
    + + + + +
    IMB_JOB IMB_MGR::jobs[IMB_MAX_JOBS]
    +
    + +
    +
    + +

    ◆ kasumi_init_f8_key_sched

    + +
    +
    + + + + +
    kasumi_init_f8_key_sched_t IMB_MGR::kasumi_init_f8_key_sched
    +
    + +
    +
    + +

    ◆ kasumi_init_f9_key_sched

    + +
    +
    + + + + +
    kasumi_init_f9_key_sched_t IMB_MGR::kasumi_init_f9_key_sched
    +
    + +
    +
    + +

    ◆ kasumi_key_sched_size

    + +
    +
    + + + + +
    kasumi_key_sched_size_t IMB_MGR::kasumi_key_sched_size
    +
    + +
    +
    + +

    ◆ keyexp_128

    + +
    +
    + + + + +
    keyexp_t IMB_MGR::keyexp_128
    +
    + +
    +
    + +

    ◆ keyexp_192

    + +
    +
    + + + + +
    keyexp_t IMB_MGR::keyexp_192
    +
    + +
    +
    + +

    ◆ keyexp_256

    + +
    +
    + + + + +
    keyexp_t IMB_MGR::keyexp_256
    +
    + +
    +
    + +

    ◆ md5_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::md5_one_block
    +
    + +
    +
    + +

    ◆ next_job

    + +
    +
    + + + + +
    int IMB_MGR::next_job
    +
    +

    byte offset

    + +
    +
    + +

    ◆ queue_size

    + +
    +
    + + + + +
    queue_size_t IMB_MGR::queue_size
    +
    + +
    +
    + +

    ◆ reserved

    + +
    +
    + + + + +
    uint64_t IMB_MGR::reserved[5]
    +
    +

    reserved for the future

    + +
    +
    + +

    ◆ sha1

    + +
    +
    + + + + +
    hash_fn_t IMB_MGR::sha1
    +
    + +
    +
    + +

    ◆ sha1_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::sha1_one_block
    +
    + +
    +
    + +

    ◆ sha224

    + +
    +
    + + + + +
    hash_fn_t IMB_MGR::sha224
    +
    + +
    +
    + +

    ◆ sha224_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::sha224_one_block
    +
    + +
    +
    + +

    ◆ sha256

    + +
    +
    + + + + +
    hash_fn_t IMB_MGR::sha256
    +
    + +
    +
    + +

    ◆ sha256_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::sha256_one_block
    +
    + +
    +
    + +

    ◆ sha384

    + +
    +
    + + + + +
    hash_fn_t IMB_MGR::sha384
    +
    + +
    +
    + +

    ◆ sha384_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::sha384_one_block
    +
    + +
    +
    + +

    ◆ sha512

    + +
    +
    + + + + +
    hash_fn_t IMB_MGR::sha512
    +
    + +
    +
    + +

    ◆ sha512_one_block

    + +
    +
    + + + + +
    hash_one_block_t IMB_MGR::sha512_one_block
    +
    + +
    +
    + +

    ◆ sha_1_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::sha_1_ooo
    +
    + +
    +
    + +

    ◆ sha_224_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::sha_224_ooo
    +
    + +
    +
    + +

    ◆ sha_256_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::sha_256_ooo
    +
    + +
    +
    + +

    ◆ sha_384_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::sha_384_ooo
    +
    + +
    +
    + +

    ◆ sha_512_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::sha_512_ooo
    +
    + +
    +
    + +

    ◆ snow3g_f8_1_buffer

    + +
    +
    + + + + +
    snow3g_f8_1_buffer_t IMB_MGR::snow3g_f8_1_buffer
    +
    + +
    +
    + +

    ◆ snow3g_f8_1_buffer_bit

    + +
    +
    + + + + +
    snow3g_f8_1_buffer_bit_t IMB_MGR::snow3g_f8_1_buffer_bit
    +
    + +
    +
    + +

    ◆ snow3g_f8_2_buffer

    + +
    +
    + + + + +
    snow3g_f8_2_buffer_t IMB_MGR::snow3g_f8_2_buffer
    +
    + +
    +
    + +

    ◆ snow3g_f8_4_buffer

    + +
    +
    + + + + +
    snow3g_f8_4_buffer_t IMB_MGR::snow3g_f8_4_buffer
    +
    + +
    +
    + +

    ◆ snow3g_f8_8_buffer

    + +
    +
    + + + + +
    snow3g_f8_8_buffer_t IMB_MGR::snow3g_f8_8_buffer
    +
    + +
    +
    + +

    ◆ snow3g_f8_8_buffer_multikey

    + +
    +
    + + + + +
    snow3g_f8_8_buffer_multikey_t IMB_MGR::snow3g_f8_8_buffer_multikey
    +
    + +
    +
    + +

    ◆ snow3g_f8_n_buffer

    + +
    +
    + + + + +
    snow3g_f8_n_buffer_t IMB_MGR::snow3g_f8_n_buffer
    +
    + +
    +
    + +

    ◆ snow3g_f8_n_buffer_multikey

    + +
    +
    + + + + +
    snow3g_f8_n_buffer_multikey_t IMB_MGR::snow3g_f8_n_buffer_multikey
    +
    + +
    +
    + +

    ◆ snow3g_f9_1_buffer

    + +
    +
    + + + + +
    snow3g_f9_1_buffer_t IMB_MGR::snow3g_f9_1_buffer
    +
    + +
    +
    + +

    ◆ snow3g_init_key_sched

    + +
    +
    + + + + +
    snow3g_init_key_sched_t IMB_MGR::snow3g_init_key_sched
    +
    + +
    +
    + +

    ◆ snow3g_key_sched_size

    + +
    +
    + + + + +
    snow3g_key_sched_size_t IMB_MGR::snow3g_key_sched_size
    +
    + +
    +
    + +

    ◆ snow3g_uea2_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::snow3g_uea2_ooo
    +
    + +
    +
    + +

    ◆ snow3g_uia2_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::snow3g_uia2_ooo
    +
    + +
    +
    + +

    ◆ submit_burst

    + +
    +
    + + + + +
    submit_burst_t IMB_MGR::submit_burst
    +
    + +
    +
    + +

    ◆ submit_burst_nocheck

    + +
    +
    + + + + +
    submit_burst_t IMB_MGR::submit_burst_nocheck
    +
    + +
    +
    + +

    ◆ submit_cipher_burst

    + +
    +
    + + + + +
    submit_cipher_burst_t IMB_MGR::submit_cipher_burst
    +
    + +
    +
    + +

    ◆ submit_cipher_burst_nocheck

    + +
    +
    + + + + +
    submit_cipher_burst_t IMB_MGR::submit_cipher_burst_nocheck
    +
    + +
    +
    + +

    ◆ submit_hash_burst

    + +
    +
    + + + + +
    submit_hash_burst_t IMB_MGR::submit_hash_burst
    +
    + +
    +
    + +

    ◆ submit_hash_burst_nocheck

    + +
    +
    + + + + +
    submit_hash_burst_t IMB_MGR::submit_hash_burst_nocheck
    +
    + +
    +
    + +

    ◆ submit_job

    + +
    +
    + + + + +
    submit_job_t IMB_MGR::submit_job
    +
    + +
    +
    + +

    ◆ submit_job_nocheck

    + +
    +
    + + + + +
    submit_job_t IMB_MGR::submit_job_nocheck
    +
    + +
    +
    + +

    ◆ used_arch

    + +
    +
    + + + + +
    uint32_t IMB_MGR::used_arch
    +
    +

    Architecture being used

    + +
    +
    + +

    ◆ xcbc_keyexp

    + +
    +
    + + + + +
    xcbc_keyexp_t IMB_MGR::xcbc_keyexp
    +
    + +
    +
    + +

    ◆ zuc256_eea3_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::zuc256_eea3_ooo
    +
    + +
    +
    + +

    ◆ zuc256_eia3_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::zuc256_eia3_ooo
    +
    + +
    +
    + +

    ◆ zuc_eea3_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::zuc_eea3_ooo
    +
    + +
    +
    + +

    ◆ zuc_eia3_ooo

    + +
    +
    + + + + +
    void* IMB_MGR::zuc_eia3_ooo
    +
    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structIMB__MGR.js b/docs/structIMB__MGR.js new file mode 100644 index 00000000..0dadc747 --- /dev/null +++ b/docs/structIMB__MGR.js @@ -0,0 +1,165 @@ +var structIMB__MGR = +[ + [ "aes128_cbcs_ooo", "structIMB__MGR.html#a9c72f3341a9a4837595338d06c3eda5c", null ], + [ "aes128_cfb_one", "structIMB__MGR.html#abd468de03563a40da0b21b0885f44cbe", null ], + [ "aes128_ooo", "structIMB__MGR.html#a97fd05a4057fc898f647955a7ba8501b", null ], + [ "aes192_ooo", "structIMB__MGR.html#afe48187946dff59f6c69103501691921", null ], + [ "aes256_ccm_ooo", "structIMB__MGR.html#a04c28bfe763544b68dac98e23aa0a8d9", null ], + [ "aes256_cmac_ooo", "structIMB__MGR.html#acbd18bab8176262daab2bd808ca30616", null ], + [ "aes256_ooo", "structIMB__MGR.html#a6abbf129001cb44003a10105115e52b7", null ], + [ "aes_ccm_ooo", "structIMB__MGR.html#a3eeeead99056e46d8cb1ed316f587aaf", null ], + [ "aes_cmac_ooo", "structIMB__MGR.html#a0bd2acac3a6c3f90a998ed53a7d28e57", null ], + [ "aes_xcbc_ooo", "structIMB__MGR.html#ad554a25793a904ae6cf7ca87250adbd3", null ], + [ "chacha20_poly1305_dec_update", "structIMB__MGR.html#a715aede4176564ac475e42c8a377bd8d", null ], + [ "chacha20_poly1305_enc_update", "structIMB__MGR.html#ab2c87c40f9844795e3987536c602ce03", null ], + [ "chacha20_poly1305_finalize", "structIMB__MGR.html#ab18b3fcba2aa252bb31f66093bc62abb", null ], + [ "chacha20_poly1305_init", "structIMB__MGR.html#a67b96b1b26fa25391f9f4b83a7a4b6c0", null ], + [ "cmac_subkey_gen_128", "structIMB__MGR.html#aa3404b7811d57821ea574994e0c88995", null ], + [ "cmac_subkey_gen_256", "structIMB__MGR.html#abd500d2cc03f6470ac6988785bebdf76", null ], + [ "crc10_iuup_data", "structIMB__MGR.html#a7d0979deda4e2ff7ac202c82611ce94c", null ], + [ "crc11_fp_header", "structIMB__MGR.html#a4b901916c2f262f1c873e4cec7b7f474", null ], + [ "crc16_fp_data", "structIMB__MGR.html#ad566ac43002c2ba8ecb93048d5edc41b", null ], + [ "crc16_x25", "structIMB__MGR.html#a52a7ab10d6434820e9b31d0e772ea0bc", null ], + [ "crc24_lte_a", "structIMB__MGR.html#a058cfc60f2fdd68678b36f4f038270ac", null ], + [ "crc24_lte_b", "structIMB__MGR.html#abe8406f3582d3540edb770e153b74c3e", null ], + [ "crc32_ethernet_fcs", "structIMB__MGR.html#a8483109fbe42f8b7fa6e7f53e016d5ec", null ], + [ "crc32_sctp", "structIMB__MGR.html#a4aed66c9e73808a65a062e1a5b0f66aa", null ], + [ "crc32_wimax_ofdma_data", "structIMB__MGR.html#ad08e44bc4edc54c556ea2245f848b974", null ], + [ "crc6_iuup_header", "structIMB__MGR.html#aa610dc7226673020be410762b995b725", null ], + [ "crc7_fp_header", "structIMB__MGR.html#ac819ddcc968604186983d58f275bd24b", null ], + [ "crc8_wimax_ofdma_hcs", "structIMB__MGR.html#a5e5be572c46ed01ac11fc95738373e2a", null ], + [ "des3_dec_ooo", "structIMB__MGR.html#a2ab69fdbef6a9a0574fcbb5ea9a7da3b", null ], + [ "des3_enc_ooo", "structIMB__MGR.html#a87f7566634911b4dba48a19a0c3a4993", null ], + [ "des_dec_ooo", "structIMB__MGR.html#af03e401b002e2e415c8a927c46e1517b", null ], + [ "des_enc_ooo", "structIMB__MGR.html#a54ec50411917f1aa9016db268ac638c1", null ], + [ "des_key_sched", "structIMB__MGR.html#aae9afac5c2c1bf7b27d187d10778a1ee", null ], + [ "docsis128_crc32_sec_ooo", "structIMB__MGR.html#acf438b50cb95a4fff1276b6666d22ad4", null ], + [ "docsis128_sec_ooo", "structIMB__MGR.html#a93cb5904608e179c49917f80c64fabd1", null ], + [ "docsis256_crc32_sec_ooo", "structIMB__MGR.html#af6531598b8dd0131a11e0cf95854605d", null ], + [ "docsis256_sec_ooo", "structIMB__MGR.html#acb6e3b70e16194274e2eaefdff8f9e89", null ], + [ "docsis_des_dec_ooo", "structIMB__MGR.html#a3d5173cea3d4932576f0a5b3abbc1f40", null ], + [ "docsis_des_enc_ooo", "structIMB__MGR.html#a937fb0d3b113d203ee1a4beaf5b5b98f", null ], + [ "earliest_job", "structIMB__MGR.html#ac5cd33ac3bf985dc31ccc716fb3a7747", null ], + [ "eea3_1_buffer", "structIMB__MGR.html#aeff54b370fbe495ce96370811d280140", null ], + [ "eea3_4_buffer", "structIMB__MGR.html#ac5ad1bf8ecdcfc51eaf681fb353c9621", null ], + [ "eea3_n_buffer", "structIMB__MGR.html#a67d8a55ba87742b8cf282d23ed9e3ef5", null ], + [ "eia3_1_buffer", "structIMB__MGR.html#a3ed58bdf37ab6d5f0d28271882d52327", null ], + [ "eia3_n_buffer", "structIMB__MGR.html#ab515c5aec39b1a9a31454b945e5ccd35", null ], + [ "end_ooo", "structIMB__MGR.html#a6e1ab772182b97f931af3d4685785353", null ], + [ "f8_1_buffer", "structIMB__MGR.html#af854e664cef84cb8be4495a1af5e9cb5", null ], + [ "f8_1_buffer_bit", "structIMB__MGR.html#aacf6ce3f508fef115dcd2dc415ca7379", null ], + [ "f8_2_buffer", "structIMB__MGR.html#ac733fc4c5ee6e80e8e71b20a9bb2dc6e", null ], + [ "f8_3_buffer", "structIMB__MGR.html#a929b9c50bc98bf37daec7a1603f10118", null ], + [ "f8_4_buffer", "structIMB__MGR.html#af0b1c5d3b269b252a75a55df8bdf472c", null ], + [ "f8_n_buffer", "structIMB__MGR.html#ae0f2316d96e46b64d08e3cf73a93b39d", null ], + [ "f9_1_buffer", "structIMB__MGR.html#afafb95a00fe6633fb6901855928b08bb", null ], + [ "f9_1_buffer_user", "structIMB__MGR.html#a4fbdac066c9d0fbd1d586a2e5b2319c3", null ], + [ "features", "structIMB__MGR.html#af877c7669265775604857c6b81063bdb", null ], + [ "flags", "structIMB__MGR.html#ae1a596c9c9d6ca1292c6116575021c14", null ], + [ "flush_job", "structIMB__MGR.html#a225e3b35eef8e18fd09c429fbb2f70b5", null ], + [ "gcm128_dec", "structIMB__MGR.html#a2ce2f6e6d61953b24a5859279534b02a", null ], + [ "gcm128_dec_finalize", "structIMB__MGR.html#af95fa831fb24629e19613e8594e5d32a", null ], + [ "gcm128_dec_update", "structIMB__MGR.html#a28ddfed1a807fedb9c77c99736278dcb", null ], + [ "gcm128_enc", "structIMB__MGR.html#a9f8b9687fbb17f7a78fc4ba79e2749fd", null ], + [ "gcm128_enc_finalize", "structIMB__MGR.html#a8efc0cf95907805289a297c024ce0cac", null ], + [ "gcm128_enc_update", "structIMB__MGR.html#ad2c64d9d5f9ffd65d13e73205cb803cd", null ], + [ "gcm128_init", "structIMB__MGR.html#ad8c004899d29974a3f4c034370422c31", null ], + [ "gcm128_init_var_iv", "structIMB__MGR.html#aebd9eab76cd056add559e6bea328dbf0", null ], + [ "gcm128_pre", "structIMB__MGR.html#a207c78e01e4aa1a35de1bbd2cc0a6ee2", null ], + [ "gcm128_precomp", "structIMB__MGR.html#a69e4dd044703cd3aa9c4c36e04b2a701", null ], + [ "gcm192_dec", "structIMB__MGR.html#adff2cf46e7cbb5e2926f348623d8ad2a", null ], + [ "gcm192_dec_finalize", "structIMB__MGR.html#a4d7879172344be144b92be68b2e2a81a", null ], + [ "gcm192_dec_update", "structIMB__MGR.html#a66fa94f9c07061f3d93b78ed505931e3", null ], + [ "gcm192_enc", "structIMB__MGR.html#aaaeb1676e0cf19e3558d10e537f71bda", null ], + [ "gcm192_enc_finalize", "structIMB__MGR.html#a91ad9ce7a4b3c9ceeffa9dd712013915", null ], + [ "gcm192_enc_update", "structIMB__MGR.html#a3b5c3407cc4595420dbe43e98090e558", null ], + [ "gcm192_init", "structIMB__MGR.html#a006315a7045996d451c7dfd03b96a076", null ], + [ "gcm192_init_var_iv", "structIMB__MGR.html#a7bfa7b17c0fb36dc832138a86948991f", null ], + [ "gcm192_pre", "structIMB__MGR.html#a149b463ecf68ee78d39c33e183ed8997", null ], + [ "gcm192_precomp", "structIMB__MGR.html#abec8bdb319ca132028d5b787bf60a895", null ], + [ "gcm256_dec", "structIMB__MGR.html#acfdfb511771cac71e032c5bf1c9273b7", null ], + [ "gcm256_dec_finalize", "structIMB__MGR.html#a5c2e4a1296e33f5c9eaff2067a6432d4", null ], + [ "gcm256_dec_update", "structIMB__MGR.html#a99fcb1d18fa45044063c6c288b0fc769", null ], + [ "gcm256_enc", "structIMB__MGR.html#a79767326692a8d4faabe5ab435dc5df8", null ], + [ "gcm256_enc_finalize", "structIMB__MGR.html#a9764d27efef2491a86d517f38dd9b6ef", null ], + [ "gcm256_enc_update", "structIMB__MGR.html#ac131ceec0dcb0d08097b76c8abe9e024", null ], + [ "gcm256_init", "structIMB__MGR.html#a8987d7d491628d9d80f12820468ac541", null ], + [ "gcm256_init_var_iv", "structIMB__MGR.html#afa275308dab467d0c68dc921bd68d1a4", null ], + [ "gcm256_pre", "structIMB__MGR.html#a44b0b9d218043be965a84c29ced2584b", null ], + [ "gcm256_precomp", "structIMB__MGR.html#a1ea15abc11b618ceb438c7f282643c52", null ], + [ "get_completed_job", "structIMB__MGR.html#a99b7111c8fc59161e0bf291d13b1f195", null ], + [ "get_next_job", "structIMB__MGR.html#a35939892cd168eb7ce9c6c29c5401709", null ], + [ "ghash", "structIMB__MGR.html#afc7d7ae10e8e6f37b9ec894059818fad", null ], + [ "ghash_pre", "structIMB__MGR.html#af6a1c323981b8c3cfe09f94cd9dd58e2", null ], + [ "gmac128_finalize", "structIMB__MGR.html#a00ce9a8422a961166ce9855d4815cc2e", null ], + [ "gmac128_init", "structIMB__MGR.html#a1098c38784f6c6e4e3a3380f8be8ecc1", null ], + [ "gmac128_update", "structIMB__MGR.html#a90aad4f47448a1959b7cf747438d7ce9", null ], + [ "gmac192_finalize", "structIMB__MGR.html#a000b24d7fbb9b57ed6bd4147d225e015", null ], + [ "gmac192_init", "structIMB__MGR.html#a3dd58a9d9b4fb6cf85fa26175ffac9a6", null ], + [ "gmac192_update", "structIMB__MGR.html#a5220f0d243291ae5a2f43fbc93d18725", null ], + [ "gmac256_finalize", "structIMB__MGR.html#a3bb98c9a9c8545f363b74627870c7038", null ], + [ "gmac256_init", "structIMB__MGR.html#aca7d6388863eb76b60651e7b16c41b82", null ], + [ "gmac256_update", "structIMB__MGR.html#a0720a1fc9c0652c6b3aa7dfab070f73d", null ], + [ "hec_32", "structIMB__MGR.html#a0863c9f64126aba5cb47628493dae582", null ], + [ "hec_64", "structIMB__MGR.html#a4128ee02f4cbef5aa1e6b0bb5fd46097", null ], + [ "hmac_md5_ooo", "structIMB__MGR.html#a855d14df536b22ceb175dc495ced891c", null ], + [ "hmac_sha_1_ooo", "structIMB__MGR.html#a49be1d77b96d338c08200dce82e452d3", null ], + [ "hmac_sha_224_ooo", "structIMB__MGR.html#ab9226cf37e41c2a13ae6003305654193", null ], + [ "hmac_sha_256_ooo", "structIMB__MGR.html#a633807f7eaae35ada650a2c22f1366be", null ], + [ "hmac_sha_384_ooo", "structIMB__MGR.html#afd72d6af1d8ff605e8aaaf984f0700ed", null ], + [ "hmac_sha_512_ooo", "structIMB__MGR.html#af0cf9149495b0805a4f6f511c0fbff5b", null ], + [ "imb_errno", "structIMB__MGR.html#a0ab7a3dfdfd8442f591699e2310df11f", null ], + [ "jobs", "structIMB__MGR.html#ab342224dce812fb13a9e935e7a7b0617", null ], + [ "kasumi_init_f8_key_sched", "structIMB__MGR.html#ab32d67f27cb6f06980080e09db133b24", null ], + [ "kasumi_init_f9_key_sched", "structIMB__MGR.html#a58ad87104b08ba5b56fc65928677a639", null ], + [ "kasumi_key_sched_size", "structIMB__MGR.html#abd0c855a3e43df48f5f21686a095f358", null ], + [ "keyexp_128", "structIMB__MGR.html#a7c97c5361b90abe325db55530b6f198e", null ], + [ "keyexp_192", "structIMB__MGR.html#a2bd7444a318de06b5f53178abf9c37ff", null ], + [ "keyexp_256", "structIMB__MGR.html#a431e99ddb3837a32bedb6cd426054bcc", null ], + [ "md5_one_block", "structIMB__MGR.html#a54b5f7b625ce226ebd46a2a19e9bc724", null ], + [ "next_job", "structIMB__MGR.html#a106ebd60d876e56bfc6bcc03d20acce4", null ], + [ "queue_size", "structIMB__MGR.html#a5a35c9c2a1c8eb5b4aa239a3b04f4fbf", null ], + [ "reserved", "structIMB__MGR.html#a136697eb37d5a06eeada0f338abd8b51", null ], + [ "sha1", "structIMB__MGR.html#a82aaedc800096aaa55062c56131e0b13", null ], + [ "sha1_one_block", "structIMB__MGR.html#a80a26be9a12f561a68b70d65e096cde9", null ], + [ "sha224", "structIMB__MGR.html#a52e26b8805b28d1298ac48e5762412b5", null ], + [ "sha224_one_block", "structIMB__MGR.html#aacf8209094d12ceceed96d4d0fae193c", null ], + [ "sha256", "structIMB__MGR.html#a8d978764a953e220b3c799990bae490b", null ], + [ "sha256_one_block", "structIMB__MGR.html#ade4a72c39980f0bba790b70529b60158", null ], + [ "sha384", "structIMB__MGR.html#a9ffcce0ba9aef3cd68bdd7f6e9e3e2a0", null ], + [ "sha384_one_block", "structIMB__MGR.html#a92b7ff145f0016dd46eabf9271ef03f1", null ], + [ "sha512", "structIMB__MGR.html#a62ebdd5e2b5a59c9e648511caf56f3f1", null ], + [ "sha512_one_block", "structIMB__MGR.html#a88f1060a2adb8098fc085fc2b812addc", null ], + [ "sha_1_ooo", "structIMB__MGR.html#acdb53e812f3e4a307a497eeab28c4256", null ], + [ "sha_224_ooo", "structIMB__MGR.html#aa174bf6846067a48f3f47ee5ebd74877", null ], + [ "sha_256_ooo", "structIMB__MGR.html#a37fc5ccae7af099a8517566a20751265", null ], + [ "sha_384_ooo", "structIMB__MGR.html#a51a499114f84eed88cb48bd3b461bd1c", null ], + [ "sha_512_ooo", "structIMB__MGR.html#a6e21e034cd011b695bac5a711be65da1", null ], + [ "snow3g_f8_1_buffer", "structIMB__MGR.html#ab70cbec09a61b5adeae37260c10234ce", null ], + [ "snow3g_f8_1_buffer_bit", "structIMB__MGR.html#ad85fdcb101391b0a2488a3244d5f442d", null ], + [ "snow3g_f8_2_buffer", "structIMB__MGR.html#a25f46509b2f8926902a73e2210d697fd", null ], + [ "snow3g_f8_4_buffer", "structIMB__MGR.html#a194a0f15a7345ee369780dbdeb4f52f6", null ], + [ "snow3g_f8_8_buffer", "structIMB__MGR.html#a620d2d75bb50784aa0000b458197afaf", null ], + [ "snow3g_f8_8_buffer_multikey", "structIMB__MGR.html#a898a4c56e1a746a328529c3665c5682e", null ], + [ "snow3g_f8_n_buffer", "structIMB__MGR.html#a829c4cf1157dfc3fcb829df31e408e9d", null ], + [ "snow3g_f8_n_buffer_multikey", "structIMB__MGR.html#a63a2a5f74e7cc2fc637896bdaeb6b262", null ], + [ "snow3g_f9_1_buffer", "structIMB__MGR.html#a9a4d2148b6e8b4426b2946e9fe695461", null ], + [ "snow3g_init_key_sched", "structIMB__MGR.html#a38ff0528815be4d92481948664366ca9", null ], + [ "snow3g_key_sched_size", "structIMB__MGR.html#a5e0002330399f643e8f0ca0c1b96a7a0", null ], + [ "snow3g_uea2_ooo", "structIMB__MGR.html#ab0193bf46978026a82803c25e11acbfb", null ], + [ "snow3g_uia2_ooo", "structIMB__MGR.html#aa7d4cfe53e0d70c22d3a274b8cd218af", null ], + [ "submit_burst", "structIMB__MGR.html#a6a790b2aa04138fce78c3347e285a31b", null ], + [ "submit_burst_nocheck", "structIMB__MGR.html#ad16230a6a671adbd597675abbc0f0a87", null ], + [ "submit_cipher_burst", "structIMB__MGR.html#a8b52f1b3ca92eff48b945929a0eab839", null ], + [ "submit_cipher_burst_nocheck", "structIMB__MGR.html#acb2ee84d30247bb257635063db3ae308", null ], + [ "submit_hash_burst", "structIMB__MGR.html#a20a3d4d1e40aed26d83db5395b7ccc83", null ], + [ "submit_hash_burst_nocheck", "structIMB__MGR.html#a809782eb9dcbd264442bcd9edd2f4c2f", null ], + [ "submit_job", "structIMB__MGR.html#ab143534d96e0d585b19dfc19553e929b", null ], + [ "submit_job_nocheck", "structIMB__MGR.html#aab7350eeecc56b1ef45606b83519a72c", null ], + [ "used_arch", "structIMB__MGR.html#a44b6c95f8ce611828b6acacdb459cdc9", null ], + [ "xcbc_keyexp", "structIMB__MGR.html#ae575c2350f5fca9a182c541795bb7d4d", null ], + [ "zuc256_eea3_ooo", "structIMB__MGR.html#a6d05b0a20196be514182920710dce2a8", null ], + [ "zuc256_eia3_ooo", "structIMB__MGR.html#a5f8744a8d64273d3f4cbcd9ad41ce3d5", null ], + [ "zuc_eea3_ooo", "structIMB__MGR.html#a26d22c5ff963b9c7ffc8af19292bce5c", null ], + [ "zuc_eia3_ooo", "structIMB__MGR.html#a910f747aa1a0e6a46f87b0dfd2189219", null ] +]; \ No newline at end of file diff --git a/docs/structIMB__SGL__IOV.html b/docs/structIMB__SGL__IOV.html new file mode 100644 index 00000000..e5130860 --- /dev/null +++ b/docs/structIMB__SGL__IOV.html @@ -0,0 +1,172 @@ + + + + + + + +intel-ipsec-mb: IMB_SGL_IOV Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    IMB_SGL_IOV Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + +

    +Data Fields

    const void * in
     
    void * out
     
    uint64_t len
     
    +

    Detailed Description

    +

    Input/output SGL segment structure.

    +

    Field Documentation

    + +

    ◆ in

    + +
    +
    + + + + +
    const void* IMB_SGL_IOV::in
    +
    +

    Input segment

    + +
    +
    + +

    ◆ len

    + +
    +
    + + + + +
    uint64_t IMB_SGL_IOV::len
    +
    + +
    +
    + +

    ◆ out

    + +
    +
    + + + + +
    void* IMB_SGL_IOV::out
    +
    +

    Output segment

    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structIMB__SGL__IOV.js b/docs/structIMB__SGL__IOV.js new file mode 100644 index 00000000..748257c9 --- /dev/null +++ b/docs/structIMB__SGL__IOV.js @@ -0,0 +1,6 @@ +var structIMB__SGL__IOV = +[ + [ "in", "structIMB__SGL__IOV.html#a56c5b0a033d949869cee2d03f1df3129", null ], + [ "len", "structIMB__SGL__IOV.html#a3a3662456b91011eb0585667492acc88", null ], + [ "out", "structIMB__SGL__IOV.html#a54b8da76fb8a400f9f7972bdd97f6f53", null ] +]; \ No newline at end of file diff --git a/docs/structchacha20__poly1305__context__data.html b/docs/structchacha20__poly1305__context__data.html new file mode 100644 index 00000000..9a1276f0 --- /dev/null +++ b/docs/structchacha20__poly1305__context__data.html @@ -0,0 +1,295 @@ + + + + + + + +intel-ipsec-mb: chacha20_poly1305_context_data Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    chacha20_poly1305_context_data Struct Reference
    +
    +
    + +

    holds Chacha20-Poly1305 operation context + More...

    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + + + + + + + + + + + + + + + +

    +Data Fields

    uint64_t hash [3]
     
    uint64_t aad_len
     
    uint64_t hash_len
     
    uint8_t last_ks [64]
     
    uint8_t poly_key [32]
     
    uint8_t poly_scratch [16]
     
    uint64_t last_block_count
     
    uint64_t remain_ks_bytes
     
    uint64_t remain_ct_bytes
     
    uint8_t IV [12]
     
    +

    Detailed Description

    +

    holds Chacha20-Poly1305 operation context

    +

    Field Documentation

    + +

    ◆ aad_len

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::aad_len
    +
    +

    Total AAD length

    + +
    +
    + +

    ◆ hash

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::hash[3]
    +
    +

    Intermediate computation of hash value

    + +
    +
    + +

    ◆ hash_len

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::hash_len
    +
    +

    Total length to digest (excluding AAD)

    + +
    +
    + +

    ◆ IV

    + +
    +
    + + + + +
    uint8_t chacha20_poly1305_context_data::IV[12]
    +
    +

    IV (12 bytes)

    + +
    +
    + +

    ◆ last_block_count

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::last_block_count
    +
    +

    Last block count used in last segment

    + +
    +
    + +

    ◆ last_ks

    + +
    +
    + + + + +
    uint8_t chacha20_poly1305_context_data::last_ks[64]
    +
    +

    Last 64 bytes of KS

    + +
    +
    + +

    ◆ poly_key

    + +
    +
    + + + + +
    uint8_t chacha20_poly1305_context_data::poly_key[32]
    +
    +

    Poly key

    + +
    +
    + +

    ◆ poly_scratch

    + +
    +
    + + + + +
    uint8_t chacha20_poly1305_context_data::poly_scratch[16]
    +
    +

    Scratchpad to compute Poly on 16 bytes

    + +
    +
    + +

    ◆ remain_ct_bytes

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::remain_ct_bytes
    +
    +

    Amount of ciphertext bytes still to use of previous segment to authenticate (up to 16 bytes)

    + +
    +
    + +

    ◆ remain_ks_bytes

    + +
    +
    + + + + +
    uint64_t chacha20_poly1305_context_data::remain_ks_bytes
    +
    +

    Amount of bytes still to use of keystream (up to 63 bytes)

    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structchacha20__poly1305__context__data.js b/docs/structchacha20__poly1305__context__data.js new file mode 100644 index 00000000..4c9d8fce --- /dev/null +++ b/docs/structchacha20__poly1305__context__data.js @@ -0,0 +1,13 @@ +var structchacha20__poly1305__context__data = +[ + [ "aad_len", "structchacha20__poly1305__context__data.html#a7b5edcebc9a6bec3776daf6d13bf950c", null ], + [ "hash", "structchacha20__poly1305__context__data.html#ae12447b2fd2e9f8a6f09f443bd303887", null ], + [ "hash_len", "structchacha20__poly1305__context__data.html#ade4568d041695b318df773787872127f", null ], + [ "IV", "structchacha20__poly1305__context__data.html#ac0d1657fc57308900691369362c56658", null ], + [ "last_block_count", "structchacha20__poly1305__context__data.html#add03511d1f95376b3ac64a7916c27368", null ], + [ "last_ks", "structchacha20__poly1305__context__data.html#afc74af4af99449f0b9f01338ae723c1a", null ], + [ "poly_key", "structchacha20__poly1305__context__data.html#a9ac4fafe2ca37395264f728c8e99a9b1", null ], + [ "poly_scratch", "structchacha20__poly1305__context__data.html#a908d80ce2c35792099dbc967818b9abd", null ], + [ "remain_ct_bytes", "structchacha20__poly1305__context__data.html#a74e971fe76c8b946e30e8f0c8cce111a", null ], + [ "remain_ks_bytes", "structchacha20__poly1305__context__data.html#a84693b96048f6130a749b633c65e2105", null ] +]; \ No newline at end of file diff --git a/docs/structgcm__context__data.html b/docs/structgcm__context__data.html new file mode 100644 index 00000000..fa2356f6 --- /dev/null +++ b/docs/structgcm__context__data.html @@ -0,0 +1,238 @@ + + + + + + + +intel-ipsec-mb: gcm_context_data Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    gcm_context_data Struct Reference
    +
    +
    + +

    holds GCM operation context + More...

    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + + + + + + + + + +

    +Data Fields

    uint8_t aad_hash [IMB_GCM_BLOCK_LEN]
     
    uint64_t aad_length
     
    uint64_t in_length
     
    uint8_t partial_block_enc_key [IMB_GCM_BLOCK_LEN]
     
    uint8_t orig_IV [IMB_GCM_BLOCK_LEN]
     
    uint8_t current_counter [IMB_GCM_BLOCK_LEN]
     
    uint64_t partial_block_length
     
    +

    Detailed Description

    +

    holds GCM operation context

    +

    init, update and finalize context data

    +

    Field Documentation

    + +

    ◆ aad_hash

    + +
    +
    + + + + +
    uint8_t gcm_context_data::aad_hash[IMB_GCM_BLOCK_LEN]
    +
    + +
    +
    + +

    ◆ aad_length

    + +
    +
    + + + + +
    uint64_t gcm_context_data::aad_length
    +
    + +
    +
    + +

    ◆ current_counter

    + +
    +
    + + + + +
    uint8_t gcm_context_data::current_counter[IMB_GCM_BLOCK_LEN]
    +
    + +
    +
    + +

    ◆ in_length

    + +
    +
    + + + + +
    uint64_t gcm_context_data::in_length
    +
    + +
    +
    + +

    ◆ orig_IV

    + +
    +
    + + + + +
    uint8_t gcm_context_data::orig_IV[IMB_GCM_BLOCK_LEN]
    +
    + +
    +
    + +

    ◆ partial_block_enc_key

    + +
    +
    + + + + +
    uint8_t gcm_context_data::partial_block_enc_key[IMB_GCM_BLOCK_LEN]
    +
    + +
    +
    + +

    ◆ partial_block_length

    + +
    +
    + + + + +
    uint64_t gcm_context_data::partial_block_length
    +
    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structgcm__context__data.js b/docs/structgcm__context__data.js new file mode 100644 index 00000000..7b085fc3 --- /dev/null +++ b/docs/structgcm__context__data.js @@ -0,0 +1,10 @@ +var structgcm__context__data = +[ + [ "aad_hash", "structgcm__context__data.html#a98c22f35eb0b4d368e2f454b347ed3b9", null ], + [ "aad_length", "structgcm__context__data.html#ab1dadabcbd2002738ec7db7da332e04f", null ], + [ "current_counter", "structgcm__context__data.html#a2a454a5e44422aaec135dc7807aa55a4", null ], + [ "in_length", "structgcm__context__data.html#a0d9e72f85aba9363fcb4f2d757342aa2", null ], + [ "orig_IV", "structgcm__context__data.html#a84bac4eabc12b81d26898b6fe6728a63", null ], + [ "partial_block_enc_key", "structgcm__context__data.html#a2e52ea863c681afe1d3370a987e252f0", null ], + [ "partial_block_length", "structgcm__context__data.html#a97411247efd14ab01e7e0da6587bfa0c", null ] +]; \ No newline at end of file diff --git a/docs/structgcm__key__data.html b/docs/structgcm__key__data.html new file mode 100644 index 00000000..219b4497 --- /dev/null +++ b/docs/structgcm__key__data.html @@ -0,0 +1,251 @@ + + + + + + + +intel-ipsec-mb: gcm_key_data Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    gcm_key_data Struct Reference
    +
    +
    + +

    holds intermediate key data needed to improve performance + More...

    + +

    #include <intel-ipsec-mb.h>

    + + + + + + + + + + + + + + + + + + + + + + + + +

    +Data Fields

    uint8_t expanded_keys [IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
     
    union {
       struct {
          uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *8]
     
          uint8_t   shifted_hkey_k [IMB_GCM_ENC_KEY_LEN *8]
     
       }   sse_avx
     
       struct {
          uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *8]
     
       }   avx2_avx512
     
       struct {
          uint8_t   shifted_hkey [IMB_GCM_ENC_KEY_LEN *48]
     
       }   vaes_avx512
     
    } ghash_keys
     
    +

    Detailed Description

    +

    holds intermediate key data needed to improve performance

    +

    gcm_key_data hold internal key information used by gcm128, gcm192 and gcm256.

    +

    Field Documentation

    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } gcm_key_data::avx2_avx512
    +
    + +
    +
    + +

    ◆ expanded_keys

    + +
    +
    + + + + +
    uint8_t gcm_key_data::expanded_keys[IMB_GCM_ENC_KEY_LEN *IMB_GCM_KEY_SETS]
    +
    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    union { ... } gcm_key_data::ghash_keys
    +
    + +
    +
    + +

    ◆ shifted_hkey

    + +
    +
    + + + + +
    uint8_t gcm_key_data::shifted_hkey[IMB_GCM_ENC_KEY_LEN *48]
    +
    +

    This is needed for schoolbook multiply purposes. (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^48<<1 mod poly)

    +

    (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^48<<1 mod poly)

    + +
    +
    + +

    ◆ shifted_hkey_k

    + +
    +
    + + + + +
    uint8_t gcm_key_data::shifted_hkey_k[IMB_GCM_ENC_KEY_LEN *8]
    +
    +

    This is needed for Karatsuba multiply purposes. Storage for XOR of High 64 bits and low 64 bits of HashKey mod poly.

    +

    (HashKey<<1 mod poly), (HashKey^2<<1 mod poly), ..., (Hashkey^128<<1 mod poly)

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } gcm_key_data::sse_avx
    +
    +

    < Storage for precomputed hash keys

    + +
    +
    + +

    ◆ 

    + +
    +
    + + + + +
    struct { ... } gcm_key_data::vaes_avx512
    +
    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structgcm__key__data.js b/docs/structgcm__key__data.js new file mode 100644 index 00000000..85451498 --- /dev/null +++ b/docs/structgcm__key__data.js @@ -0,0 +1,10 @@ +var structgcm__key__data = +[ + [ "avx2_avx512", "structgcm__key__data.html#a3f231fdc1d521cd15c111667e4aea818", null ], + [ "expanded_keys", "structgcm__key__data.html#aa5bda4f55b8393be151cfe28a9dc1ca9", null ], + [ "ghash_keys", "structgcm__key__data.html#af5adb4225d569050207cfcfe00fe273a", null ], + [ "shifted_hkey", "structgcm__key__data.html#a73b61a77635c0357b36b83aaf69d1237", null ], + [ "shifted_hkey_k", "structgcm__key__data.html#a570a82f076ceb5da6719d1a554584820", null ], + [ "sse_avx", "structgcm__key__data.html#a908988e25607cd3e092bfcea9964deb1", null ], + [ "vaes_avx512", "structgcm__key__data.html#a7ad44e105b23e41b7f79530b6370e509", null ] +]; \ No newline at end of file diff --git a/docs/structimb__uint128__t.html b/docs/structimb__uint128__t.html new file mode 100644 index 00000000..7bd12640 --- /dev/null +++ b/docs/structimb__uint128__t.html @@ -0,0 +1,152 @@ + + + + + + + +intel-ipsec-mb: imb_uint128_t Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    imb_uint128_t Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + + + +

    +Data Fields

    uint64_t low
     
    uint64_t high
     
    +

    Field Documentation

    + +

    ◆ high

    + +
    +
    + + + + +
    uint64_t imb_uint128_t::high
    +
    + +
    +
    + +

    ◆ low

    + +
    +
    + + + + +
    uint64_t imb_uint128_t::low
    +
    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structimb__uint128__t.js b/docs/structimb__uint128__t.js new file mode 100644 index 00000000..53aa26ad --- /dev/null +++ b/docs/structimb__uint128__t.js @@ -0,0 +1,5 @@ +var structimb__uint128__t = +[ + [ "high", "structimb__uint128__t.html#aa65815c9b383e888758ae21d0c6159c9", null ], + [ "low", "structimb__uint128__t.html#a30e60cc86acb6eafaa4079c0e46ba622", null ] +]; \ No newline at end of file diff --git a/docs/structkasumi__key__sched__s.html b/docs/structkasumi__key__sched__s.html new file mode 100644 index 00000000..28acc4c3 --- /dev/null +++ b/docs/structkasumi__key__sched__s.html @@ -0,0 +1,156 @@ + + + + + + + +intel-ipsec-mb: kasumi_key_sched_s Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    kasumi_key_sched_s Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + + + +

    +Data Fields

    uint16_t sk16 [KASUMI_KEY_SCHEDULE_SIZE]
     
    uint16_t msk16 [KASUMI_KEY_SCHEDULE_SIZE]
     
    +

    Detailed Description

    +

    Structure to maintain internal key scheduling

    +

    Field Documentation

    + +

    ◆ msk16

    + +
    +
    + + + + +
    uint16_t kasumi_key_sched_s::msk16[KASUMI_KEY_SCHEDULE_SIZE]
    +
    +

    modified key schedule

    + +
    +
    + +

    ◆ sk16

    + +
    +
    + + + + +
    uint16_t kasumi_key_sched_s::sk16[KASUMI_KEY_SCHEDULE_SIZE]
    +
    +

    < Kasumi internal scheduling key schedule

    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structkasumi__key__sched__s.js b/docs/structkasumi__key__sched__s.js new file mode 100644 index 00000000..013ef260 --- /dev/null +++ b/docs/structkasumi__key__sched__s.js @@ -0,0 +1,5 @@ +var structkasumi__key__sched__s = +[ + [ "msk16", "structkasumi__key__sched__s.html#a091988e7975015d654ccb6904c4fdf39", null ], + [ "sk16", "structkasumi__key__sched__s.html#a7e69de411b81753c10776442bbe03100", null ] +]; \ No newline at end of file diff --git a/docs/structsnow3g__key__schedule__s.html b/docs/structsnow3g__key__schedule__s.html new file mode 100644 index 00000000..e259fb2e --- /dev/null +++ b/docs/structsnow3g__key__schedule__s.html @@ -0,0 +1,138 @@ + + + + + + + +intel-ipsec-mb: snow3g_key_schedule_s Struct Reference + + + + + + + + + + + + + + +
    +
    + + + + + + + +
    +
    intel-ipsec-mb +
    +
    Documentation of the Intel(R) IPSec Multi-Buffer library
    +
    + + + + + + +
    +
    +
    + + + +
    +
    + +
    +
    +
    + +
    + +
    +
    + + +
    + +
    + +
    +
    +Data Fields
    +
    snow3g_key_schedule_s Struct Reference
    +
    +
    + +

    #include <intel-ipsec-mb.h>

    + + + + +

    +Data Fields

    uint32_t k [4]
     
    +

    Detailed Description

    +

    Snow3G key scheduling structure

    +

    Field Documentation

    + +

    ◆ k

    + +
    +
    + + + + +
    uint32_t snow3g_key_schedule_s::k[4]
    +
    + +
    +
    +
    The documentation for this struct was generated from the following file: +
    +
    + + + + diff --git a/docs/structsnow3g__key__schedule__s.js b/docs/structsnow3g__key__schedule__s.js new file mode 100644 index 00000000..bffd16eb --- /dev/null +++ b/docs/structsnow3g__key__schedule__s.js @@ -0,0 +1,4 @@ +var structsnow3g__key__schedule__s = +[ + [ "k", "structsnow3g__key__schedule__s.html#ae0ca8cd2581836c0297b34eea1effb12", null ] +]; \ No newline at end of file diff --git a/docs/sync_off.png b/docs/sync_off.png new file mode 100644 index 0000000000000000000000000000000000000000..3b443fc62892114406e3d399421b2a881b897acc GIT binary patch literal 853 zcmV-b1FHOqP)oT|#XixUYy%lpuf3i8{fX!o zUyDD0jOrAiT^tq>fLSOOABs-#u{dV^F$b{L9&!2=9&RmV;;8s^x&UqB$PCj4FdKbh zoB1WTskPUPu05XzFbA}=KZ-GP1fPpAfSs>6AHb12UlR%-i&uOlTpFNS7{jm@mkU1V zh`nrXr~+^lsV-s1dkZOaI|kYyVj3WBpPCY{n~yd%u%e+d=f%`N0FItMPtdgBb@py; zq@v6NVArhyTC7)ULw-Jy8y42S1~4n(3LkrW8mW(F-4oXUP3E`e#g**YyqI7h-J2zK zK{m9##m4ri!7N>CqQqCcnI3hqo1I;Yh&QLNY4T`*ptiQGozK>FF$!$+84Z`xwmeMh zJ0WT+OH$WYFALEaGj2_l+#DC3t7_S`vHpSivNeFbP6+r50cO8iu)`7i%Z4BTPh@_m3Tk!nAm^)5Bqnr%Ov|Baunj#&RPtRuK& z4RGz|D5HNrW83-#ydk}tVKJrNmyYt-sTxLGlJY5nc&Re zU4SgHNPx8~Yxwr$bsju?4q&%T1874xxzq+_%?h8_ofw~(bld=o3iC)LUNR*BY%c0y zWd_jX{Y8`l%z+ol1$@Qa?Cy!(0CVIEeYpKZ`(9{z>3$CIe;pJDQk$m3p}$>xBm4lb zKo{4S)`wdU9Ba9jJbVJ0C=SOefZe%d$8=2r={nu<_^a3~>c#t_U6dye5)JrR(_a^E f@}b6j1K9lwFJq@>o)+Ry00000NkvXXu0mjfWa5j* literal 0 HcmV?d00001 diff --git a/docs/sync_on.png b/docs/sync_on.png new file mode 100644 index 0000000000000000000000000000000000000000..e08320fb64e6fa33b573005ed6d8fe294e19db76 GIT binary patch literal 845 zcmV-T1G4;yP)Y;xxyHF2B5Wzm| zOOGupOTn@c(JmBOl)e;XMNnZuiTJP>rM8<|Q`7I_))aP?*T)ow&n59{}X4$3Goat zgjs?*aasfbrokzG5cT4K=uG`E14xZl@z)F={P0Y^?$4t z>v!teRnNZym<6h{7sLyF1V0HsfEl+l6TrZpsfr1}luH~F7L}ktXu|*uVX^RG$L0`K zWs3j|0tIvVe(N%_?2{(iCPFGf#B6Hjy6o&}D$A%W%jfO8_W%ZO#-mh}EM$LMn7joJ z05dHr!5Y92g+31l<%i1(=L1a1pXX+OYnalY>31V4K}BjyRe3)9n#;-cCVRD_IG1fT zOKGeNY8q;TL@K{dj@D^scf&VCs*-Jb>8b>|`b*osv52-!A?BpbYtTQBns5EAU**$m zSnVSm(teh>tQi*S*A>#ySc=n;`BHz`DuG4&g4Kf8lLhca+zvZ7t7RflD6-i-mcK=M z!=^P$*u2)bkY5asG4gsss!Hn%u~>}kIW`vMs%lJLH+u*9<4PaV_c6U`KqWXQH%+Nu zTv41O(^ZVi@qhjQdG!fbZw&y+2o!iYymO^?ud3{P*HdoX83YV*Uu_HB=?U&W9%AU# z80}k1SS-CXTU7dcQlsm<^oYLxVSseqY6NO}dc`Nj?8vrhNuCdm@^{a3AQ_>6myOj+ z`1RsLUXF|dm|3k7s2jD(B{rzE>WI2scH8i1;=O5Cc9xB3^aJk%fQjqsu+kH#0=_5a z0nCE8@dbQa-|YIuUVvG0L_IwHMEhOj$Mj4Uq05 X8=0q~qBNan00000NkvXXu0mjfptF>5 literal 0 HcmV?d00001 diff --git a/docs/tab_a.png b/docs/tab_a.png new file mode 100644 index 0000000000000000000000000000000000000000..3b725c41c5a527a3a3e40097077d0e206a681247 GIT binary patch literal 142 zcmeAS@N?(olHy`uVBq!ia0vp^j6kfy!2~3aiye;!QlXwMjv*C{Z|8b*H5dputLHD# z=<0|*y7z(Vor?d;H&?EG&cXR}?!j-Lm&u1OOI7AIF5&c)RFE;&p0MYK>*Kl@eiymD r@|NpwKX@^z+;{u_Z~trSBfrMKa%3`zocFjEXaR$#tDnm{r-UW|TZ1%4 literal 0 HcmV?d00001 diff --git a/docs/tab_b.png b/docs/tab_b.png new file mode 100644 index 0000000000000000000000000000000000000000..e2b4a8638cb3496a016eaed9e16ffc12846dea18 GIT binary patch literal 169 zcmeAS@N?(olHy`uVBq!ia0vp^j6kfy!2~3aiye;!QU#tajv*C{Z}0l@H7kg?K0Lnr z!j&C6_(~HV9oQ0Pa6x{-v0AGV_E?vLn=ZI-;YrdjIl`U`uzuDWSP?o#Dmo{%SgM#oan kX~E1%D-|#H#QbHoIja2U-MgvsK&LQxy85}Sb4q9e0Efg%P5=M^ literal 0 HcmV?d00001 diff --git a/docs/tabs.css b/docs/tabs.css new file mode 100644 index 00000000..00d1c602 --- /dev/null +++ b/docs/tabs.css @@ -0,0 +1 @@ +.sm{position:relative;z-index:9999}.sm,.sm ul,.sm li{display:block;list-style:none;margin:0;padding:0;line-height:normal;direction:ltr;text-align:left;-webkit-tap-highlight-color:rgba(0,0,0,0)}.sm-rtl,.sm-rtl ul,.sm-rtl li{direction:rtl;text-align:right}.sm>li>h1,.sm>li>h2,.sm>li>h3,.sm>li>h4,.sm>li>h5,.sm>li>h6{margin:0;padding:0}.sm ul{display:none}.sm li,.sm a{position:relative}.sm a{display:block}.sm a.disabled{cursor:not-allowed}.sm:after{content:"\00a0";display:block;height:0;font:0/0 serif;clear:both;visibility:hidden;overflow:hidden}.sm,.sm *,.sm *:before,.sm *:after{-moz-box-sizing:border-box;-webkit-box-sizing:border-box;box-sizing:border-box}.main-menu-btn{position:relative;display:inline-block;width:36px;height:36px;text-indent:36px;margin-left:8px;white-space:nowrap;overflow:hidden;cursor:pointer;-webkit-tap-highlight-color:rgba(0,0,0,0)}.main-menu-btn-icon,.main-menu-btn-icon:before,.main-menu-btn-icon:after{position:absolute;top:50%;left:2px;height:2px;width:24px;background:#666;-webkit-transition:all .25s;transition:all .25s}.main-menu-btn-icon:before{content:'';top:-7px;left:0}.main-menu-btn-icon:after{content:'';top:7px;left:0}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon{height:0}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon:before{top:0;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}#main-menu-state:checked ~ .main-menu-btn .main-menu-btn-icon:after{top:0;-webkit-transform:rotate(45deg);transform:rotate(45deg)}#main-menu-state{position:absolute;width:1px;height:1px;margin:-1px;border:0;padding:0;overflow:hidden;clip:rect(1px,1px,1px,1px)}#main-menu-state:not(:checked) ~ #main-menu{display:none}#main-menu-state:checked ~ #main-menu{display:block}@media(min-width:768px){.main-menu-btn{position:absolute;top:-99999px}#main-menu-state:not(:checked) ~ #main-menu{display:block}}.sm-dox{background-image:url("tab_b.png")}.sm-dox a,.sm-dox a:focus,.sm-dox a:hover,.sm-dox a:active{padding:0 12px;padding-right:43px;font-family:"Lucida Grande","Geneva","Helvetica",Arial,sans-serif;font-size:13px;font-weight:bold;line-height:36px;text-decoration:none;text-shadow:0 1px 1px rgba(255,255,255,0.9);color:#283a5d;outline:0}.sm-dox a:hover{background-image:url("tab_a.png");background-repeat:repeat-x;color:white;text-shadow:0 1px 1px black}.sm-dox a.current{color:#d23600}.sm-dox a.disabled{color:#bbb}.sm-dox a span.sub-arrow{position:absolute;top:50%;margin-top:-14px;left:auto;right:3px;width:28px;height:28px;overflow:hidden;font:bold 12px/28px monospace !important;text-align:center;text-shadow:none;background:rgba(255,255,255,0.5);-moz-border-radius:5px;-webkit-border-radius:5px;border-radius:5px}.sm-dox a span.sub-arrow:before{display:block;content:'+'}.sm-dox a.highlighted span.sub-arrow:before{display:block;content:'-'}.sm-dox>li:first-child>a,.sm-dox>li:first-child>:not(ul) a{-moz-border-radius:5px 5px 0 0;-webkit-border-radius:5px;border-radius:5px 5px 0 0}.sm-dox>li:last-child>a,.sm-dox>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul{-moz-border-radius:0 0 5px 5px;-webkit-border-radius:0;border-radius:0 0 5px 5px}.sm-dox>li:last-child>a.highlighted,.sm-dox>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>a.highlighted,.sm-dox>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>ul>li:last-child>*:not(ul) a.highlighted{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0}.sm-dox ul{background:rgba(162,162,162,0.1)}.sm-dox ul a,.sm-dox ul a:focus,.sm-dox ul a:hover,.sm-dox ul a:active{font-size:12px;border-left:8px solid transparent;line-height:36px;text-shadow:none;background-color:white;background-image:none}.sm-dox ul a:hover{background-image:url("tab_a.png");background-repeat:repeat-x;color:white;text-shadow:0 1px 1px black}.sm-dox ul ul a,.sm-dox ul ul a:hover,.sm-dox ul ul a:focus,.sm-dox ul ul a:active{border-left:16px solid transparent}.sm-dox ul ul ul a,.sm-dox ul ul ul a:hover,.sm-dox ul ul ul a:focus,.sm-dox ul ul ul a:active{border-left:24px solid transparent}.sm-dox ul ul ul ul a,.sm-dox ul ul ul ul a:hover,.sm-dox ul ul ul ul a:focus,.sm-dox ul ul ul ul a:active{border-left:32px solid transparent}.sm-dox ul ul ul ul ul a,.sm-dox ul ul ul ul ul a:hover,.sm-dox ul ul ul ul ul a:focus,.sm-dox ul ul ul ul ul a:active{border-left:40px solid transparent}@media(min-width:768px){.sm-dox ul{position:absolute;width:12em}.sm-dox li{float:left}.sm-dox.sm-rtl li{float:right}.sm-dox ul li,.sm-dox.sm-rtl ul li,.sm-dox.sm-vertical li{float:none}.sm-dox a{white-space:nowrap}.sm-dox ul a,.sm-dox.sm-vertical a{white-space:normal}.sm-dox .sm-nowrap>li>a,.sm-dox .sm-nowrap>li>:not(ul) a{white-space:nowrap}.sm-dox{padding:0 10px;background-image:url("tab_b.png");line-height:36px}.sm-dox a span.sub-arrow{top:50%;margin-top:-2px;right:12px;width:0;height:0;border-width:4px;border-style:solid dashed dashed dashed;border-color:#283a5d transparent transparent transparent;background:transparent;-moz-border-radius:0;-webkit-border-radius:0;border-radius:0}.sm-dox a,.sm-dox a:focus,.sm-dox a:active,.sm-dox a:hover,.sm-dox a.highlighted{padding:0 12px;background-image:url("tab_s.png");background-repeat:no-repeat;background-position:right;-moz-border-radius:0 !important;-webkit-border-radius:0;border-radius:0 !important}.sm-dox a:hover{background-image:url("tab_a.png");background-repeat:repeat-x;color:white;text-shadow:0 1px 1px black}.sm-dox a:hover span.sub-arrow{border-color:white transparent transparent transparent}.sm-dox a.has-submenu{padding-right:24px}.sm-dox li{border-top:0}.sm-dox>li>ul:before,.sm-dox>li>ul:after{content:'';position:absolute;top:-18px;left:30px;width:0;height:0;overflow:hidden;border-width:9px;border-style:dashed dashed solid dashed;border-color:transparent transparent #bbb transparent}.sm-dox>li>ul:after{top:-16px;left:31px;border-width:8px;border-color:transparent transparent #fff transparent}.sm-dox ul{border:1px solid #bbb;padding:5px 0;background:#fff;-moz-border-radius:5px !important;-webkit-border-radius:5px;border-radius:5px !important;-moz-box-shadow:0 5px 9px rgba(0,0,0,0.2);-webkit-box-shadow:0 5px 9px rgba(0,0,0,0.2);box-shadow:0 5px 9px rgba(0,0,0,0.2)}.sm-dox ul a span.sub-arrow{right:8px;top:50%;margin-top:-5px;border-width:5px;border-color:transparent transparent transparent #555;border-style:dashed dashed dashed solid}.sm-dox ul a,.sm-dox ul a:hover,.sm-dox ul a:focus,.sm-dox ul a:active,.sm-dox ul a.highlighted{color:#555;background-image:none;border:0 !important;color:#555;background-image:none}.sm-dox ul a:hover{background-image:url("tab_a.png");background-repeat:repeat-x;color:white;text-shadow:0 1px 1px black}.sm-dox ul a:hover span.sub-arrow{border-color:transparent transparent transparent white}.sm-dox span.scroll-up,.sm-dox span.scroll-down{position:absolute;display:none;visibility:hidden;overflow:hidden;background:#fff;height:36px}.sm-dox span.scroll-up:hover,.sm-dox span.scroll-down:hover{background:#eee}.sm-dox span.scroll-up:hover span.scroll-up-arrow,.sm-dox span.scroll-up:hover span.scroll-down-arrow{border-color:transparent transparent #d23600 transparent}.sm-dox span.scroll-down:hover span.scroll-down-arrow{border-color:#d23600 transparent transparent transparent}.sm-dox span.scroll-up-arrow,.sm-dox span.scroll-down-arrow{position:absolute;top:0;left:50%;margin-left:-6px;width:0;height:0;overflow:hidden;border-width:6px;border-style:dashed dashed solid dashed;border-color:transparent transparent #555 transparent}.sm-dox span.scroll-down-arrow{top:8px;border-style:solid dashed dashed dashed;border-color:#555 transparent transparent transparent}.sm-dox.sm-rtl a.has-submenu{padding-right:12px;padding-left:24px}.sm-dox.sm-rtl a span.sub-arrow{right:auto;left:12px}.sm-dox.sm-rtl.sm-vertical a.has-submenu{padding:10px 20px}.sm-dox.sm-rtl.sm-vertical a span.sub-arrow{right:auto;left:8px;border-style:dashed solid dashed dashed;border-color:transparent #555 transparent transparent}.sm-dox.sm-rtl>li>ul:before{left:auto;right:30px}.sm-dox.sm-rtl>li>ul:after{left:auto;right:31px}.sm-dox.sm-rtl ul a.has-submenu{padding:10px 20px !important}.sm-dox.sm-rtl ul a span.sub-arrow{right:auto;left:8px;border-style:dashed solid dashed dashed;border-color:transparent #555 transparent transparent}.sm-dox.sm-vertical{padding:10px 0;-moz-border-radius:5px;-webkit-border-radius:5px;border-radius:5px}.sm-dox.sm-vertical a{padding:10px 20px}.sm-dox.sm-vertical a:hover,.sm-dox.sm-vertical a:focus,.sm-dox.sm-vertical a:active,.sm-dox.sm-vertical a.highlighted{background:#fff}.sm-dox.sm-vertical a.disabled{background-image:url("tab_b.png")}.sm-dox.sm-vertical a span.sub-arrow{right:8px;top:50%;margin-top:-5px;border-width:5px;border-style:dashed dashed dashed solid;border-color:transparent transparent transparent #555}.sm-dox.sm-vertical>li>ul:before,.sm-dox.sm-vertical>li>ul:after{display:none}.sm-dox.sm-vertical ul a{padding:10px 20px}.sm-dox.sm-vertical ul a:hover,.sm-dox.sm-vertical ul a:focus,.sm-dox.sm-vertical ul a:active,.sm-dox.sm-vertical ul a.highlighted{background:#eee}.sm-dox.sm-vertical ul a.disabled{background:#fff}} \ No newline at end of file diff --git a/lib/api_doxygen.conf b/lib/api_doxygen.conf index 5443fd20..6324903b 100644 --- a/lib/api_doxygen.conf +++ b/lib/api_doxygen.conf @@ -18,7 +18,7 @@ PROJECT_NUMBER = # for a project that appears at the top of each page and should give viewer a # quick idea about the purpose of the project. Keep the description short. -PROJECT_BRIEF = +PROJECT_BRIEF = "Documentation of the Intel(R) IPSec Multi-Buffer library" # With the PROJECT_LOGO tag one can specify a logo or an icon that is included # in the documentation. The maximum height of the logo should not exceed 55 -- GitLab From 1ffb00707889773c34aaec09ca7d9d563a57df67 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 26 Jul 2022 15:14:41 +0000 Subject: [PATCH 273/369] avx512: [poly1305] enforce EVEX encoding of PMADD52 instructions --- lib/avx512_t2/poly_fma_avx512.asm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index d13d7b58..6da2e441 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -32,6 +32,10 @@ %include "include/clear_regs.asm" %include "include/cet.inc" +;; Enforce EVEX encoding for AVX512 capable systems +%xdefine vpmadd52luq {evex}vpmadd52luq +%xdefine vpmadd52huq {evex}vpmadd52huq + [bits 64] default rel -- GitLab From 8e45dfc992843981567171e1264fb65416d7cbd7 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 13 Jul 2022 14:30:30 +0000 Subject: [PATCH 274/369] test [ACVP] support for CMAC --- lib/include/mb_mgr_code.h | 3 +- test/acvp_app_main.c | 79 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 30d38acf..f0b2886f 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2057,8 +2057,9 @@ is_job_invalid(IMB_MGR *state, const IMB_JOB *job, } /* T is 128 bits but 96 bits is also allowed due to * IPsec use case (RFC 4494) and 32 bits for CMAC 3GPP. + * ACVP validation requires tag size of 8 bits. */ - if (job->auth_tag_output_len_in_bytes < UINT64_C(4) || + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || job->auth_tag_output_len_in_bytes > UINT64_C(16)) { imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); return 1; diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 3847f3db..ffe14dae 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -512,6 +512,81 @@ static int aes_ccm_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_cmac_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_CMAC_TC *tc; + IMB_JOB *job = NULL; + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint32_t skey1[4], skey2[4]; + uint8_t res_tag[MAX_TAG_LENGTH] = {0}; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.cmac; + + switch (tc->key_len) { + case 16: + IMB_AES_KEYEXP_128(mb_mgr, tc->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_128(mb_mgr, expkey, skey1, skey2); + break; + case 32: + IMB_AES_KEYEXP_256(mb_mgr, tc->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_256(mb_mgr, expkey, skey1, skey2); + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + + if (tc->key_len == 32) + job->hash_alg = IMB_AUTH_AES_CMAC_256; + else + job->hash_alg = IMB_AUTH_AES_CMAC; + + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + job->auth_tag_output_len_in_bytes = tc->mac_len; + + if (tc->verify == 1) + job->auth_tag_output = res_tag; + else /* verify == 0 */ + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + + if (tc->verify == 1) { + if (memcmp(res_tag, tc->mac, tc->mac_len != 0)) { + if (verbose) { + hexdump(stdout, "result tag: ", + res_tag, (tc->mac_len)); + hexdump(stdout, "reference tag: ", + tc->mac, tc->mac_len); + fprintf(stderr, "Invalid tag\n"); + } + tc->ver_disposition = ACVP_TEST_DISPOSITION_FAIL; + } else + tc->ver_disposition = ACVP_TEST_DISPOSITION_PASS; + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -631,6 +706,10 @@ int main(int argc, char **argv) &aes_ccm_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_cmac_enable(ctx, ACVP_CMAC_AES, + &aes_cmac_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 12e106f36c8409cd35d3dbad53500cbe6ec159bc Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Tue, 26 Jul 2022 15:13:54 +0000 Subject: [PATCH 275/369] avx2: [ZUC] fix avx-sse transition issues --- lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm index 528f6b06..1e73cb51 100644 --- a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm @@ -408,7 +408,7 @@ mksection .text %%return_submit_eea3: %ifdef SAFE_DATA - clear_all_xmms_avx_asm + clear_scratch_ymms_asm %else vzeroupper %endif @@ -688,7 +688,7 @@ APPEND3(%%skip_eea3_copy_,I,J): or [state + _zuc_unused_lane_bitmask], BYTE(tmp3) %%return_flush_eea3: %ifdef SAFE_DATA - clear_all_xmms_avx_asm + clear_scratch_ymms_asm %else vzeroupper %endif @@ -914,7 +914,7 @@ FLUSH_JOB_ZUC256_EEA3: %%return_submit_eia3: %ifdef SAFE_DATA - clear_all_xmms_avx_asm + clear_scratch_ymms_asm %else vzeroupper %endif @@ -1082,7 +1082,7 @@ APPEND(%%skip_eia3_,I): %%return_flush_eia3: %ifdef SAFE_DATA - clear_all_xmms_avx_asm + clear_scratch_ymms_asm %else vzeroupper %endif -- GitLab From 3cf98162d5007c82e80a88dfb181488d26902aef Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Thu, 28 Jul 2022 14:40:05 +0000 Subject: [PATCH 276/369] sse_t1: [SNOW3G] Optimize keystream generation loop Use loop instead of preprocessor "rep" to reduce number of instructions. --- lib/sse_t1/snow3g_uea2_by4_sse.asm | 55 +++++++++++++++++++----------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index 350f4972..9fcf8b14 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -601,7 +601,7 @@ endstruc %define %%TMP14 %15 ;; [clobbered] temporary xmm register %define %%TMP15 %16 ;; [out] generated keystream %define %%TMP16 %17 ;; [out] generated keystream -%define %%DWORD_ITER %18 ;; [in] offset for stack storing keystream +%define %%DWORD_ITER %18 ;; [in] gp reg with offset for stack storing keystream ;; Calculate F = (LFSR_S[15] + FSM_R1) ^ FSM_R2; movdqa %%TMP1, [ %%STATE + _snow3g_args_LFSR_15 ] @@ -616,7 +616,8 @@ endstruc pxor %%TMP1, [%%STATE + _snow3g_args_LFSR_0] pand %%TMP3, %%TMP1 ;; zero in init mode por %%TMP2, %%TMP3 - movdqa [rsp + _keystream + %%DWORD_ITER * 16], %%TMP2 + shl %%DWORD_ITER, 4 + movdqa [rsp + _keystream + %%DWORD_ITER], %%TMP2 ;; FSM Clock SNOW3G_FSM_CLOCK {%%STATE + _snow3g_args_FSM_1}, \ @@ -654,11 +655,13 @@ endstruc ;; in init mode mask is 0, so this is applies only in init mode movdqa %%TMP2, [%%STATE + _snow3g_args_LD_ST_MASK + 4*4] - pandn %%TMP2, [rsp + _keystream + %%DWORD_ITER * 16] + pandn %%TMP2, [rsp + _keystream + %%DWORD_ITER] pxor %%TMP15, %%TMP2 SHIFT_LFSRS %%STATE, %%TMP15, %%TMP1, %%TMP2, %%TMP3, \ %%TMP4, %%TMP5, %%TMP6, %%TMP7 + ;; restore offset + shr %%DWORD_ITER, 4 %endmacro ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -872,15 +875,18 @@ endstruc shr %%LENGTH, 2 je %%no_dqws + %%next_dqw: -%assign i 0 -%rep 4 + xor %%TMP1_64, %%TMP1_64 + +%%next_dqw_round: SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, i + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 -%assign i (i+1) -%endrep + inc %%TMP1_64 + cmp %%TMP1_64, 4 + jb %%next_dqw_round TRANSPOSE_4X32 %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, %%TMP6 SNOW3G_OUTPUT 16, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%TMP5, %%TMP1 @@ -898,9 +904,10 @@ endstruc je %%no_full_dws_to_write_out %%next_dw: + xor %%TMP1_64, %%TMP1_64 SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 SNOW3G_OUTPUT 4, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 SNOW3G_OUTPUT 4, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 @@ -912,9 +919,10 @@ endstruc %%no_full_dws_to_write_out: ;; Process last dw/bytes: + xor %%TMP1_64, %%TMP1_64 SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 SNOW3G_OUTPUT 3, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%LENGTH SNOW3G_OUTPUT 3, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%LENGTH @@ -964,32 +972,39 @@ endstruc %endrep ;; Run 32 iteration in INIT mode (reject keystreams) -%rep 32 + mov %%TMP1_64, 32 + xor %%TMP2_64, %%TMP2_64 + +%%next_auth_round: SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 -%endrep + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 + dec %%TMP1_64 + jnz %%next_auth_round + ;; Mark INIT1 phase done for all lanes movdqa %%TMP1, [rel all_fs] movdqa [state + INIT1_DONE], %%TMP1 SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 ;; Put all lanes in KEYGEN state movdqa %%TMP1, [rel all_fs] movdqa [state + KEYGEN_STAGE], %%TMP1 ;; Generate 4 dw of keystream for each lane -%assign i 0 -%rep 4 + xor %%TMP1_64, %%TMP1_64 + +%%next_auth_round2: SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, i + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 + inc %%TMP1_64 + cmp %%TMP1_64, 4 + jb %%next_auth_round2 -%assign i (i+1) -%endrep TRANSPOSE_4X32 %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, %%TMP6 ;; Store 4 dw of keystream for each lane @@ -1001,7 +1016,7 @@ endstruc ;; Generate final dw of keystream for each lane SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ - %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, 0 + %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 ;; Store final dw of keystream for each lane mov DWORD(%%TMP1_64), [rsp + _keystream + 0*4] -- GitLab From 9bb9abafda86e1a7d38d103bc6da4929ba4bf4a8 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 25 Jul 2022 14:49:34 +0000 Subject: [PATCH 277/369] lib: Add async burst API error types --- lib/include/error.inc | 6 +++++- lib/intel-ipsec-mb.h | 4 ++++ lib/x86_64/error.c | 14 +++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/lib/include/error.inc b/lib/include/error.inc index e70167e2..48afb5c1 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -93,7 +93,11 @@ SET_ERRNO_TYPES \ IMB_ERR_JOB_CIPH_DIR, \ IMB_ERR_JOB_NULL_GHASH_INIT_TAG, \ IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, \ - IMB_ERR_NULL_JOB + IMB_ERR_NULL_JOB, \ + IMB_ERR_QUEUE_SPACE, \ + IMB_ERR_NULL_BURST, \ + IMB_ERR_BURST_OOO, \ + IMB_ERR_BURST_SIZE ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 31a0ac8f..78f0d12b 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -224,6 +224,10 @@ typedef enum { IMB_ERR_JOB_NULL_GHASH_INIT_TAG, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, IMB_ERR_NULL_JOB, + IMB_ERR_QUEUE_SPACE, + IMB_ERR_NULL_BURST, + IMB_ERR_BURST_SIZE, + IMB_ERR_BURST_OOO, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index d4fe80df..9fc210d0 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -87,7 +87,11 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_JOB_CIPH_DIR, IMB_ERR_JOB_NULL_GHASH_INIT_TAG, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR, - IMB_ERR_NULL_JOB + IMB_ERR_NULL_JOB, + IMB_ERR_QUEUE_SPACE, + IMB_ERR_NULL_BURST, + IMB_ERR_BURST_OOO, + IMB_ERR_BURST_SIZE }; #ifdef DEBUG @@ -211,6 +215,14 @@ imb_get_strerror(int errnum) "required CPU flags"; case IMB_ERR_NULL_JOB: return "NULL job pointer"; + case IMB_ERR_QUEUE_SPACE: + return "Not enough space in job queue"; + case IMB_ERR_NULL_BURST: + return "NULL pointer to burst job array"; + case IMB_ERR_BURST_SIZE: + return "Invalid burst size"; + case IMB_ERR_BURST_OOO: + return "Burst jobs out of order"; default: return strerror(errnum); } -- GitLab From 8d8096c6aad68da8e4f707bbdd1d1f87249e91bc Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 25 Jul 2022 14:50:20 +0000 Subject: [PATCH 278/369] lib: Rework burst API to be asynchronous --- lib/avx/mb_mgr_avx.c | 4 + lib/avx2_t1/mb_mgr_avx2.c | 4 + lib/avx512_t1/mb_mgr_avx512.c | 4 + lib/include/ipsec_ooo_mgr.h | 40 +++- lib/include/mb_mgr_code.h | 291 ++++++++++++++++++++++------- lib/include/noaesni.h | 15 +- lib/intel-ipsec-mb.h | 71 +++++-- lib/no-aesni/mb_mgr_sse_no_aesni.c | 4 + lib/sse_t1/mb_mgr_sse.c | 4 + 9 files changed, 334 insertions(+), 103 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 2757f087..ead350eb 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -171,8 +171,10 @@ #define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx #define GET_NEXT_JOB get_next_job_avx #define GET_COMPLETED_JOB get_completed_job_avx +#define GET_NEXT_BURST get_next_burst_avx #define SUBMIT_BURST submit_burst_avx #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx +#define FLUSH_BURST flush_burst_avx #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx #define SUBMIT_HASH_BURST submit_hash_burst_avx @@ -492,8 +494,10 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) /* set AVX handlers */ state->get_next_job = get_next_job_avx; state->submit_job = submit_job_avx; + state->get_next_burst = get_next_burst_avx; state->submit_burst = submit_burst_avx; state->submit_burst_nocheck= submit_burst_nocheck_avx; + state->flush_burst = flush_burst_avx; state->submit_cipher_burst = submit_cipher_burst_avx; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx; state->submit_hash_burst = submit_hash_burst_avx; diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 44a652ef..25d6ce58 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -180,8 +180,10 @@ #define QUEUE_SIZE queue_size_avx2 #define GET_NEXT_JOB get_next_job_avx2 #define GET_COMPLETED_JOB get_completed_job_avx2 +#define GET_NEXT_BURST get_next_burst_avx2 #define SUBMIT_BURST submit_burst_avx2 #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2 +#define FLUSH_BURST flush_burst_avx2 #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2 #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2 #define SUBMIT_HASH_BURST submit_hash_burst_avx2 @@ -482,8 +484,10 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) /* set handlers */ state->get_next_job = get_next_job_avx2; state->submit_job = submit_job_avx2; + state->get_next_burst = get_next_burst_avx2; state->submit_burst = submit_burst_avx2; state->submit_burst_nocheck= submit_burst_nocheck_avx2; + state->flush_burst = flush_burst_avx2; state->submit_cipher_burst = submit_cipher_burst_avx2; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx2; state->submit_hash_burst = submit_hash_burst_avx2; diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index d21afb69..f981f6dc 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -285,8 +285,10 @@ SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, #define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 #define GET_NEXT_JOB get_next_job_avx512 #define GET_COMPLETED_JOB get_completed_job_avx512 +#define GET_NEXT_BURST get_next_burst_avx512 #define SUBMIT_BURST submit_burst_avx512 #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512 +#define FLUSH_BURST flush_burst_avx512 #define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512 #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512 #define SUBMIT_HASH_BURST submit_hash_burst_avx512 @@ -1128,8 +1130,10 @@ init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) /* set handlers */ state->get_next_job = get_next_job_avx512; state->submit_job = submit_job_avx512; + state->get_next_burst = get_next_burst_avx512; state->submit_burst = submit_burst_avx512; state->submit_burst_nocheck= submit_burst_nocheck_avx512; + state->flush_burst = flush_burst_avx512; state->submit_cipher_burst = submit_cipher_burst_avx512; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx512; state->submit_hash_burst = submit_hash_burst_avx512; diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 49f92d5f..24c98c9a 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -423,23 +423,45 @@ IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t -submit_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +get_next_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +get_next_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +get_next_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +get_next_burst_avx512(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +submit_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +submit_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs); +submit_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs); +submit_burst_avx512(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); + +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx2(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx512(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); + +IMB_DLL_EXPORT uint32_t +flush_burst_sse(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_avx(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_avx2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_avx512(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); + IMB_DLL_EXPORT uint32_t submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index f0b2886f..27b6dec8 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -39,6 +39,7 @@ * to be usable at the next call to get_next_job() */ +#include #include /* memcpy(), memset() */ #include "include/clear_regs_mem.h" @@ -80,6 +81,14 @@ void ADV_JOBS(int *ptr) *ptr = 0; } +__forceinline +void ADV_N_JOBS(int *ptr, const uint32_t n_jobs) +{ + *ptr += (sizeof(IMB_JOB) * n_jobs); + if (*ptr >= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB))) + *ptr -= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB)); +} + /* ========================================================================= */ /* Lower level "out of order" schedulers */ /* ========================================================================= */ @@ -2621,9 +2630,44 @@ FLUSH_JOB(IMB_MGR *state) return job; } +/* get number of jobs between job_offset and the end of the queue */ +__forceinline uint32_t +get_queue_sz_end(const int job_offset) +{ + return IMB_MAX_JOBS - (job_offset / sizeof(IMB_JOB)); +} + +__forceinline uint32_t +get_queue_sz(IMB_MGR *state) +{ + const int a = (state->next_job - state->earliest_job) / sizeof(IMB_JOB); + + return a & (IMB_MAX_JOBS-1); +} + +__forceinline uint32_t +queue_sz(IMB_MGR *state) +{ + if (state->earliest_job < 0) + return 0; + + return get_queue_sz(state); +} + +__forceinline uint32_t +queue_sz_remaining(IMB_MGR *state) +{ + if (state->earliest_job < 0) + return IMB_MAX_JOBS; + + return IMB_MAX_JOBS - get_queue_sz(state); +} + + /* ========================================================================= */ /* ========================================================================= */ + uint32_t QUEUE_SIZE(IMB_MGR *state) { @@ -2636,13 +2680,7 @@ QUEUE_SIZE(IMB_MGR *state) return 0; } #endif - int a, b; - - if (state->earliest_job < 0) - return 0; - a = state->next_job / sizeof(IMB_JOB); - b = state->earliest_job / sizeof(IMB_JOB); - return ((a-b) & (IMB_MAX_JOBS-1)); + return queue_sz(state); } IMB_JOB * @@ -2690,103 +2728,216 @@ GET_NEXT_JOB(IMB_MGR *state) return JOBS(state, state->next_job); } -/** - * Submit multiple jobs to be processed synchronously - * - * @param [in] state pointer to multi-buffer manager - * @param [in] jobs pointer to array of jobs - * @param [in] n_jobs number of jobs to process - * - * @return number of completed jobs - */ -__forceinline -uint32_t submit_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, const int run_check) +uint32_t +GET_NEXT_BURST(IMB_MGR *state, const uint32_t n_req_jobs, IMB_JOB **jobs) { - uint32_t i, completed_jobs = 0; + uint32_t i, num_jobs, n_ret_jobs, filled_jobs = 0; + IMB_JOB *job = NULL; + + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + if (n_req_jobs > IMB_MAX_BURST_SIZE) { + imb_set_errno(state, IMB_ERR_BURST_SIZE); + return 0; + } +#endif + /* set number of jobs to return */ + n_ret_jobs = queue_sz_remaining(state); + if (n_ret_jobs > n_req_jobs) + n_ret_jobs = n_req_jobs; + + /* start filling list from next available job */ + job = JOBS(state, state->next_job); + + /* check enough jobs available before end of queue */ + num_jobs = get_queue_sz_end(state->next_job); + + if (num_jobs < n_ret_jobs) { + /* fill jobs to the end of the queue */ + for (i = 0; i < num_jobs; i++) { + jobs[filled_jobs++] = job; + job++; + } + /* fill remaining jobs from beginning of queue */ + num_jobs = n_ret_jobs - num_jobs; + job = &state->jobs[0]; + } else + /* fill all jobs */ + num_jobs = n_ret_jobs; + + for (i = 0; i < num_jobs; i++) { + jobs[filled_jobs++] = job; + job++; + } + + return filled_jobs; +} + +__forceinline uint32_t +submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs, const int run_check) +{ + uint32_t i, n_ret_jobs = 0, num_jobs = n_jobs; + IMB_JOB *job = NULL; /* reset error status */ imb_set_errno(state, 0); +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + if (n_jobs > IMB_MAX_BURST_SIZE) { + imb_set_errno(state, IMB_ERR_BURST_SIZE); + return 0; + } +#endif if (run_check) { - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); + int job_offset = state->next_job; + + /* check enough space in queue */ + if (queue_sz_remaining(state) < n_jobs) { + imb_set_errno(state, IMB_ERR_QUEUE_SPACE); return 0; } - /* validate jobs */ for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; +#ifdef SAFE_PARAM + if (jobs[i] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } +#endif + if (jobs[i] != JOBS(state, job_offset)) { + imb_set_errno(state, IMB_ERR_BURST_OOO); + goto return_invalid_job; + } + ADV_JOBS(&job_offset); /* validate job */ - if (is_job_invalid(state, job, - job->cipher_mode, job->hash_alg, - job->cipher_direction, - job->key_len_in_bytes)) { - job->status = IMB_STATUS_INVALID_ARGS; - return 0; + if (is_job_invalid(state, jobs[i], + jobs[i]->cipher_mode, + jobs[i]->hash_alg, + jobs[i]->cipher_direction, + jobs[i]->key_len_in_bytes)) { + goto return_invalid_job; } } } + /* state was previously empty */ + if (state->earliest_job < 0) + state->earliest_job = state->next_job; + /* submit all jobs */ for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; + jobs[i]->status = IMB_STATUS_BEING_PROCESSED; + submit_new_job(state, jobs[i]); + } + ADV_N_JOBS(&state->next_job, n_jobs); - job->status = IMB_STATUS_BEING_PROCESSED; + /* + * return completed jobs + * - may need 2 passes if jobs wrap in queue + */ + num_jobs = get_queue_sz_end(state->earliest_job); + if (num_jobs > n_jobs) + num_jobs = n_jobs; - if (job->cipher_mode == IMB_CIPHER_GCM) { - if (job->cipher_direction == IMB_DIR_ENCRYPT) - SUBMIT_JOB_AES_GCM_ENC(state, job); - else - SUBMIT_JOB_AES_GCM_DEC(state, job); - completed_jobs++; - } else if (job->cipher_mode == IMB_CIPHER_GCM_SGL) { - if (job->cipher_direction == IMB_DIR_ENCRYPT) - submit_gcm_sgl_enc(state, job); - else - submit_gcm_sgl_dec(state, job); - completed_jobs++; - } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { - SUBMIT_JOB_CHACHA20_POLY1305(state, job); - completed_jobs++; - } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == - job->cipher_mode) { - SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); - completed_jobs++; - } else { - if (submit_new_job(state, job) != NULL) - completed_jobs++; - } + /* start returning from earliest job */ + job = JOBS(state, state->earliest_job); + +return_jobs: + for (i = 0; i < num_jobs; i++) { + if (job->status < IMB_STATUS_COMPLETED) + goto return_jobs_done; + jobs[n_ret_jobs++] = job; + job++; } - /* return if all jobs complete */ - if (completed_jobs == n_jobs) - return completed_jobs; + /* check if all jobs returned + * if not, return remaining jobs from beginning of queue + */ + if (n_ret_jobs < n_jobs) { + num_jobs = n_jobs - num_jobs; + job = &state->jobs[0]; + goto return_jobs; + } - /* otherwise complete remaining jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; +return_jobs_done: + ADV_N_JOBS(&state->earliest_job, n_ret_jobs); - if (job->status < IMB_STATUS_COMPLETED) { - /* force job to completion */ - completed_jobs += complete_job(state, job); - } + if (state->earliest_job == state->next_job) { + state->earliest_job = -1; /* becomes empty */ + state->next_job = 0; } - return completed_jobs; + return n_ret_jobs; + +return_invalid_job: + jobs[i]->status = IMB_STATUS_INVALID_ARGS; + jobs[0] = jobs[i]; + return 0; } uint32_t -SUBMIT_BURST(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) +SUBMIT_BURST(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) { - return submit_burst_and_check(state, jobs, n_jobs, 1); + return submit_burst_and_check(state, n_jobs, jobs, 1); } uint32_t -SUBMIT_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs) +SUBMIT_BURST_NOCHECK(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) { - return submit_burst_and_check(state, jobs, n_jobs, 0); + return submit_burst_and_check(state, n_jobs, jobs, 0); +} + +uint32_t +FLUSH_BURST(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs) +{ + uint32_t i, max_ret_jobs, n_ret_jobs = 0; + + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } +#endif + /* check if any jobs in queue */ + max_ret_jobs = queue_sz(state); + if (max_ret_jobs == 0) + return 0; + + /* set max number of jobs to return */ + if (max_ret_jobs > max_jobs) + max_ret_jobs = max_jobs; + + for (i = 0; i < max_ret_jobs; i++) { + IMB_JOB *job = JOBS(state, state->earliest_job); + + if (job->status < IMB_STATUS_COMPLETED) + complete_job(state, job); + + jobs[n_ret_jobs++] = job; + ADV_JOBS(&state->earliest_job); + } + + if (state->earliest_job == state->next_job) { + state->earliest_job = -1; /* becomes empty */ + state->next_job = 0; + } + + return n_ret_jobs; } __forceinline diff --git a/lib/include/noaesni.h b/lib/include/noaesni.h index 272b3a75..e809326d 100644 --- a/lib/include/noaesni.h +++ b/lib/include/noaesni.h @@ -39,12 +39,17 @@ IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_no_aesni(IMB_MGR *state); IMB_DLL_EXPORT uint32_t -submit_burst_sse_no_aesni(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs); +get_next_burst_sse_no_aesni(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs); +submit_burst_sse_no_aesni(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse_no_aesni(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_sse_no_aesni(IMB_MGR *state, const uint32_t max_jobs, + IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_cipher_burst_sse_no_aesni(IMB_MGR *state, IMB_JOB *jobs, diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 78f0d12b..15cf5647 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -786,8 +786,9 @@ typedef IMB_JOB *(*submit_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*get_completed_job_t)(struct IMB_MGR *); typedef IMB_JOB *(*flush_job_t)(struct IMB_MGR *); typedef uint32_t (*queue_size_t)(struct IMB_MGR *); -typedef uint32_t (*submit_burst_t)(struct IMB_MGR *, - struct IMB_JOB *, const uint32_t); +typedef uint32_t (*burst_fn_t)(struct IMB_MGR *, + const uint32_t, + struct IMB_JOB **); typedef uint32_t (*submit_cipher_burst_t)(struct IMB_MGR *, struct IMB_JOB *, const uint32_t, @@ -1042,7 +1043,8 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); /* TOP LEVEL (IMB_MGR) Data structure fields */ -#define IMB_MAX_JOBS 128 +#define IMB_MAX_BURST_SIZE 128 +#define IMB_MAX_JOBS (IMB_MAX_BURST_SIZE * 2) typedef struct IMB_MGR { @@ -1179,8 +1181,10 @@ typedef struct IMB_MGR { chacha_poly_enc_dec_update_t chacha20_poly1305_dec_update; chacha_poly_finalize_t chacha20_poly1305_finalize; - submit_burst_t submit_burst; - submit_burst_t submit_burst_nocheck; + burst_fn_t get_next_burst; + burst_fn_t submit_burst; + burst_fn_t submit_burst_nocheck; + burst_fn_t flush_burst; submit_cipher_burst_t submit_cipher_burst; submit_cipher_burst_t submit_cipher_burst_nocheck; submit_hash_burst_t submit_hash_burst; @@ -1591,30 +1595,59 @@ IMB_DLL_EXPORT void init_mb_mgr_auto(IMB_MGR *state, IMB_ARCH *arch); #define IMB_QUEUE_SIZE(_mgr) ((_mgr)->queue_size((_mgr))) /** - * Submit multiple jobs to be processed synchronously after validating. + * @brief Get next available burst + * (list of pointers to available IMB_JOB structures). * - * @param [in] _mgr Pointer to initialized IMB_MGR structure - * @param [in,out] _jobs Pointer to array of IMB_JOB structures - * @param [in] _n_jobs Number of jobs to process + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * @param [in] _n_jobs Requested number of burst jobs + * @param [out] _jobs List of pointers to returned jobs * - * @return Number of completed jobs + * @return Number of returned jobs. + * May be less than number of requested jobs if not enough space in + * queue. IMB_FLUSH_BURST() can be used to free up space. */ -#define IMB_SUBMIT_BURST(_mgr, _jobs, _n_jobs) \ - ((_mgr)->submit_burst((_mgr), (_jobs), (_n_jobs))) +#define IMB_GET_NEXT_BURST(_mgr, _n_jobs, _jobs) \ + ((_mgr)->get_next_burst((_mgr), (_n_jobs), (_jobs))) /** - * Submit multiple jobs to be processed synchronously without validating. + * Submit multiple jobs to be processed after validating. * - * This is more performant but less secure than IMB_SUBMIT_BURST(). + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * @param [in] _n_jobs Number of jobs to submit for processing + * @param [in,out] _jobs In: List of pointers to jobs for submission + * Out: List of pointers to completed jobs * - * @param [in] _mgr Pointer to initialized IMB_MGR structure - * @param [in,out] _jobs Pointer to array of IMB_JOB structures - * @param [in] _n_jobs Number of jobs to process + * @return Number of completed jobs or zero on error. + * If zero, imb_get_errno() can be used to check for potential + * error conditions and _jobs[0] contains pointer to invalid job + */ +#define IMB_SUBMIT_BURST(_mgr, _n_jobs, _jobs) \ + ((_mgr)->submit_burst((_mgr), (_n_jobs), (_jobs))) + +/** + * Submit multiple jobs to be processed without validating. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * @param [in] _n_jobs Number of jobs to submit for processing + * @param [in,out] _jobs In: List of pointers to jobs for submission + * Out: List of pointers to completed jobs + * + * @return Number of completed jobs or zero on error + */ +#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _n_jobs, _jobs) \ + ((_mgr)->submit_burst_nocheck((_mgr), (_n_jobs), (_jobs))) + +/** + * @brief Force up to \a max_jobs outstanding jobs to completion. + * + * @param [in,out] _mgr Pointer to initialized IMB_MGR structure + * @param [in] _max_jobs Maximum number of jobs to flush + * @param [out] _jobs List of pointers to completed jobs * * @return Number of completed jobs */ -#define IMB_SUBMIT_BURST_NOCHECK(_mgr, _jobs, _n_jobs) \ - ((_mgr)->submit_burst_nocheck((_mgr), (_jobs), (_n_jobs))) +#define IMB_FLUSH_BURST(_mgr, _max_jobs, _jobs) \ + ((_mgr)->flush_burst((_mgr), (_max_jobs), (_jobs))) /** * Submit multiple cipher jobs to be processed synchronously after validating. diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index ca57083c..d3b8ed1c 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -164,8 +164,10 @@ #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_no_aesni #define GET_NEXT_JOB get_next_job_sse_no_aesni #define GET_COMPLETED_JOB get_completed_job_sse_no_aesni +#define GET_NEXT_BURST get_next_burst_sse_no_aesni #define SUBMIT_BURST submit_burst_sse_no_aesni #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_no_aesni +#define FLUSH_BURST flush_burst_sse_no_aesni #define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_no_aesni #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_no_aesni #define SUBMIT_HASH_BURST submit_hash_burst_sse_no_aesni @@ -497,8 +499,10 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs) /* set SSE NO AESNI handlers */ state->get_next_job = get_next_job_sse_no_aesni; state->submit_job = submit_job_sse_no_aesni; + state->get_next_burst = get_next_burst_sse_no_aesni; state->submit_burst = submit_burst_sse_no_aesni; state->submit_burst_nocheck= submit_burst_nocheck_sse_no_aesni; + state->flush_burst = flush_burst_sse_no_aesni; state->submit_cipher_burst = submit_cipher_burst_sse_no_aesni; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse_no_aesni; diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index bf58083c..1ee34f46 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -168,8 +168,10 @@ #define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse #define GET_NEXT_JOB get_next_job_sse #define GET_COMPLETED_JOB get_completed_job_sse +#define GET_NEXT_BURST get_next_burst_sse #define SUBMIT_BURST submit_burst_sse #define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse +#define FLUSH_BURST flush_burst_sse #define SUBMIT_CIPHER_BURST submit_cipher_burst_sse #define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse #define SUBMIT_HASH_BURST submit_hash_burst_sse @@ -807,8 +809,10 @@ init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) /* set SSE handlers */ state->get_next_job = get_next_job_sse; state->submit_job = submit_job_sse; + state->get_next_burst = get_next_burst_sse; state->submit_burst = submit_burst_sse; state->submit_burst_nocheck= submit_burst_nocheck_sse; + state->flush_burst = flush_burst_sse; state->submit_cipher_burst = submit_cipher_burst_sse; state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse; state->submit_hash_burst = submit_hash_burst_sse; -- GitLab From 586273206a79a7fca4b11d487a0aca6a219d74ef Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 25 Jul 2022 14:51:21 +0000 Subject: [PATCH 279/369] perf: Update perf app for async burst API --- perf/ipsec_perf.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 2d29a4c4..eac860cb 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -2108,14 +2108,18 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, /* test burst api */ if (test_api == TEST_API_BURST) { uint32_t num_jobs = num_iter; + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = {NULL}; while (num_jobs && timebox_on) { - uint32_t n_jobs = - (num_jobs / burst_size) ? burst_size : num_jobs; + uint32_t n = (num_jobs / burst_size) ? + burst_size : num_jobs; + + while (IMB_GET_NEXT_BURST(mb_mgr, n, jobs) < n) + IMB_FLUSH_BURST(mb_mgr, n, jobs); /* set all job params */ - for (i = 0; i < n_jobs; i++) { - job = &jobs[i]; + for (i = 0; i < n; i++) { + IMB_JOB *job = jobs[i]; *job = job_template; if (segment_size != 0) @@ -2133,10 +2137,8 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } /* submit burst */ #ifdef DEBUG - const uint32_t completed_jobs = - IMB_SUBMIT_BURST(mb_mgr, jobs, n_jobs); - - if (completed_jobs != n_jobs) { + jobs_done += IMB_SUBMIT_BURST(mb_mgr, n, jobs); + if (jobs_done == 0) { const int err = imb_get_errno(mb_mgr); if (err != 0) { @@ -2145,11 +2147,13 @@ do_test(IMB_MGR *mb_mgr, struct params_s *params, } } #else - IMB_SUBMIT_BURST_NOCHECK(mb_mgr, jobs, n_jobs); + jobs_done += + IMB_SUBMIT_BURST_NOCHECK(mb_mgr, n, jobs); #endif - num_jobs -= n_jobs; + num_jobs -= n; } - jobs_done = num_iter - num_jobs; + jobs_done += + IMB_FLUSH_BURST(mb_mgr, IMB_MAX_BURST_SIZE, jobs); /* test cipher-only burst api */ } else if (test_api == TEST_API_CIPHER_BURST) { -- GitLab From 48de2da1d376085350f838864441d3384655ba38 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Mon, 25 Jul 2022 14:52:34 +0000 Subject: [PATCH 280/369] test: Rework tests for async burst API --- test/aes_test.c | 212 +++++++++++++------ test/api_test.c | 124 +++++++++-- test/chacha20_poly1305_test.c | 13 +- test/ctr_test.c | 361 +++++++++++++++++++++++++-------- test/gcm_test.c | 11 +- test/hmac_sha1_test.c | 158 +++++++++++++-- test/hmac_sha256_sha512_test.c | 255 +++++++++++++++++++++-- 7 files changed, 927 insertions(+), 207 deletions(-) diff --git a/test/aes_test.c b/test/aes_test.c index 8aa17004..2b1c2540 100644 --- a/test/aes_test.c +++ b/test/aes_test.c @@ -36,10 +36,7 @@ #include "gcm_ctr_vectors_test.h" #include "utils.h" -typedef enum { - BURST_TYPE_GENERIC = 0, - BURST_TYPE_CIPHER, -} BURST_TYPE; +#define MAX_BURST_JOBS 64 int aes_test(struct IMB_MGR *mb_mgr); @@ -1990,10 +1987,126 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, - const int num_jobs, - const BURST_TYPE burst_type) + const int num_jobs) { - struct IMB_JOB *job, jobs[24]; + struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = {NULL}; + uint8_t padding[16]; + uint8_t **targets = malloc(num_jobs * sizeof(void *)); + int i, completed_jobs, jobs_rx = 0, ret = -1; + + if (targets == NULL) + goto end_alloc; + + memset(targets, 0, num_jobs * sizeof(void *)); + memset(padding, -1, sizeof(padding)); + + for (i = 0; i < num_jobs; i++) { + targets[i] = malloc(text_len + (sizeof(padding) * 2)); + if (targets[i] == NULL) + goto end_alloc; + memset(targets[i], -1, text_len + (sizeof(padding) * 2)); + if (in_place) { + /* copy input text to the allocated buffer */ + memcpy(targets[i] + sizeof(padding), in_text, text_len); + } + } + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < (uint32_t)num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + + job->cipher_direction = dir; + job->chain_order = order; + job->key_len_in_bytes = key_len; + job->cipher_mode = cipher; + job->hash_alg = IMB_AUTH_NULL; + + if (!in_place) { + job->dst = targets[i] + sizeof(padding); + job->src = in_text; + } else { + job->dst = targets[i] + sizeof(padding); + job->src = targets[i] + sizeof(padding); + } + + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + job->iv = iv; + job->iv_len_in_bytes = 16; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = text_len; + job->user_data = targets[i]; + job->user_data2 = (void *)((uint64_t)i); + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + if (completed_jobs == 0) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + } + +check_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + + if (!aes_job_ok(job, out_text, job->user_data, padding, + sizeof(padding), text_len)) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %d jobs, received %d\n", + num_jobs, jobs_rx); + goto end; + } + goto check_burst_jobs; + } + ret = 0; + + end: + + end_alloc: + if (targets != NULL) { + for (i = 0; i < num_jobs; i++) + free(targets[i]); + free(targets); + } + + return ret; +} + +static int +test_aes_many_cipher_burst(struct IMB_MGR *mb_mgr, + void *enc_keys, + void *dec_keys, + const void *iv, + const uint8_t *in_text, + const uint8_t *out_text, + const unsigned text_len, + const int dir, + const IMB_CIPHER_MODE cipher, + const int in_place, + const int key_len, + const int num_jobs) +{ + struct IMB_JOB *job, jobs[MAX_BURST_JOBS]; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -2019,13 +2132,6 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, job = &jobs[i]; /* only set fields for generic burst API */ - if (burst_type == BURST_TYPE_GENERIC) { - job->cipher_direction = dir; - job->chain_order = order; - job->key_len_in_bytes = key_len; - job->cipher_mode = cipher; - job->hash_alg = IMB_AUTH_NULL; - } if (!in_place) { job->dst = targets[i] + sizeof(padding); job->src = in_text; @@ -2044,12 +2150,8 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, job->user_data2 = (void *)((uint64_t)i); } - if (burst_type == BURST_TYPE_GENERIC) - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); - else - completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, - cipher, dir, key_len); - + completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, + cipher, dir, key_len); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -2157,8 +2259,7 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, cipher, 0, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_GENERIC)) { + vec_tab[vect].Klen, num_jobs)) { printf("error #%d encrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2184,8 +2285,7 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, cipher, 0, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_GENERIC)) { + vec_tab[vect].Klen, num_jobs)) { printf("error #%d decrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2211,8 +2311,7 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, cipher, 1, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_GENERIC)) { + vec_tab[vect].Klen, num_jobs)) { printf("error #%d encrypt burst in-place\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2238,8 +2337,7 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, (unsigned) vec_tab[vect].Plen, IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, cipher, 1, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_GENERIC)) { + vec_tab[vect].Klen, num_jobs)) { printf("error #%d decrypt burst in-place\n", vect + 1); test_suite_update(ctx, 0, 1); } else { @@ -2253,42 +2351,36 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, if (cipher != IMB_CIPHER_CBC) continue; - if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, - vec_tab[vect].IV, - vec_tab[vect].P, vec_tab[vect].C, - (unsigned) vec_tab[vect].Plen, - IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, - cipher, 0, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_CIPHER)) { + if (test_aes_many_cipher_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, cipher, 0, + vec_tab[vect].Klen, num_jobs)) { printf("error #%d encrypt cipher burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { test_suite_update(ctx, 1, 0); } - if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, - vec_tab[vect].IV, - vec_tab[vect].C, vec_tab[vect].P, - (unsigned) vec_tab[vect].Plen, - IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, - cipher, 0, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_CIPHER)) { + if (test_aes_many_cipher_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, cipher, 0, + vec_tab[vect].Klen, num_jobs)) { printf("error #%d decrypt cipher burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { test_suite_update(ctx, 1, 0); } - if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, - vec_tab[vect].IV, - vec_tab[vect].P, vec_tab[vect].C, - (unsigned) vec_tab[vect].Plen, - IMB_DIR_ENCRYPT, IMB_ORDER_CIPHER_HASH, - cipher, 1, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_CIPHER)) { + if (test_aes_many_cipher_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].P, vec_tab[vect].C, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_ENCRYPT, cipher, 1, + vec_tab[vect].Klen, num_jobs)) { printf("error #%d encrypt cipher burst " "in-place\n", vect + 1); test_suite_update(ctx, 0, 1); @@ -2296,14 +2388,12 @@ test_aes_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ctx, 1, 0); } - if (test_aes_many_burst(mb_mgr, enc_keys, dec_keys, - vec_tab[vect].IV, - vec_tab[vect].C, vec_tab[vect].P, - (unsigned) vec_tab[vect].Plen, - IMB_DIR_DECRYPT, IMB_ORDER_HASH_CIPHER, - cipher, 1, - vec_tab[vect].Klen, num_jobs, - BURST_TYPE_CIPHER)) { + if (test_aes_many_cipher_burst(mb_mgr, enc_keys, dec_keys, + vec_tab[vect].IV, + vec_tab[vect].C, vec_tab[vect].P, + (unsigned) vec_tab[vect].Plen, + IMB_DIR_DECRYPT, cipher, 1, + vec_tab[vect].Klen, num_jobs)) { printf("error #%d decrypt cipher " "burst in-place\n", vect + 1); test_suite_update(ctx, 0, 1); @@ -2639,7 +2729,7 @@ int aes_test(struct IMB_MGR *mb_mgr) { const int num_jobs_tab[] = { - 1, 3, 4, 5, 7, 8, 9, 15, 16, 17 + 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; unsigned i; int errors = 0; diff --git a/test/api_test.c b/test/api_test.c index 7effce3e..7217794e 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -643,16 +643,19 @@ static int is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const int test_num, int expected_errnum) { - IMB_JOB jobs[MAX_BURST_JOBS] = {0}; - uint32_t i, completed_jobs; + IMB_JOB * jobs[MAX_BURST_JOBS] = {NULL}; + uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; int err; + while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) + IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); + /* duplicate job to test */ - for (i = 0; i < MAX_BURST_JOBS; i++) - jobs[i] = *job; + for (i = 0; i < n_jobs; i++) + *jobs[i] = *job; /* submit the job for processing */ - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, MAX_BURST_JOBS); + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); if (completed_jobs != 0) { printf("%s : test %d, hash_alg %d, chain_order %d, " "cipher_dir %d, cipher_mode %d : " @@ -673,7 +676,7 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, return 0; } - if (jobs[0].status != IMB_STATUS_INVALID_ARGS) { + if (jobs[0]->status != IMB_STATUS_INVALID_ARGS) { printf("%s : test %d, hash_alg %d, chain_order %d, " "cipher_dir %d, cipher_mode %d : " "unexpected job->status %d != IMB_STATUS_INVALID_ARGS\n", @@ -693,44 +696,123 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, static int test_burst_api(struct IMB_MGR *mb_mgr) { - struct IMB_JOB *job = NULL, jobs[MAX_BURST_JOBS] = {0}; + struct IMB_JOB *job = NULL, *jobs[MAX_BURST_JOBS] = {NULL}; uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; int err; printf("BURST API behavior test:\n"); - /* ======== test 1 : NULL jobs array */ + /* ======== test 1 : NULL pointer to jobs array */ + + if (mb_mgr->features & IMB_FEATURE_SAFE_PARAM) { + IMB_JOB **null_jobs = NULL; + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, null_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed " + "jobs\n", __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_BURST) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf("."); + + /* ======== test 2 : NULL jobs array */ + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed " + "jobs\n", __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_JOB) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf("."); + + /* ========== test 3: invalid burst size */ + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, + IMB_MAX_BURST_SIZE + 1, jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed " + "jobs\n", __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, job, n_jobs); + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_BURST_SIZE) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf("."); + } + + /* ======== test 4 : invalid job order */ + + while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) + IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); + + /* fill in valid jobs */ + for (i = 0; i < n_jobs; i++) { + job = jobs[i]; + fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, + IMB_ORDER_CIPHER_HASH, NULL, NULL); + } + + /* set invalid job order */ + jobs[n_jobs / 2] = jobs[n_jobs - 1]; + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); if (completed_jobs != 0) { - printf("%s: test %d, unexpected number of completed jobs\n", - __func__, TEST_INVALID_BURST); + printf("%s: test %d, unexpected number of completed " + "jobs\n", __func__, TEST_INVALID_BURST); return 1; } printf("."); err = imb_get_errno(mb_mgr); - if (err != IMB_ERR_NULL_JOB) { + if (err != IMB_ERR_BURST_OOO) { printf("%s: test %d, unexpected error: %s\n", - __func__, TEST_INVALID_BURST, imb_get_strerror(err)); + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); return 1; } printf("."); - /* ======== test 2 : invalid job */ + /* ======== test 5 : invalid job */ + + while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) + IMB_FLUSH_BURST(mb_mgr, n_jobs, jobs); /* fill in valid jobs */ for (i = 0; i < n_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; fill_in_job(job, IMB_CIPHER_CBC, IMB_DIR_ENCRYPT, IMB_AUTH_NULL, IMB_ORDER_CIPHER_HASH, NULL, NULL); } /* set a single invalid field */ - jobs[n_jobs - 1].enc_keys = NULL; + jobs[n_jobs - 1]->enc_keys = NULL; /* no jobs should complete if any job is invalid */ - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, n_jobs); + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, jobs); if (completed_jobs != 0) { printf("%s: test %d, unexpected number of completed jobs\n", __func__, TEST_INVALID_BURST); @@ -746,6 +828,14 @@ test_burst_api(struct IMB_MGR *mb_mgr) } printf("."); + /* check invalid job returned in jobs[0] */ + if (jobs[0] != jobs[n_jobs - 1]) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + "invalid job not returned in burst_job[0]"); + return 1; + } + printf("\n"); return 0; } diff --git a/test/chacha20_poly1305_test.c b/test/chacha20_poly1305_test.c index 88689196..4009a0ef 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/chacha20_poly1305_test.c @@ -444,7 +444,7 @@ test_aead(struct IMB_MGR *mb_mgr, */ /* create job array */ - IMB_JOB jobs[32] = {0}; + IMB_JOB * jobs[32] = {NULL}; jobs_rx = 0; @@ -463,11 +463,14 @@ test_aead(struct IMB_MGR *mb_mgr, } } + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < (uint32_t)num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + /** * Set all job params before submitting burst */ for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; job->cipher_direction = dir; job->chain_order = IMB_ORDER_HASH_CIPHER; job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; @@ -501,9 +504,9 @@ test_aead(struct IMB_MGR *mb_mgr, job->user_data = auths[i]; } - int completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + uint32_t completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); - if (completed_jobs != num_jobs) { + if (completed_jobs != (uint32_t)num_jobs) { int err = imb_get_errno(mb_mgr); if (err != 0) { @@ -518,7 +521,7 @@ test_aead(struct IMB_MGR *mb_mgr, } for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { printf("job %d status not complete!\n", i+1); diff --git a/test/ctr_test.c b/test/ctr_test.c index b1f2077a..153f2ba4 100644 --- a/test/ctr_test.c +++ b/test/ctr_test.c @@ -34,11 +34,6 @@ #include "gcm_ctr_vectors_test.h" #include "utils.h" -typedef enum { - BURST_TYPE_GENERIC = 0, - BURST_TYPE_CIPHER, -} BURST_TYPE; - #define MAX_CTR_JOBS 32 #define BYTE_ROUND_UP(x) ((x + 7) / 8) @@ -1464,11 +1459,10 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, const IMB_CIPHER_DIRECTION dir, const IMB_CHAIN_ORDER order, const IMB_CIPHER_MODE alg, - const uint32_t num_jobs, - const BURST_TYPE burst_type) + const uint32_t num_jobs) { uint32_t text_byte_len, i, completed_jobs, jobs_rx = 0; - struct IMB_JOB *job, jobs[MAX_CTR_JOBS]; + struct IMB_JOB *job, *jobs[MAX_CTR_JOBS]; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int ret = -1; @@ -1492,8 +1486,11 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, memset(targets[i], -1, text_byte_len + (sizeof(padding) * 2)); } + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; job->cipher_direction = dir; job->chain_order = order; job->dst = targets[i] + sizeof(padding); @@ -1514,13 +1511,133 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, job->user_data2 = (void *)((uint64_t)i); } + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; - if (burst_type == BURST_TYPE_GENERIC) - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + if (memcmp(out_text, targets[i] + sizeof(padding), + text_byte_len)) { + printf("mismatched\n"); + hexdump(stderr, "Target", targets[i] + sizeof(padding), + text_byte_len); + hexdump(stderr, "Expected", out_text, text_byte_len); + goto end; + } + if (memcmp(padding, targets[i], sizeof(padding))) { + printf("overwrite head\n"); + hexdump(stderr, "Target", targets[i], text_byte_len + + (sizeof(padding) * 2)); + goto end; + } + if (memcmp(padding, targets[i] + sizeof(padding) + + text_byte_len, sizeof(padding))) { + printf("overwrite tail\n"); + hexdump(stderr, "Target", targets[i], text_byte_len + + (sizeof(padding) * 2)); + goto end; + } + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + end: + + end_alloc: + if (targets != NULL) { + for (i = 0; i < num_jobs; i++) + free(targets[i]); + free(targets); + } + + return ret; +} + +static int +test_ctr_cipher_burst(struct IMB_MGR *mb_mgr, + const void *expkey, + unsigned key_len, + const void *iv, + unsigned iv_len, + const uint8_t *in_text, + const uint8_t *out_text, + unsigned text_len, + const IMB_CIPHER_DIRECTION dir, + const IMB_CHAIN_ORDER order, + const IMB_CIPHER_MODE alg, + const uint32_t num_jobs) +{ + uint32_t text_byte_len, i, completed_jobs, jobs_rx = 0; + struct IMB_JOB *job, jobs[MAX_CTR_JOBS]; + uint8_t padding[16]; + uint8_t **targets = malloc(num_jobs * sizeof(void *)); + int ret = -1; + + if (targets == NULL) + goto end_alloc; + + /* Get number of bytes (in case algo is CNTR_BITLEN) */ + if (alg == IMB_CIPHER_CNTR) + text_byte_len = text_len; else - completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, - alg, dir, key_len); + text_byte_len = BYTE_ROUND_UP(text_len); + + memset(targets, 0, num_jobs * sizeof(void *)); + memset(padding, -1, sizeof(padding)); + + for (i = 0; i < num_jobs; i++) { + targets[i] = malloc(text_byte_len + (sizeof(padding) * 2)); + if (targets[i] == NULL) + goto end_alloc; + memset(targets[i], -1, text_byte_len + (sizeof(padding) * 2)); + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + job->cipher_direction = dir; + job->chain_order = order; + job->dst = targets[i] + sizeof(padding); + job->src = in_text; + job->cipher_mode = alg; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->key_len_in_bytes = key_len; + job->iv = iv; + job->iv_len_in_bytes = iv_len; + job->cipher_start_src_offset_in_bytes = 0; + if (alg == IMB_CIPHER_CNTR) + job->msg_len_to_cipher_in_bytes = text_byte_len; + else + job->msg_len_to_cipher_in_bits = text_len; + job->hash_alg = IMB_AUTH_NULL; + job->user_data = targets[i]; + job->user_data2 = (void *)((uint64_t)i); + } + + completed_jobs = IMB_SUBMIT_CIPHER_BURST(mb_mgr, jobs, num_jobs, + alg, dir, key_len); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -1719,7 +1836,7 @@ test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, const uint32_t vectors_cnt, const IMB_CIPHER_MODE alg, const uint32_t num_jobs) { - uint32_t vect, burst_type; + uint32_t vect; DECLARE_ALIGNED(uint32_t expkey[4*15], 16); DECLARE_ALIGNED(uint32_t dust[4*15], 16); @@ -1765,26 +1882,62 @@ test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, return; } - for (burst_type = BURST_TYPE_GENERIC; - burst_type <= BURST_TYPE_CIPHER; burst_type++) { - const char *bt = (burst_type == BURST_TYPE_CIPHER) ? - "cipher-only burst" : "burst"; + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].P, vectors[vect].C, + (unsigned) vectors[vect].Plen, + IMB_DIR_ENCRYPT, + IMB_ORDER_CIPHER_HASH, alg, num_jobs)) { + printf("error #%u encrypt burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ctr_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].C, vectors[vect].P, + (unsigned) vectors[vect].Plen, + IMB_DIR_DECRYPT, + IMB_ORDER_HASH_CIPHER, alg, num_jobs)) { + printf("error #%u decrypt burst\n", vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } - /* skip bitlen cipher-only burst api tests */ - if (alg == IMB_CIPHER_CNTR_BITLEN && - burst_type == BURST_TYPE_CIPHER) - continue; + if (vectors[vect].IVlen == 12 && + alg == IMB_CIPHER_CNTR) { + /* IV in the table didn't + * include block counter (12 bytes). + * Let's encrypt & decrypt the same but + * with 16 byte IV that includes block counter. + */ + const unsigned new_iv_len = 16; + const unsigned orig_iv_len = 12; + uint8_t local_iv[16]; + + memcpy(local_iv, vectors[vect].IV, orig_iv_len); + /* 32-bit 0x1 in BE == 0x01000000 in LE */ + local_iv[12] = 0x00; + local_iv[13] = 0x00; + local_iv[14] = 0x00; + local_iv[15] = 0x01; if (test_ctr_burst(mb_mgr, expkey, vectors[vect].Klen, - vectors[vect].IV, - (unsigned) vectors[vect].IVlen, - vectors[vect].P, vectors[vect].C, - (unsigned) vectors[vect].Plen, + local_iv, new_iv_len, + vectors[vect].P, + vectors[vect].C, (unsigned) + vectors[vect].Plen, IMB_DIR_ENCRYPT, - IMB_ORDER_CIPHER_HASH, alg, - num_jobs, burst_type)) { - printf("error #%u encrypt %s\n", vect + 1, bt); + IMB_ORDER_CIPHER_HASH, + alg, num_jobs)) { + printf("error #%u encrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { test_suite_update(ctx, 1, 0); @@ -1792,71 +1945,107 @@ test_ctr_vectors_burst(struct IMB_MGR *mb_mgr, if (test_ctr_burst(mb_mgr, expkey, vectors[vect].Klen, - vectors[vect].IV, - (unsigned) vectors[vect].IVlen, - vectors[vect].C, vectors[vect].P, - (unsigned) vectors[vect].Plen, + local_iv, new_iv_len, + vectors[vect].C, + vectors[vect].P, (unsigned) + vectors[vect].Plen, IMB_DIR_DECRYPT, - IMB_ORDER_HASH_CIPHER, alg, - num_jobs, burst_type)) { - printf("error #%u decrypt %s\n", vect + 1, bt); + IMB_ORDER_HASH_CIPHER, + alg, num_jobs)) { + printf("error #%u decrypt burst\n", vect + 1); test_suite_update(ctx, 0, 1); } else { test_suite_update(ctx, 1, 0); } + } + + /* skip bitlen cipher-only burst api tests */ + if (alg == IMB_CIPHER_CNTR_BITLEN) + continue; + + if (test_ctr_cipher_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].P, vectors[vect].C, + (unsigned) vectors[vect].Plen, + IMB_DIR_ENCRYPT, + IMB_ORDER_CIPHER_HASH, alg, + num_jobs)) { + printf("error #%u encrypt cipher-only burst\n", + vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ctr_cipher_burst(mb_mgr, + expkey, vectors[vect].Klen, + vectors[vect].IV, + (unsigned) vectors[vect].IVlen, + vectors[vect].C, vectors[vect].P, + (unsigned) vectors[vect].Plen, + IMB_DIR_DECRYPT, + IMB_ORDER_HASH_CIPHER, alg, + num_jobs)) { + printf("error #%u decrypt cipher-only burst\n", + vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } - if (vectors[vect].IVlen == 12 && - alg == IMB_CIPHER_CNTR) { - /* IV in the table didn't - * include block counter (12 bytes). - * Let's encrypt & decrypt the same but - * with 16 byte IV that includes block counter. - */ - const unsigned new_iv_len = 16; - const unsigned orig_iv_len = 12; - uint8_t local_iv[16]; - - memcpy(local_iv, vectors[vect].IV, orig_iv_len); - /* 32-bit 0x1 in BE == 0x01000000 in LE */ - local_iv[12] = 0x00; - local_iv[13] = 0x00; - local_iv[14] = 0x00; - local_iv[15] = 0x01; - - if (test_ctr_burst(mb_mgr, - expkey, vectors[vect].Klen, - local_iv, new_iv_len, - vectors[vect].P, - vectors[vect].C, (unsigned) - vectors[vect].Plen, - IMB_DIR_ENCRYPT, - IMB_ORDER_CIPHER_HASH, - alg, num_jobs, burst_type)) { - printf("error #%u encrypt %s\n", - vect + 1, bt); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } - - if (test_ctr_burst(mb_mgr, - expkey, vectors[vect].Klen, - local_iv, new_iv_len, - vectors[vect].C, - vectors[vect].P, (unsigned) - vectors[vect].Plen, - IMB_DIR_DECRYPT, - IMB_ORDER_HASH_CIPHER, - alg, num_jobs, burst_type)) { - printf("error #%u decrypt %s\n", - vect + 1, bt); - test_suite_update(ctx, 0, 1); - } else { - test_suite_update(ctx, 1, 0); - } + if (vectors[vect].IVlen == 12 && + alg == IMB_CIPHER_CNTR) { + /* IV in the table didn't + * include block counter (12 bytes). + * Let's encrypt & decrypt the same but + * with 16 byte IV that includes block counter. + */ + const unsigned new_iv_len = 16; + const unsigned orig_iv_len = 12; + uint8_t local_iv[16]; + + memcpy(local_iv, vectors[vect].IV, orig_iv_len); + /* 32-bit 0x1 in BE == 0x01000000 in LE */ + local_iv[12] = 0x00; + local_iv[13] = 0x00; + local_iv[14] = 0x00; + local_iv[15] = 0x01; + + if (test_ctr_cipher_burst(mb_mgr, + expkey, vectors[vect].Klen, + local_iv, new_iv_len, + vectors[vect].P, + vectors[vect].C, (unsigned) + vectors[vect].Plen, + IMB_DIR_ENCRYPT, + IMB_ORDER_CIPHER_HASH, + alg, num_jobs)) { + printf("error #%u encrypt cipher-only burst\n", + vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ctr_cipher_burst(mb_mgr, + expkey, vectors[vect].Klen, + local_iv, new_iv_len, + vectors[vect].C, + vectors[vect].P, (unsigned) + vectors[vect].Plen, + IMB_DIR_DECRYPT, + IMB_ORDER_HASH_CIPHER, + alg, num_jobs)) { + printf("error #%u decrypt cipher-only burst\n", + vect + 1); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); } } - } + } printf("\n"); } diff --git a/test/gcm_test.c b/test/gcm_test.c index 93094027..ba1bd767 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -1644,11 +1644,14 @@ aes_gcm_burst(IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher_mode, const IMB_SGL_STATE sgl_state, const uint32_t num_jobs) { - IMB_JOB *job, jobs[GCM_MAX_JOBS]; + IMB_JOB *job, *jobs[GCM_MAX_JOBS]; uint32_t i; + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; job->cipher_mode = cipher_mode; job->chain_order = @@ -1678,7 +1681,7 @@ aes_gcm_burst(IMB_MGR *mb_mgr, } const uint32_t completed_jobs = - IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); + IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -1695,7 +1698,7 @@ aes_gcm_burst(IMB_MGR *mb_mgr, } for (i = 0; i < num_jobs; i++) { - job = &jobs[i]; + job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { printf("job %d status not complete!\n", i+1); diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c index 64c8e4d4..fa45adc9 100644 --- a/test/hmac_sha1_test.c +++ b/test/hmac_sha1_test.c @@ -36,11 +36,6 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr); -typedef enum { - BURST_TYPE_GENERIC = 0, - BURST_TYPE_HASH, -} BURST_TYPE; - #define block_size 64 #define digest_size 20 #define digest96_size 12 @@ -487,7 +482,144 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, static int test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, const struct hmac_sha1_rfc2202_vector *vec, - const uint32_t num_jobs, BURST_TYPE burst_type) + const uint32_t num_jobs) +{ + struct IMB_JOB *job, *jobs[max_burst_jobs] = {NULL}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0; + int ret = -1, err; + uint8_t key[block_size]; + uint8_t buf[block_size]; + DECLARE_ALIGNED(uint8_t ipad_hash[digest_size], 16); + DECLARE_ALIGNED(uint8_t opad_hash[digest_size], 16); + uint32_t completed_jobs = 0, key_len = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + vec->digest_len + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (vec->key_len <= block_size) { + memcpy(key, vec->key, vec->key_len); + key_len = (int) vec->key_len; + } else { + IMB_SHA1(mb_mgr, vec->key, vec->key_len, key); + key_len = digest_size; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->digest_len; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = vec->data; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->data_len; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + + job->user_data = auths[i]; + + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + +check_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + + if (!hmac_sha1_job_ok(vec, job, job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %d jobs, received %d\n", + num_jobs, jobs_rx); + goto end; + } + goto check_burst_jobs; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, + const struct hmac_sha1_rfc2202_vector *vec, + const uint32_t num_jobs) { struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; uint8_t padding[16]; @@ -568,12 +700,8 @@ test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, } - if (burst_type == BURST_TYPE_GENERIC) - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); - else - completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, - IMB_AUTH_HMAC_SHA_1); - + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + IMB_AUTH_HMAC_SHA_1); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -651,14 +779,14 @@ test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, test_suite_update(ts, 1, 0); } if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], - num_jobs, BURST_TYPE_GENERIC)) { + num_jobs)) { printf("error #%d - burst API\n", vect); test_suite_update(ts, 0, 1); } else { test_suite_update(ts, 1, 0); } - if (test_hmac_sha1_burst(mb_mgr, &hmac_sha1_vectors[idx], - num_jobs, BURST_TYPE_HASH)) { + if (test_hmac_sha1_hash_burst(mb_mgr, &hmac_sha1_vectors[idx], + num_jobs)) { printf("error #%d - hash-only burst API\n", vect); test_suite_update(ts, 0, 1); } else { diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index 4348d0c3..8549934e 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -38,11 +38,6 @@ int hmac_sha256_sha512_test(struct IMB_MGR *mb_mgr); -typedef enum { - BURST_TYPE_GENERIC = 0, - BURST_TYPE_HASH, -} BURST_TYPE; - /* * Test vectors from https://tools.ietf.org/html/rfc4231 */ @@ -1049,8 +1044,236 @@ static int test_hmac_shax_burst(struct IMB_MGR *mb_mgr, const struct hmac_rfc4231_vector *vec, const uint32_t num_jobs, - const int sha_type, - const BURST_TYPE burst_type) + const int sha_type) +{ + struct IMB_JOB *job, *jobs[max_burst_jobs] = {NULL}; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + uint32_t i = 0, jobs_rx = 0, completed_jobs = 0; + int ret = -1, err; + uint8_t key[IMB_SHA_512_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint32_t key_len = 0; + size_t digest_len = 0; + size_t block_size = 0; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + switch (sha_type) { + case 224: + digest_len = vec->hmac_sha224_len; + block_size = IMB_SHA_256_BLOCK_SIZE; + break; + case 256: + digest_len = vec->hmac_sha256_len; + block_size = IMB_SHA_256_BLOCK_SIZE; + break; + case 384: + digest_len = vec->hmac_sha384_len; + block_size = IMB_SHA_384_BLOCK_SIZE; + break; + case 512: + digest_len = vec->hmac_sha512_len; + block_size = IMB_SHA_512_BLOCK_SIZE; + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = + digest_len + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (vec->key_len <= block_size) { + memcpy(key, vec->key, vec->key_len); + key_len = (int) vec->key_len; + } else { + switch (sha_type) { + case 224: + IMB_SHA224(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + break; + case 256: + IMB_SHA256(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + break; + case 384: + IMB_SHA384(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; + break; + case 512: + IMB_SHA512(mb_mgr, vec->key, vec->key_len, key); + key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; + break; + default: + fprintf(stderr, "Wrong SHA type selection 'SHA-%d'!\n", + sha_type); + goto end; + } + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + + switch (sha_type) { + case 224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + case 512: + default: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); + break; + } + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + + switch (sha_type) { + case 224: + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 256: + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 384: + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + case 512: + default: + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); + break; + } + + while (IMB_GET_NEXT_BURST(mb_mgr, num_jobs, jobs) < num_jobs) + IMB_FLUSH_BURST(mb_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->dst = NULL; + job->key_len_in_bytes = 0; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = digest_len; + job->iv = NULL; + job->iv_len_in_bytes = 0; + job->src = vec->data; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = vec->data_len; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->cipher_mode = IMB_CIPHER_NULL; + + switch (sha_type) { + case 224: + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + break; + case 256: + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + break; + case 384: + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + break; + case 512: + default: + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + break; + } + + job->user_data = auths[i]; + + } + + completed_jobs = IMB_SUBMIT_BURST(mb_mgr, num_jobs, jobs); + err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, + imb_get_strerror(err)); + goto end; + } + + check_burst_jobs: + for (i = 0; i < completed_jobs; i++) { + job = jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %d status not complete!\n", i+1); + goto end; + } + + if (!hmac_shax_job_ok(vec, job, sha_type, + job->user_data, + padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + completed_jobs = IMB_FLUSH_BURST(mb_mgr, + num_jobs - completed_jobs, + jobs); + if (completed_jobs == 0) { + printf("Expected %d jobs, received %d\n", + num_jobs, jobs_rx); + goto end; + } + goto check_burst_jobs; + } + ret = 0; + + end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + + end2: + if (auths != NULL) + free(auths); + + return ret; +} + +static int +test_hmac_shax_hash_burst(struct IMB_MGR *mb_mgr, + const struct hmac_rfc4231_vector *vec, + const uint32_t num_jobs, + const int sha_type) { struct IMB_JOB *job, jobs[max_burst_jobs] = {0}; uint8_t padding[16]; @@ -1225,12 +1448,8 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, } - if (burst_type == BURST_TYPE_GENERIC) - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, jobs, num_jobs); - else - completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, - job->hash_alg); - + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, + job->hash_alg); if (completed_jobs != num_jobs) { int err = imb_get_errno(mb_mgr); @@ -1327,17 +1546,15 @@ test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, } if (test_hmac_shax_burst(mb_mgr, &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type, - BURST_TYPE_GENERIC)) { + num_jobs, sha_type)) { printf("error #%d - burst API\n", vect); test_suite_update(ts, 0, 1); } else { test_suite_update(ts, 1, 0); } - if (test_hmac_shax_burst(mb_mgr, - &hmac_sha256_sha512_vectors[idx], - num_jobs, sha_type, - BURST_TYPE_HASH)) { + if (test_hmac_shax_hash_burst(mb_mgr, + &hmac_sha256_sha512_vectors[idx], + num_jobs, sha_type)) { printf("error #%d - hash-only burst API\n", vect); test_suite_update(ts, 0, 1); } else { -- GitLab From 27257a521b61337cfb0ad75cfa46bd9604b5c444 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 26 Jul 2022 20:40:11 +0000 Subject: [PATCH 281/369] test: [API] Extend burst API tests - Test IMB_GET_NEXT_BURST() API - Test IMB_FLUSH_BURST() API --- test/api_test.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/test/api_test.c b/test/api_test.c index 7217794e..434883bb 100644 --- a/test/api_test.c +++ b/test/api_test.c @@ -698,15 +698,14 @@ test_burst_api(struct IMB_MGR *mb_mgr) { struct IMB_JOB *job = NULL, *jobs[MAX_BURST_JOBS] = {NULL}; uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + struct IMB_JOB **null_jobs = NULL; int err; - printf("BURST API behavior test:\n"); + printf("SUBMIT_BURST() API behavior test:\n"); /* ======== test 1 : NULL pointer to jobs array */ if (mb_mgr->features & IMB_FEATURE_SAFE_PARAM) { - IMB_JOB **null_jobs = NULL; - completed_jobs = IMB_SUBMIT_BURST(mb_mgr, n_jobs, null_jobs); if (completed_jobs != 0) { printf("%s: test %d, unexpected number of completed " @@ -837,6 +836,70 @@ test_burst_api(struct IMB_MGR *mb_mgr) } printf("\n"); + + if ((mb_mgr->features & IMB_FEATURE_SAFE_PARAM) == 0) + return 0; + + printf("GET_NEXT_BURST() API behavior test:\n"); + + /* ======== test 6 : NULL pointer to burst job array */ + + completed_jobs = IMB_GET_NEXT_BURST(mb_mgr, n_jobs, null_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_BURST) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf("."); + + /* ======== test 7 : Invalid burst size */ + + completed_jobs = IMB_GET_NEXT_BURST(mb_mgr, + IMB_MAX_BURST_SIZE + 1, jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_BURST_SIZE) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf(".\n"); + + printf("FLUSH_BURST() API behavior test:\n"); + + completed_jobs = IMB_FLUSH_BURST(mb_mgr, n_jobs, null_jobs); + if (completed_jobs != 0) { + printf("%s: test %d, unexpected number of completed jobs\n", + __func__, TEST_INVALID_BURST); + return 1; + } + printf("."); + + err = imb_get_errno(mb_mgr); + if (err != IMB_ERR_NULL_BURST) { + printf("%s: test %d, unexpected error: %s\n", + __func__, TEST_INVALID_BURST, + imb_get_strerror(err)); + return 1; + } + printf(".\n"); + return 0; } -- GitLab From 09f6adc26b39db0e30c40a5940bfc92bdc6618a4 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 26 Jul 2022 20:50:12 +0000 Subject: [PATCH 282/369] lib: Update .def file for async burst API --- lib/libIPSec_MB.def | 70 ++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index ead3896c..ba157f31 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -544,33 +544,43 @@ EXPORTS imb_get_mb_mgr_size @518 imb_set_pointers_mb_mgr @519 imb_get_feature_flags @520 - submit_burst_sse @521 - submit_burst_sse_no_aesni @522 - submit_burst_avx @523 - submit_burst_avx2 @524 - submit_burst_avx512 @525 - submit_burst_nocheck_sse @526 - submit_burst_nocheck_sse_no_aesni @527 - submit_burst_nocheck_avx @528 - submit_burst_nocheck_avx2 @529 - submit_burst_nocheck_avx512 @530 - submit_cipher_burst_sse @531 - submit_cipher_burst_sse_no_aesni @532 - submit_cipher_burst_avx @533 - submit_cipher_burst_avx2 @534 - submit_cipher_burst_avx512 @535 - submit_cipher_burst_nocheck_sse @536 - submit_cipher_burst_nocheck_sse_no_aesni @537 - submit_cipher_burst_nocheck_avx @538 - submit_cipher_burst_nocheck_avx2 @539 - submit_cipher_burst_nocheck_avx512 @540 - submit_hash_burst_sse @541 - submit_hash_burst_sse_no_aesni @542 - submit_hash_burst_avx @543 - submit_hash_burst_avx2 @544 - submit_hash_burst_avx512 @545 - submit_hash_burst_nocheck_sse @546 - submit_hash_burst_nocheck_sse_no_aesni @547 - submit_hash_burst_nocheck_avx @548 - submit_hash_burst_nocheck_avx2 @549 - submit_hash_burst_nocheck_avx512 @550 + get_next_burst_sse @521 + get_next_burst_sse_no_aesni @522 + get_next_burst_avx @523 + get_next_burst_avx2 @524 + get_next_burst_avx512 @525 + submit_burst_sse @526 + submit_burst_sse_no_aesni @527 + submit_burst_avx @528 + submit_burst_avx2 @529 + submit_burst_avx512 @530 + submit_burst_nocheck_sse @531 + submit_burst_nocheck_sse_no_aesni @532 + submit_burst_nocheck_avx @533 + submit_burst_nocheck_avx2 @534 + submit_burst_nocheck_avx512 @535 + flush_burst_sse @536 + flush_burst_sse_no_aesni @537 + flush_burst_avx @538 + flush_burst_avx2 @539 + flush_burst_avx512 @540 + submit_cipher_burst_sse @541 + submit_cipher_burst_sse_no_aesni @542 + submit_cipher_burst_avx @543 + submit_cipher_burst_avx2 @544 + submit_cipher_burst_avx512 @545 + submit_cipher_burst_nocheck_sse @546 + submit_cipher_burst_nocheck_sse_no_aesni @547 + submit_cipher_burst_nocheck_avx @548 + submit_cipher_burst_nocheck_avx2 @549 + submit_cipher_burst_nocheck_avx512 @550 + submit_hash_burst_sse @551 + submit_hash_burst_sse_no_aesni @552 + submit_hash_burst_avx @553 + submit_hash_burst_avx2 @554 + submit_hash_burst_avx512 @555 + submit_hash_burst_nocheck_sse @556 + submit_hash_burst_nocheck_sse_no_aesni @557 + submit_hash_burst_nocheck_avx @558 + submit_hash_burst_nocheck_avx2 @559 + submit_hash_burst_nocheck_avx512 @560 -- GitLab From 3a5eb457329b456c7c41409e181fef65f1697585 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 26 Jul 2022 21:42:04 +0000 Subject: [PATCH 283/369] lib: Update parameter checking for burst API --- lib/include/mb_mgr_code.h | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 27b6dec8..3973e8bc 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -2789,19 +2789,17 @@ submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, /* reset error status */ imb_set_errno(state, 0); -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(state, IMB_ERR_NULL_BURST); - return 0; - } - if (n_jobs > IMB_MAX_BURST_SIZE) { - imb_set_errno(state, IMB_ERR_BURST_SIZE); - return 0; - } -#endif if (run_check) { int job_offset = state->next_job; + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + if (n_jobs > IMB_MAX_BURST_SIZE) { + imb_set_errno(state, IMB_ERR_BURST_SIZE); + return 0; + } /* check enough space in queue */ if (queue_sz_remaining(state) < n_jobs) { imb_set_errno(state, IMB_ERR_QUEUE_SPACE); @@ -2809,12 +2807,10 @@ submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, } for (i = 0; i < n_jobs; i++) { -#ifdef SAFE_PARAM if (jobs[i] == NULL) { imb_set_errno(state, IMB_ERR_NULL_JOB); return 0; } -#endif if (jobs[i] != JOBS(state, job_offset)) { imb_set_errno(state, IMB_ERR_BURST_OOO); goto return_invalid_job; @@ -3140,12 +3136,11 @@ uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, /* reset error status */ imb_set_errno(state, 0); - if (run_check) { + if (run_check) if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); + imb_set_errno(state, IMB_ERR_NULL_BURST); return 0; } - } switch (cipher) { case IMB_CIPHER_CBC: -- GitLab From c1b1c1660001382366d90828e44b46d8d5d95ef2 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Fri, 29 Jul 2022 13:49:07 +0000 Subject: [PATCH 284/369] test: [ACVP] support for HMAC-SHA-1 --- test/acvp_app_main.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index ffe14dae..5b63a8dd 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -587,6 +587,72 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_hmac_sha1_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HMAC_TC *tc; + IMB_JOB *job = NULL; + uint32_t i = 0; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + uint8_t key[IMB_SHA1_BLOCK_SIZE]; + uint8_t buf[IMB_SHA1_BLOCK_SIZE]; + uint32_t key_len = 0; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hmac; + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (tc->key_len <= IMB_SHA1_BLOCK_SIZE) { + memcpy(key, tc->key, tc->key_len); + key_len = (uint32_t) tc->key_len; + } else { + IMB_SHA1(mb_mgr, tc->key, tc->key_len, key); + key_len = IMB_SHA1_DIGEST_SIZE_IN_BYTES; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA1_ONE_BLOCK(mb_mgr, buf, opad_hash); + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + /* + * The library only supports 12 or 20-byte tags and therefore, + * we are outputting 20 bytes always + */ + job->auth_tag_output_len_in_bytes = IMB_SHA1_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -710,6 +776,10 @@ int main(int argc, char **argv) &aes_cmac_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA1, + &aes_hmac_sha1_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From c1373503995bdb0ea8ae4d58fad437abdc8529f9 Mon Sep 17 00:00:00 2001 From: Kamila Lipinska Date: Tue, 2 Aug 2022 10:34:30 +0000 Subject: [PATCH 285/369] sse_t1: [SNOW3G-UEA2]: Fix not double word-aligned bytes calculation --- lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index 3f9cae16..a55a9fe9 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -261,8 +261,7 @@ mksection .text pshufd %%TMP_XMM_2, %%TMP_XMM_2, 0x88 ;; lane order: 1,0,1,0 pshufd %%TMP_XMM_3, %%TMP_XMM_3, 0x88 ;; lane order: 3,2,3,2 pblendw %%TMP_XMM_2, %%TMP_XMM_3, 0xf0 - - pandn %%TMP_XMM_2, %%TMP_XMM_0 + pandn %%TMP_XMM_2, %%TMP_XMM_0 ;; Decrease rouded dw lengths remaining for processing movdqa %%TMP_XMM_5, [state + ROUNDED_DW_LENS] @@ -278,14 +277,15 @@ mksection .text movdqa %%TMP_XMM_2, %%TMP_XMM_1 pand %%TMP_XMM_1, [state + _snow3g_args_byte_length] - pandn %%TMP_XMM_0, %%TMP_XMM_2 + pxor %%TMP_XMM_2, [rel all_fs] + pand %%TMP_XMM_0, %%TMP_XMM_2 por %%TMP_XMM_0, %%TMP_XMM_1 ;; Write outstanding bytes to _snow3g_lens dwords [0:3] and adjust ;; _snow3g_args_byte_length so after common dw length subtraction ;; it is set to 0 pand %%TMP_XMM_1, [rel last_3_bytes] - pandn %%TMP_XMM_2, [state+_snow3g_lens] + pand %%TMP_XMM_2, [state+_snow3g_lens] por %%TMP_XMM_1, %%TMP_XMM_2 movdqa [state + _snow3g_lens], %%TMP_XMM_1 -- GitLab From 99f357c756173a5af9e12fda8d9e24e7a8e7c130 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Tue, 2 Aug 2022 13:29:15 +0000 Subject: [PATCH 286/369] test: [ACVP] support for HMAC-SHA-256 --- test/acvp_app_main.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 5b63a8dd..446d963a 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -653,6 +653,72 @@ static int aes_hmac_sha1_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_hmac_sha256_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HMAC_TC *tc; + IMB_JOB *job = NULL; + uint32_t i = 0; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + uint8_t key[IMB_SHA_256_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_256_BLOCK_SIZE]; + uint32_t key_len = 0; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hmac; + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (tc->key_len <= IMB_SHA_256_BLOCK_SIZE) { + memcpy(key, tc->key, tc->key_len); + key_len = (uint32_t) tc->key_len; + } else { + IMB_SHA256(mb_mgr, tc->key, tc->key_len, key); + key_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA256_ONE_BLOCK(mb_mgr, buf, opad_hash); + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + /* + * The library only supports 16 or 32-byte tags and therefore, + * we are outputting 32 bytes always + */ + job->auth_tag_output_len_in_bytes = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -780,6 +846,10 @@ int main(int argc, char **argv) &aes_hmac_sha1_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_256, + &aes_hmac_sha256_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 59f6b4fa1b14c314b9b1db99ca9d1e15a7133ff2 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 3 Aug 2022 13:55:25 +0000 Subject: [PATCH 287/369] test: [ACVP] support for HMAC-SHA-224 --- test/acvp_app_main.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 446d963a..8d9f09d2 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -719,6 +719,72 @@ static int aes_hmac_sha256_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_hmac_sha224_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HMAC_TC *tc; + IMB_JOB *job = NULL; + uint32_t i = 0; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA224_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA224_DIGEST_SIZE_IN_BYTES], 16); + uint8_t key[IMB_SHA_256_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_256_BLOCK_SIZE]; + uint32_t key_len = 0; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hmac; + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (tc->key_len <= IMB_SHA_256_BLOCK_SIZE) { + memcpy(key, tc->key, tc->key_len); + key_len = (uint32_t) tc->key_len; + } else { + IMB_SHA224(mb_mgr, tc->key, tc->key_len, key); + key_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA224_ONE_BLOCK(mb_mgr, buf, opad_hash); + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + /* + * The library only supports 14 or 28-byte tags and therefore, + * we are outputting 28 bytes always + */ + job->auth_tag_output_len_in_bytes = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -850,6 +916,10 @@ int main(int argc, char **argv) &aes_hmac_sha256_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_224, + &aes_hmac_sha224_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 78423d0ad1a3fc8f47db6e44bd261a500ee1ec34 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 26 Jul 2022 15:56:59 +0000 Subject: [PATCH 288/369] lib: remove obsolete CLEAR_SCRATCH_SIMD_REGS macro definition --- lib/avx/mb_mgr_avx.c | 1 - lib/avx2_t1/mb_mgr_avx2.c | 1 - lib/avx512_t1/mb_mgr_avx512.c | 1 - lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 -- lib/sse_t1/mb_mgr_sse.c | 1 - 5 files changed, 6 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index ead350eb..eb7eb954 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -30,7 +30,6 @@ #include #define AVX -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_avx #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 25d6ce58..bdd26901 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -30,7 +30,6 @@ #include #define AVX2 -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_ymms #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index f981f6dc..8b6290f0 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -30,7 +30,6 @@ #include #define AVX512 -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_zmms #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index d3b8ed1c..4a2d0872 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -29,8 +29,6 @@ #include #include -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse - #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" #include "include/kasumi_internal.h" diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 1ee34f46..a87f9531 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -30,7 +30,6 @@ #include #define SSE -#define CLEAR_SCRATCH_SIMD_REGS clear_scratch_xmms_sse #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -- GitLab From e6cd22752999989dc57aa3954dee08e73c20d3ac Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 29 Jul 2022 10:52:18 +0000 Subject: [PATCH 289/369] lib: [kasumi] extract kasumi API's out from kasumi_internal.h into kasumi_interface.h --- lib/avx/mb_mgr_avx.c | 2 +- lib/avx2_t1/mb_mgr_avx2.c | 2 +- lib/avx512_t1/mb_mgr_avx512.c | 2 +- lib/include/kasumi_interface.h | 141 +++++++++++++++++++++++++++++ lib/include/kasumi_internal.h | 116 +----------------------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 +- lib/sse_t1/mb_mgr_sse.c | 2 +- 7 files changed, 147 insertions(+), 120 deletions(-) create mode 100644 lib/include/kasumi_interface.h diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index eb7eb954..8a24bedc 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -33,7 +33,7 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" +#include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" #include "include/gcm.h" diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index bdd26901..24925f5b 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -33,7 +33,7 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" +#include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" #include "include/gcm.h" diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 8b6290f0..45c85ead 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -33,7 +33,7 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" +#include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" #include "include/gcm.h" diff --git a/lib/include/kasumi_interface.h b/lib/include/kasumi_interface.h new file mode 100644 index 00000000..b8c6e869 --- /dev/null +++ b/lib/include/kasumi_interface.h @@ -0,0 +1,141 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + + +#ifndef _KASUMI_INTERFACE_H_ +#define _KASUMI_INTERFACE_H_ + +#include +#include "intel-ipsec-mb.h" + +/* Range of input data for KASUMI is from 1 to 20000 bits */ +#define KASUMI_MIN_LEN 1 +#define KASUMI_MAX_LEN 20000 + +#define BYTESIZE (8) +#define BITSIZE(x) ((int)(sizeof(x)*BYTESIZE)) + +/* SSE */ +size_t kasumi_key_sched_size_sse(void); +int kasumi_init_f8_key_sched_sse(const void *pKey, kasumi_key_sched_t *pCtx); +int kasumi_init_f9_key_sched_sse(const void *pKey, kasumi_key_sched_t *pCtx); + +void kasumi_f8_1_buffer_sse(const kasumi_key_sched_t *pCtx, const uint64_t IV, + const void *pBufferIn, void *pBufferOut, + const uint32_t cipherLengthInBytes); + +void kasumi_f8_1_buffer_bit_sse(const kasumi_key_sched_t *pCtx, + const uint64_t IV, + const void *pBufferIn, void *pBufferOut, + const uint32_t cipherLengthInBits, + const uint32_t offsetInBits); + +void kasumi_f8_2_buffer_sse(const kasumi_key_sched_t *pCtx, + const uint64_t IV1, const uint64_t IV2, + const void *pBufferIn1, void *pBufferOut1, + const uint32_t lengthInBytes1, + const void *pBufferIn2, void *pBufferOut2, + const uint32_t lengthInBytes2); + +void kasumi_f8_3_buffer_sse(const kasumi_key_sched_t *pCtx, const uint64_t IV1, + const uint64_t IV2, const uint64_t IV3, + const void *pBufferIn1, void *pBufferOut1, + const void *pBufferIn2, void *pBufferOut2, + const void *pBufferIn3, void *pBufferOut3, + const uint32_t lengthInBytes); + +void kasumi_f8_4_buffer_sse(const kasumi_key_sched_t *pCtx, + const uint64_t IV1, const uint64_t IV2, + const uint64_t IV3, const uint64_t IV4, + const void *pBufferIn1, void *pBufferOut1, + const void *pBufferIn2, void *pBufferOut2, + const void *pBufferIn3, void *pBufferOut3, + const void *pBufferIn4, void *pBufferOut4, + const uint32_t lengthInBytes); + +void kasumi_f8_n_buffer_sse(const kasumi_key_sched_t *pKeySchedule, + const uint64_t IV[], + const void * const pDataIn[], void *pDataOut[], + const uint32_t dataLen[], const uint32_t dataCount); + +void kasumi_f9_1_buffer_sse(const kasumi_key_sched_t *pCtx, + const void *pBufferIn, + const uint32_t lengthInBytes, void *pDigest); + +void kasumi_f9_1_buffer_user_sse(const kasumi_key_sched_t *pCtx, + const uint64_t IV, const void *pBufferIn, + const uint32_t lengthInBits, + void *pDigest, const uint32_t direction); + +/* AVX */ +size_t kasumi_key_sched_size_avx(void); +int kasumi_init_f8_key_sched_avx(const void *pKey, kasumi_key_sched_t *pCtx); +int kasumi_init_f9_key_sched_avx(const void *pKey, kasumi_key_sched_t *pCtx); + +void kasumi_f8_1_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV, + const void *pBufferIn, void *pBufferOut, + const uint32_t cipherLengthInBytes); +void kasumi_f8_1_buffer_bit_avx(const kasumi_key_sched_t *pCtx, + const uint64_t IV, + const void *pBufferIn, void *pBufferOut, + const uint32_t cipherLengthInBits, + const uint32_t offsetInBits); +void kasumi_f8_2_buffer_avx(const kasumi_key_sched_t *pCtx, + const uint64_t IV1, const uint64_t IV2, + const void *pBufferIn1, void *pBufferOut1, + const uint32_t lengthInBytes1, + const void *pBufferIn2, void *pBufferOut2, + const uint32_t lengthInBytes2); +void kasumi_f8_3_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV1, + const uint64_t IV2, const uint64_t IV3, + const void *pBufferIn1, void *pBufferOut1, + const void *pBufferIn2, void *pBufferOut2, + const void *pBufferIn3, void *pBufferOut3, + const uint32_t lengthInBytes); +void kasumi_f8_4_buffer_avx(const kasumi_key_sched_t *pCtx, + const uint64_t IV1, const uint64_t IV2, + const uint64_t IV3, const uint64_t IV4, + const void *pBufferIn1, void *pBufferOut1, + const void *pBufferIn2, void *pBufferOut2, + const void *pBufferIn3, void *pBufferOut3, + const void *pBufferIn4, void *pBufferOut4, + const uint32_t lengthInBytes); +void kasumi_f8_n_buffer_avx(const kasumi_key_sched_t *pKeySchedule, + const uint64_t IV[], + const void * const pDataIn[], void *pDataOut[], + const uint32_t dataLen[], const uint32_t dataCount); + +void kasumi_f9_1_buffer_avx(const kasumi_key_sched_t *pCtx, + const void *pBufferIn, + const uint32_t lengthInBytes, void *pDigest); + +void kasumi_f9_1_buffer_user_avx(const kasumi_key_sched_t *pCtx, + const uint64_t IV, const void *pBufferIn, + const uint32_t lengthInBits, + void *pDigest, const uint32_t direction); +#endif /*_KASUMI_INTERFACE_H_*/ + diff --git a/lib/include/kasumi_internal.h b/lib/include/kasumi_internal.h index bb9e76ab..a781f381 100755 --- a/lib/include/kasumi_internal.h +++ b/lib/include/kasumi_internal.h @@ -44,6 +44,7 @@ #include "include/constant_lookup.h" #include "memcpy.h" #include "error.h" +#include "kasumi_interface.h" /*--------------------------------------------------------------------- * Kasumi Inner S-Boxes @@ -152,19 +153,12 @@ static const uint16_t sso_kasumi_S9e[] = { 0x1008, 0xdaed, 0x1e0f, 0xf178, 0x69b4, 0xa1d0, 0x763b, 0x9bcd }; -/* Range of input data for KASUMI is from 1 to 20000 bits */ -#define KASUMI_MIN_LEN 1 -#define KASUMI_MAX_LEN 20000 - /* KASUMI cipher definitions */ #define NUM_KASUMI_ROUNDS (8) /* 8 rounds in the kasumi spec */ #define QWORDSIZEINBITS (64) #define QWORDSIZEINBYTES (8) #define LAST_PADDING_BIT (1) -#define BYTESIZE (8) -#define BITSIZE(x) ((int)(sizeof(x)*BYTESIZE)) - /*--------- 16 bit rotate left ------------------------------------------*/ #define ROL16(a,b) (uint16_t)((a<>(16-b))) @@ -747,25 +741,6 @@ kasumi_init_f9_key_sched(const void *const pKey, return kasumi_compute_sched(0xAA, pKey, pCtx); } -size_t -kasumi_key_sched_size_sse(void); - -int -kasumi_init_f8_key_sched_sse(const void *pKey, kasumi_key_sched_t *pCtx); - -int -kasumi_init_f9_key_sched_sse(const void *pKey, kasumi_key_sched_t *pCtx); - -size_t -kasumi_key_sched_size_avx(void); - -int -kasumi_init_f8_key_sched_avx(const void *pKey, kasumi_key_sched_t *pCtx); - -int -kasumi_init_f9_key_sched_avx(const void *pKey, kasumi_key_sched_t *pCtx); - - static inline void kasumi_f8_1_buffer(const kasumi_key_sched_t *pCtx, const uint64_t IV, const void *pIn, void *pOut, @@ -1812,94 +1787,5 @@ kasumi_f9_1_buffer_user(const kasumi_key_sched_t *pCtx, const uint64_t IV, #endif } -void kasumi_f8_1_buffer_sse(const kasumi_key_sched_t *pCtx, const uint64_t IV, - const void *pBufferIn, void *pBufferOut, - const uint32_t cipherLengthInBytes); - -void kasumi_f8_1_buffer_bit_sse(const kasumi_key_sched_t *pCtx, - const uint64_t IV, - const void *pBufferIn, void *pBufferOut, - const uint32_t cipherLengthInBits, - const uint32_t offsetInBits); - -void kasumi_f8_2_buffer_sse(const kasumi_key_sched_t *pCtx, - const uint64_t IV1, const uint64_t IV2, - const void *pBufferIn1, void *pBufferOut1, - const uint32_t lengthInBytes1, - const void *pBufferIn2, void *pBufferOut2, - const uint32_t lengthInBytes2); - -void kasumi_f8_3_buffer_sse(const kasumi_key_sched_t *pCtx, const uint64_t IV1, - const uint64_t IV2, const uint64_t IV3, - const void *pBufferIn1, void *pBufferOut1, - const void *pBufferIn2, void *pBufferOut2, - const void *pBufferIn3, void *pBufferOut3, - const uint32_t lengthInBytes); - -void kasumi_f8_4_buffer_sse(const kasumi_key_sched_t *pCtx, - const uint64_t IV1, const uint64_t IV2, - const uint64_t IV3, const uint64_t IV4, - const void *pBufferIn1, void *pBufferOut1, - const void *pBufferIn2, void *pBufferOut2, - const void *pBufferIn3, void *pBufferOut3, - const void *pBufferIn4, void *pBufferOut4, - const uint32_t lengthInBytes); - -void kasumi_f8_n_buffer_sse(const kasumi_key_sched_t *pKeySchedule, - const uint64_t IV[], - const void * const pDataIn[], void *pDataOut[], - const uint32_t dataLen[], const uint32_t dataCount); - -void kasumi_f9_1_buffer_sse(const kasumi_key_sched_t *pCtx, - const void *pBufferIn, - const uint32_t lengthInBytes, void *pDigest); - -void kasumi_f9_1_buffer_user_sse(const kasumi_key_sched_t *pCtx, - const uint64_t IV, const void *pBufferIn, - const uint32_t lengthInBits, - void *pDigest, const uint32_t direction); - - -void kasumi_f8_1_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV, - const void *pBufferIn, void *pBufferOut, - const uint32_t cipherLengthInBytes); -void kasumi_f8_1_buffer_bit_avx(const kasumi_key_sched_t *pCtx, - const uint64_t IV, - const void *pBufferIn, void *pBufferOut, - const uint32_t cipherLengthInBits, - const uint32_t offsetInBits); -void kasumi_f8_2_buffer_avx(const kasumi_key_sched_t *pCtx, - const uint64_t IV1, const uint64_t IV2, - const void *pBufferIn1, void *pBufferOut1, - const uint32_t lengthInBytes1, - const void *pBufferIn2, void *pBufferOut2, - const uint32_t lengthInBytes2); -void kasumi_f8_3_buffer_avx(const kasumi_key_sched_t *pCtx, const uint64_t IV1, - const uint64_t IV2, const uint64_t IV3, - const void *pBufferIn1, void *pBufferOut1, - const void *pBufferIn2, void *pBufferOut2, - const void *pBufferIn3, void *pBufferOut3, - const uint32_t lengthInBytes); -void kasumi_f8_4_buffer_avx(const kasumi_key_sched_t *pCtx, - const uint64_t IV1, const uint64_t IV2, - const uint64_t IV3, const uint64_t IV4, - const void *pBufferIn1, void *pBufferOut1, - const void *pBufferIn2, void *pBufferOut2, - const void *pBufferIn3, void *pBufferOut3, - const void *pBufferIn4, void *pBufferOut4, - const uint32_t lengthInBytes); -void kasumi_f8_n_buffer_avx(const kasumi_key_sched_t *pKeySchedule, - const uint64_t IV[], - const void * const pDataIn[], void *pDataOut[], - const uint32_t dataLen[], const uint32_t dataCount); - -void kasumi_f9_1_buffer_avx(const kasumi_key_sched_t *pCtx, - const void *pBufferIn, - const uint32_t lengthInBytes, void *pDigest); - -void kasumi_f9_1_buffer_user_avx(const kasumi_key_sched_t *pCtx, - const uint64_t IV, const void *pBufferIn, - const uint32_t lengthInBits, - void *pDigest, const uint32_t direction); #endif /*_KASUMI_INTERNAL_H_*/ diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 4a2d0872..21f1697e 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -31,7 +31,7 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" +#include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" #include "include/chacha20_poly1305.h" diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index a87f9531..4d507e5c 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -33,7 +33,7 @@ #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_internal.h" +#include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" #include "include/gcm.h" -- GitLab From 79a149ea1e5de60a9366c87bb6d900b305ce9d36 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 29 Jul 2022 12:16:59 +0000 Subject: [PATCH 290/369] avx512: [MB_MGR] split type-1 and type-2 manager implementations - get rid of the stubs and multiple checks for available CPU features - produce different code paths type-1 and type-2 - create avx512 JOB API interface that hides type-1 and type-2 differences behind - this is only needed for cases where legacy apps use avx512 JOB API directly --- lib/Makefile | 2 + lib/avx512_t1/mb_mgr_avx512.c | 1355 +----------------------------- lib/avx512_t1/mb_mgr_avx512_t1.c | 687 +++++++++++++++ lib/avx512_t2/mb_mgr_avx512_t2.c | 723 ++++++++++++++++ lib/include/arch_avx512_type2.h | 3 +- lib/include/arch_avx_type1.h | 2 + lib/include/ipsec_ooo_mgr.h | 92 +- lib/intel-ipsec-mb.h | 3 + lib/libIPSec_MB.def | 92 +- lib/win_x64.mak | 2 + 10 files changed, 1583 insertions(+), 1378 deletions(-) create mode 100644 lib/avx512_t1/mb_mgr_avx512_t1.c create mode 100644 lib/avx512_t2/mb_mgr_avx512_t2.c diff --git a/lib/Makefile b/lib/Makefile index ccfc3fd4..95264b18 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -249,6 +249,8 @@ c_lib_objs := \ mb_mgr_avx.o \ mb_mgr_avx2.o \ mb_mgr_avx512.o \ + mb_mgr_avx512_t1.o \ + mb_mgr_avx512_t2.o \ mb_mgr_sse.o \ alloc.o \ aes_xcbc_expand_key.o \ diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 45c85ead..3fb8783b 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -25,1357 +25,72 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ -#include -#include -#include - -#define AVX512 - #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_interface.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" -#include "include/snow3g_submit.h" - -#include "include/save_xmms.h" -#include "include/des.h" -#include "include/gcm.h" +#include "include/error.h" #include "include/cpu_feature.h" -#include "include/noaesni.h" #include "include/aesni_emu.h" -#include "include/error.h" - -#include "include/arch_avx_type1.h" /* AESNI */ -#include "include/arch_avx2_type1.h" /* MD5 */ -#include "include/arch_avx512_type1.h" -#include "include/arch_avx512_type2.h" - -#include "include/ooo_mgr_reset.h" - -#define SAVE_XMMS save_xmms_avx -#define RESTORE_XMMS restore_xmms_avx - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx512 -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx512 -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx512 - -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx512 -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx512 -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx512 - -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx512 -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx512 -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx512 - -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx512 -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx512 -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx512 -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx512 -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx512 -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx512 - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx512 -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx512 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx512 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx512 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx512 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx512 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx512 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx512 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx512 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx512 - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx512 -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx512 -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx512 - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx512 -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx512 - -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx512 -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx512 -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx512 -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx512 -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx512 -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx512 - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx512 -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx512 -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx512 -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx512 - -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx512 -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx512 - -#define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 -#define FLUSH_JOB_SHA1 flush_job_sha1_avx512 -#define SUBMIT_JOB_SHA224 submit_job_sha224_avx512 -#define FLUSH_JOB_SHA224 flush_job_sha224_avx512 -#define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 -#define FLUSH_JOB_SHA256 flush_job_sha256_avx512 -#define SUBMIT_JOB_SHA384 submit_job_sha384_avx512 -#define FLUSH_JOB_SHA384 flush_job_sha384_avx512 -#define SUBMIT_JOB_SHA512 submit_job_sha512_avx512 -#define FLUSH_JOB_SHA512 flush_job_sha512_avx512 - -#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 -#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 - -#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 -#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 - -#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 -#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 - -#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 -#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 - -#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 -#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 - -#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 -#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx512 -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx512 -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx512 -#define POLY1305_MAC poly1305_mac_avx512 - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - -static IMB_JOB *submit_snow3g_uea2_job_vaes_avx512(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - if ((job->msg_len_to_cipher_in_bits & 7) || - (job->cipher_start_offset_in_bits & 7)) - return def_submit_snow3g_uea2_job(state, job); - - return submit_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo, job); -} - -static IMB_JOB *flush_snow3g_uea2_job_vaes_avx512(IMB_MGR *state) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - return flush_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo); -} - -static IMB_JOB *submit_snow3g_uea2_job_avx512(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - if ((job->msg_len_to_cipher_in_bits & 7) || - (job->cipher_start_offset_in_bits & 7)) - return def_submit_snow3g_uea2_job(state, job); - - return submit_job_snow3g_uea2_avx512(snow3g_uea2_ooo, job); -} - -static IMB_JOB *flush_snow3g_uea2_job_avx512(IMB_MGR *state) -{ - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - return flush_job_snow3g_uea2_avx512(snow3g_uea2_ooo); -} - -static IMB_JOB *(*submit_job_snow3g_uea2_avx512_ptr) - (IMB_MGR *state, IMB_JOB *job) = - submit_snow3g_uea2_job_avx512; - -static IMB_JOB *(*flush_job_snow3g_uea2_avx512_ptr)(IMB_MGR *state) = - flush_snow3g_uea2_job_avx512; - -#define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_avx512_ptr -#define FLUSH_JOB_SNOW3G_UEA2 flush_job_snow3g_uea2_avx512_ptr - -static IMB_JOB *(*submit_job_snow3g_uia2_avx512_ptr) - (MB_MGR_SNOW3G_OOO *state, IMB_JOB *job) = - submit_job_snow3g_uia2_avx512; - -static IMB_JOB *(*flush_job_snow3g_uia2_avx512_ptr) - (MB_MGR_SNOW3G_OOO *state) = flush_job_snow3g_uia2_avx512; - -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx512_ptr -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx512_ptr - - -/* ====================================================================== */ - -static void (*poly1305_mac_avx512) - (IMB_JOB *) = poly1305_mac_plain_avx512; - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, - const uint64_t key_size); -__forceinline -IMB_JOB * -FLUSH_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, - const uint64_t key_size); - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, - const uint64_t key_size); - -#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 -#define FLUSH_JOB_HMAC flush_job_hmac_avx512 -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 - -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 - -#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 -#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 -#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 -#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 -#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 -#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx512 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx512 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx512 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx512 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx512 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx512 - -#define AES_GCM_DEC_IV_128_VAES aes_gcm_dec_var_iv_128_vaes_avx512 -#define AES_GCM_ENC_IV_128_VAES aes_gcm_enc_var_iv_128_vaes_avx512 -#define AES_GCM_DEC_IV_192_VAES aes_gcm_dec_var_iv_192_vaes_avx512 -#define AES_GCM_ENC_IV_192_VAES aes_gcm_enc_var_iv_192_vaes_avx512 -#define AES_GCM_DEC_IV_256_VAES aes_gcm_dec_var_iv_256_vaes_avx512 -#define AES_GCM_ENC_IV_256_VAES aes_gcm_enc_var_iv_256_vaes_avx512 - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx512 -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx512 - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx512 -#define FLUSH_JOB flush_job_avx512 -#define QUEUE_SIZE queue_size_avx512 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512 -#define GET_NEXT_JOB get_next_job_avx512 -#define GET_COMPLETED_JOB get_completed_job_avx512 -#define GET_NEXT_BURST get_next_burst_avx512 -#define SUBMIT_BURST submit_burst_avx512 -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512 -#define FLUSH_BURST flush_burst_avx512 -#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512 -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512 -#define SUBMIT_HASH_BURST submit_hash_burst_avx512 -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512 - -/* ====================================================================== */ - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 - -/* ====================================================================== */ -#define AES_CFB_128_ONE aes_cfb_128_one_avx512 -#define AES_CFB_256_ONE aes_cfb_256_one_avx512 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx512 -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx512 - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx512 -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx512 - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx512 -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx512 - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx512 -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx512 - -/* ====================================================================== */ - -extern uint32_t -ethernet_fcs_avx512_local(const void *msg, const uint64_t len, - const void *tag_ouput); -extern uint32_t -ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_avx_local - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx512 -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx512 -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx512 -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx512 - -/* ====================================================================== */ - -/* - * GCM submit / flush API for AVX512 arch - */ -static IMB_JOB * -plain_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) +IMB_DLL_LOCAL void +init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) { - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; } +#endif - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -plain_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); + if (!(state->features & IMB_FEATURE_AESNI)) { + fallback_no_aesni(state, 1); + return; } - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -vaes_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_DEC_IV_128_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_DEC_IV_192_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_DEC_IV_256_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -vaes_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_ENC_IV_128_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_ENC_IV_192_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_ENC_IV_256_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB *(*submit_job_aes_gcm_enc_avx512) - (IMB_MGR *state, IMB_JOB *job) = plain_submit_gcm_enc_avx512; - -static IMB_JOB *(*submit_job_aes_gcm_dec_avx512) - (IMB_MGR *state, IMB_JOB *job) = plain_submit_gcm_dec_avx512; - -static IMB_JOB *(*submit_job_aes_cntr_avx512) - (IMB_JOB *job) = submit_job_aes_cntr_avx; -static IMB_JOB *(*submit_job_aes_cntr_bit_avx512) - (IMB_JOB *job) = submit_job_aes_cntr_bit_avx; - -static IMB_JOB *(*submit_job_pon_enc_avx512) - (IMB_JOB *job) = submit_job_pon_enc_avx; -static IMB_JOB *(*submit_job_pon_dec_avx512) - (IMB_JOB *job) = submit_job_pon_dec_avx; -static IMB_JOB *(*submit_job_pon_enc_no_ctr_avx512) - (IMB_JOB *job) = submit_job_pon_enc_no_ctr_avx; -static IMB_JOB *(*submit_job_pon_dec_no_ctr_avx512) - (IMB_JOB *job) = submit_job_pon_dec_no_ctr_avx; - -static IMB_JOB * -vaes_submit_cntr_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_256_submit_vaes_avx512(job); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} + /* reset error status */ + imb_set_errno(state, 0); -static IMB_JOB * -vaes_submit_cntr_bit_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_bit_256_submit_vaes_avx512(job); + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; + if ((state->features & IMB_CPUFLAGS_AVX512_T2) == + IMB_CPUFLAGS_AVX512_T2) + init_mb_mgr_avx512_t2_internal(state, reset_mgrs); + else + init_mb_mgr_avx512_t1_internal(state, reset_mgrs); } -/* ====================================================================== */ - -static IMB_JOB * -(*submit_job_aes128_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes128_enc_avx; - -static IMB_JOB * -(*submit_job_aes192_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes192_enc_avx; - -static IMB_JOB * -(*submit_job_aes256_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = submit_job_aes256_enc_avx; - -static IMB_JOB * -(*flush_job_aes128_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes128_enc_avx; - -static IMB_JOB * -(*flush_job_aes192_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes192_enc_avx; - -static IMB_JOB * -(*flush_job_aes256_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes256_enc_avx; - -static void -(*aes_cbc_dec_128_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_128_avx; -static void -(*aes_cbc_dec_192_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_192_avx; -static void -(*aes_cbc_dec_256_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = aes_cbc_dec_256_avx; - -static IMB_JOB * -(*submit_job_aes128_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state, - IMB_JOB *job) = submit_job_aes128_cmac_auth_avx; - -static IMB_JOB * -(*flush_job_aes128_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state) = flush_job_aes128_cmac_auth_avx; - -static IMB_JOB * -(*submit_job_aes256_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state, - IMB_JOB *job) = submit_job_aes256_cmac_auth_avx; - -static IMB_JOB * -(*flush_job_aes256_cmac_auth_avx512) - (MB_MGR_CMAC_OOO *state) = flush_job_aes256_cmac_auth_avx; - -static IMB_JOB * -(*submit_job_aes128_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state, - IMB_JOB *job) = submit_job_aes128_ccm_auth_avx; - -static IMB_JOB * -(*flush_job_aes128_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state) = flush_job_aes128_ccm_auth_avx; - -static IMB_JOB * -(*submit_job_aes256_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state, - IMB_JOB *job) = submit_job_aes256_ccm_auth_avx; - -static IMB_JOB * -(*flush_job_aes256_ccm_auth_avx512) - (MB_MGR_CCM_OOO *state) = flush_job_aes256_ccm_auth_avx; - -static IMB_JOB * -(*aes_cntr_ccm_128_avx512) (IMB_JOB *job) = aes_cntr_ccm_128_avx; - -static IMB_JOB * -(*aes_cntr_ccm_256_avx512) (IMB_JOB *job) = aes_cntr_ccm_256_avx; - -static IMB_JOB * -(*submit_job_zuc_eea3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eea3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc_eea3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eea3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc256_eea3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc256_eea3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc256_eea3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc256_eea3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc_eia3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eia3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc_eia3_avx512) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eia3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_zuc256_eia3_avx512) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz) = - submit_job_zuc256_eia3_no_gfni_avx512; - -static IMB_JOB * -(*flush_job_zuc256_eia3_avx512) - (MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) = - flush_job_zuc256_eia3_no_gfni_avx512; - -static IMB_JOB * -(*submit_job_aes_xcbc_avx512) - (MB_MGR_AES_XCBC_OOO *state, - IMB_JOB *job) = submit_job_aes_xcbc_avx; - -static IMB_JOB * -(*flush_job_aes_xcbc_avx512) - (MB_MGR_AES_XCBC_OOO *state) = flush_job_aes_xcbc_avx; - - -static IMB_JOB * -(*submit_job_aes128_cbcs_1_9_enc_avx512) - (MB_MGR_AES_OOO *state, IMB_JOB *job) = - submit_job_aes128_cbcs_1_9_enc_avx; - -static IMB_JOB * -(*flush_job_aes128_cbcs_1_9_enc_avx512) - (MB_MGR_AES_OOO *state) = flush_job_aes128_cbcs_1_9_enc_avx; - -static void -(*aes_cbcs_1_9_dec_128_avx512) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes, - void *next_iv) = aes_cbcs_1_9_dec_128_avx; - -static void -(*aes_ecb_enc_128_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_enc_128_avx; - -static void -(*aes_ecb_enc_192_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_enc_192_avx; - -static void -(*aes_ecb_enc_256_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_enc_256_avx; - -static void -(*aes_ecb_dec_128_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_dec_128_avx; - -static void -(*aes_ecb_dec_192_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_dec_192_avx; - -static void -(*aes_ecb_dec_256_avx512) (const void *in, const void *keys, - void *out, const uint64_t len_bytes) = - aes_ecb_dec_256_avx; - -/* ====================================================================== */ - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS128_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); - -__forceinline -IMB_JOB * -SUBMIT_JOB_DOCSIS256_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); - -static IMB_JOB * -submit_aes_docsis128_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) +void +init_mb_mgr_avx512(IMB_MGR *state) { - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis128_dec_crc32_avx512(job); - - return job; + init_mb_mgr_avx512_internal(state, 1); } -static IMB_JOB * -submit_aes_docsis256_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) +IMB_JOB *submit_job_avx512(IMB_MGR *state) { - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis256_dec_crc32_avx512(job); - - return job; + return IMB_SUBMIT_JOB(state); } -static IMB_JOB * -submit_job_docsis128_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) +IMB_JOB *flush_job_avx512(IMB_MGR *state) { - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis128_dec_crc32_vaes_avx512(job); - - return job; + return IMB_FLUSH_JOB(state); } -static IMB_JOB * -submit_job_docsis256_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, - IMB_JOB *job) +uint32_t queue_size_avx512(IMB_MGR *state) { - (void) state; - - if (job->msg_len_to_hash_in_bytes == 0) { - if (job->msg_len_to_cipher_in_bytes == 0) { - /* NO cipher, NO CRC32 */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } - - /* Cipher, NO CRC32 */ - return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); - } - - /* Cipher + CRC32 // CRC32 */ - aes_docsis256_dec_crc32_vaes_avx512(job); - - return job; + return IMB_QUEUE_SIZE(state); } -static IMB_JOB * -(*submit_job_docsis128_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_job_aes_docsis128_enc_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis256_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_job_aes_docsis256_enc_crc32_avx512; - -static IMB_JOB * -(*flush_job_docsis128_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state) = - flush_job_aes_docsis128_enc_crc32_avx512; - -static IMB_JOB * -(*flush_job_docsis256_sec_crc_enc_fn) - (MB_MGR_DOCSIS_AES_OOO *state) = - flush_job_aes_docsis256_enc_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis128_sec_crc_dec_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_aes_docsis128_dec_crc32_avx512; - -static IMB_JOB * -(*submit_job_docsis256_sec_crc_dec_fn) - (MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job) = - submit_aes_docsis256_dec_crc32_avx512; - -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_docsis128_sec_crc_enc_fn -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_docsis256_sec_crc_enc_fn -#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_docsis128_sec_crc_enc_fn -#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_docsis256_sec_crc_enc_fn -#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_job_docsis128_sec_crc_dec_fn -#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_job_docsis256_sec_crc_dec_fn - - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) +IMB_JOB *submit_job_nocheck_avx512(IMB_MGR *state) { - /* Init AES out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - ooo_mgr_aes_reset(state->aes128_ooo, 16); - ooo_mgr_aes_reset(state->aes192_ooo, 16); - ooo_mgr_aes_reset(state->aes256_ooo, 16); - } else { - /* init 8 lanes */ - ooo_mgr_aes_reset(state->aes128_ooo, 8); - ooo_mgr_aes_reset(state->aes192_ooo, 8); - ooo_mgr_aes_reset(state->aes256_ooo, 8); - } - - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) - * uses same settings as AES CBC. - */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 16); - ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 16); - ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 16); - ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 16); - } else { - /* init 8 lanes */ - ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); - } - - /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ - ooo_mgr_des_reset(state->des_enc_ooo, AVX512_NUM_DES_LANES); - ooo_mgr_des_reset(state->des_dec_ooo, AVX512_NUM_DES_LANES); - ooo_mgr_des_reset(state->des3_enc_ooo, AVX512_NUM_DES_LANES); - ooo_mgr_des_reset(state->des3_dec_ooo, AVX512_NUM_DES_LANES); - ooo_mgr_des_reset(state->docsis_des_enc_ooo, AVX512_NUM_DES_LANES); - ooo_mgr_des_reset(state->docsis_des_dec_ooo, AVX512_NUM_DES_LANES); - - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 16); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 16); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 16); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 16); - - /* Init HMAC/SHA1 out-of-order fields */ - ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); - - /* Init HMAC/SHA224 out-of-order fields */ - ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, - AVX512_NUM_SHA256_LANES); - - /* Init HMAC/SHA256 out-of-order fields */ - ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, - AVX512_NUM_SHA256_LANES); - - /* Init HMAC/SHA384 out-of-order fields */ - ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, - AVX512_NUM_SHA512_LANES); - - /* Init HMAC/SHA512 out-of-order fields */ - ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, - AVX512_NUM_SHA512_LANES); - - /* Init HMAC/MD5 out-of-order fields */ - ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); - - /* Init AES/XCBC OOO fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) - ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 16); - else - ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); - - /* Init AES-CCM auth out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - ooo_mgr_ccm_reset(state->aes_ccm_ooo, 16); - ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 16); - } else { - /* init 8 lanes */ - ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); - ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); - } - - /* Init AES-CMAC auth out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - /* init 16 lanes */ - ooo_mgr_cmac_reset(state->aes_cmac_ooo, 16); - ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 16); - } else { - /* init 8 lanes */ - ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); - ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); - } - - /* Init AES CBC-S out-of-order fields */ - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) - ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 12); - else - ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); - - /* Init SNOW3G out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 16); - ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 16); - - /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, AVX512_NUM_SHA1_LANES); - - /* Init SHA224 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, AVX512_NUM_SHA256_LANES); - - /* Init SHA256 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); - - /* Init SHA384 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_384_ooo, AVX512_NUM_SHA512_LANES); - - /* Init SHA512 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_512_ooo, AVX512_NUM_SHA512_LANES); + return IMB_SUBMIT_JOB_NOCHECK(state); } -IMB_DLL_LOCAL void -init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) +IMB_JOB *get_next_job_avx512(IMB_MGR *state) { -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Check if CPU flags needed for AVX512 interface are present */ - if ((state->features & IMB_CPUFLAGS_AVX512) != IMB_CPUFLAGS_AVX512) { - imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AVX512; - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - aes_cbc_dec_128_avx512 = aes_cbc_dec_128_vaes_avx512; - aes_cbc_dec_192_avx512 = aes_cbc_dec_192_vaes_avx512; - aes_cbc_dec_256_avx512 = aes_cbc_dec_256_vaes_avx512; - aes_ecb_enc_128_avx512 = aes_ecb_enc_128_vaes_avx512; - aes_ecb_enc_192_avx512 = aes_ecb_enc_192_vaes_avx512; - aes_ecb_enc_256_avx512 = aes_ecb_enc_256_vaes_avx512; - aes_ecb_dec_128_avx512 = aes_ecb_dec_128_vaes_avx512; - aes_ecb_dec_192_avx512 = aes_ecb_dec_192_vaes_avx512; - aes_ecb_dec_256_avx512 = aes_ecb_dec_256_vaes_avx512; - submit_job_aes128_enc_avx512 = - submit_job_aes128_enc_vaes_avx512; - flush_job_aes128_enc_avx512 = - flush_job_aes128_enc_vaes_avx512; - submit_job_aes192_enc_avx512 = - submit_job_aes192_enc_vaes_avx512; - flush_job_aes192_enc_avx512 = - flush_job_aes192_enc_vaes_avx512; - submit_job_aes256_enc_avx512 = - submit_job_aes256_enc_vaes_avx512; - flush_job_aes256_enc_avx512 = - flush_job_aes256_enc_vaes_avx512; - submit_job_aes128_cmac_auth_avx512 = - submit_job_aes128_cmac_auth_vaes_avx512; - flush_job_aes128_cmac_auth_avx512 = - flush_job_aes128_cmac_auth_vaes_avx512; - submit_job_aes256_cmac_auth_avx512 = - submit_job_aes256_cmac_auth_vaes_avx512; - flush_job_aes256_cmac_auth_avx512 = - flush_job_aes256_cmac_auth_vaes_avx512; - submit_job_aes128_ccm_auth_avx512 = - submit_job_aes128_ccm_auth_vaes_avx512; - flush_job_aes128_ccm_auth_avx512 = - flush_job_aes128_ccm_auth_vaes_avx512; - submit_job_aes256_ccm_auth_avx512 = - submit_job_aes256_ccm_auth_vaes_avx512; - flush_job_aes256_ccm_auth_avx512 = - flush_job_aes256_ccm_auth_vaes_avx512; - aes_cntr_ccm_128_avx512 = aes_cntr_ccm_128_vaes_avx512; - aes_cntr_ccm_256_avx512 = aes_cntr_ccm_256_vaes_avx512; - - submit_job_docsis128_sec_crc_enc_fn = - submit_job_aes_docsis128_enc_crc32_vaes_avx512; - submit_job_docsis256_sec_crc_enc_fn = - submit_job_aes_docsis256_enc_crc32_vaes_avx512; - flush_job_docsis128_sec_crc_enc_fn = - flush_job_aes_docsis128_enc_crc32_vaes_avx512; - flush_job_docsis256_sec_crc_enc_fn = - flush_job_aes_docsis256_enc_crc32_vaes_avx512; - - submit_job_docsis128_sec_crc_dec_fn = - submit_job_docsis128_sec_crc_dec_vaes_avx512; - submit_job_docsis256_sec_crc_dec_fn = - submit_job_docsis256_sec_crc_dec_vaes_avx512; - - submit_job_aes_xcbc_avx512 = submit_job_aes_xcbc_vaes_avx512; - flush_job_aes_xcbc_avx512 = flush_job_aes_xcbc_vaes_avx512; - - submit_job_aes128_cbcs_1_9_enc_avx512 = - submit_job_aes128_cbcs_1_9_enc_vaes_avx512; - flush_job_aes128_cbcs_1_9_enc_avx512 = - flush_job_aes128_cbcs_1_9_enc_vaes_avx512; - aes_cbcs_1_9_dec_128_avx512 = aes_cbcs_1_9_dec_128_vaes_avx512; - - submit_job_snow3g_uea2_avx512_ptr = - submit_snow3g_uea2_job_vaes_avx512; - flush_job_snow3g_uea2_avx512_ptr = - flush_snow3g_uea2_job_vaes_avx512; - } - - if ((state->features & IMB_FEATURE_GFNI) && - (state->features & IMB_FEATURE_VAES)) { - submit_job_zuc_eea3_avx512 = submit_job_zuc_eea3_gfni_avx512; - flush_job_zuc_eea3_avx512 = flush_job_zuc_eea3_gfni_avx512; - submit_job_zuc_eia3_avx512 = submit_job_zuc_eia3_gfni_avx512; - flush_job_zuc_eia3_avx512 = flush_job_zuc_eia3_gfni_avx512; - submit_job_zuc256_eea3_avx512 = - submit_job_zuc256_eea3_gfni_avx512; - flush_job_zuc256_eea3_avx512 = - flush_job_zuc256_eea3_gfni_avx512; - submit_job_zuc256_eia3_avx512 = - submit_job_zuc256_eia3_gfni_avx512; - flush_job_zuc256_eia3_avx512 = - flush_job_zuc256_eia3_gfni_avx512; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set handlers */ - state->get_next_job = get_next_job_avx512; - state->submit_job = submit_job_avx512; - state->get_next_burst = get_next_burst_avx512; - state->submit_burst = submit_burst_avx512; - state->submit_burst_nocheck= submit_burst_nocheck_avx512; - state->flush_burst = flush_burst_avx512; - state->submit_cipher_burst = submit_cipher_burst_avx512; - state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx512; - state->submit_hash_burst = submit_hash_burst_avx512; - state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx512; - state->submit_job_nocheck = submit_job_nocheck_avx512; - state->get_completed_job = get_completed_job_avx512; - state->flush_job = flush_job_avx512; - state->queue_size = queue_size_avx512; - state->keyexp_128 = aes_keyexp_128_avx512; - state->keyexp_192 = aes_keyexp_192_avx512; - state->keyexp_256 = aes_keyexp_256_avx512; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx512; - state->xcbc_keyexp = aes_xcbc_expand_key_avx512; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx512; - state->sha1 = sha1_avx512; - state->sha224_one_block = sha224_one_block_avx512; - state->sha224 = sha224_avx512; - state->sha256_one_block = sha256_one_block_avx512; - state->sha256 = sha256_avx512; - state->sha384_one_block = sha384_one_block_avx512; - state->sha384 = sha384_avx512; - state->sha512_one_block = sha512_one_block_avx512; - state->sha512 = sha512_avx512; - state->md5_one_block = md5_one_block_avx512; - state->aes128_cfb_one = aes_cfb_128_one_avx512; - - state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; - state->eea3_4_buffer = zuc_eea3_4_buffer_avx; - state->eia3_1_buffer = zuc_eia3_1_buffer_avx512; - - if ((state->features & IMB_FEATURE_GFNI) && - (state->features & IMB_FEATURE_VAES)) { - state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_avx512; - state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_avx512; - } else { - state->eea3_n_buffer = zuc_eea3_n_buffer_avx512; - state->eia3_n_buffer = zuc_eia3_n_buffer_avx512; - } - - state->f8_1_buffer = kasumi_f8_1_buffer_avx; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; - state->f8_2_buffer = kasumi_f8_2_buffer_avx; - state->f8_3_buffer = kasumi_f8_3_buffer_avx; - state->f8_4_buffer = kasumi_f8_4_buffer_avx; - state->f8_n_buffer = kasumi_f8_n_buffer_avx; - state->f9_1_buffer = kasumi_f9_1_buffer_avx; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; - state->kasumi_key_sched_size = kasumi_key_sched_size_avx; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx512; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx512; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx512; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx512; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx512; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx512; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx512; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx512; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx512; - state->snow3g_init_key_sched = snow3g_init_key_sched_avx512; - state->snow3g_key_sched_size = snow3g_key_sched_size_avx512; - - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; - - if ((state->features & IMB_FEATURE_VPCLMULQDQ) == - IMB_FEATURE_VPCLMULQDQ) { - state->crc32_ethernet_fcs = ethernet_fcs_avx512; - state->crc16_x25 = crc16_x25_avx512; - state->crc32_sctp = crc32_sctp_avx512; - state->crc24_lte_a = crc24_lte_a_avx512; - state->crc24_lte_b = crc24_lte_b_avx512; - state->crc16_fp_data = crc16_fp_data_avx512; - state->crc11_fp_header = crc11_fp_header_avx512; - state->crc7_fp_header = crc7_fp_header_avx512; - state->crc10_iuup_data = crc10_iuup_data_avx512; - state->crc6_iuup_header = crc6_iuup_header_avx512; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx512; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx512; -#ifndef _WIN32 - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_vaes_avx512; -#endif - } - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - submit_job_aes_cntr_avx512 = vaes_submit_cntr_avx512; - submit_job_aes_cntr_bit_avx512 = vaes_submit_cntr_bit_avx512; - submit_job_pon_enc_avx512 = submit_job_pon_enc_vaes_avx512; - submit_job_pon_enc_no_ctr_avx512 = - submit_job_pon_enc_no_ctr_vaes_avx512; - submit_job_pon_dec_avx512 = submit_job_pon_dec_vaes_avx512; - submit_job_pon_dec_no_ctr_avx512 = - submit_job_pon_dec_no_ctr_vaes_avx512; - submit_job_snow3g_uea2_avx512_ptr = - submit_snow3g_uea2_job_vaes_avx512; - flush_job_snow3g_uea2_avx512_ptr = - flush_snow3g_uea2_job_vaes_avx512; - } - - if (state->features & IMB_FEATURE_AVX512_IFMA) { - poly1305_mac_avx512 = poly1305_mac_fma_avx512; - state->chacha20_poly1305_init = - init_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_enc_update = - update_enc_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_dec_update = - update_dec_chacha20_poly1305_fma_avx512; - state->chacha20_poly1305_finalize = - finalize_chacha20_poly1305_fma_avx512; - } else { - state->chacha20_poly1305_init = init_chacha20_poly1305_avx512; - state->chacha20_poly1305_enc_update = - update_enc_chacha20_poly1305_avx512; - state->chacha20_poly1305_dec_update = - update_dec_chacha20_poly1305_avx512; - state->chacha20_poly1305_finalize = - finalize_chacha20_poly1305_avx512; - } - - if ((state->features & (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) == - (IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ)) { - state->gcm128_enc = aes_gcm_enc_128_vaes_avx512; - state->gcm192_enc = aes_gcm_enc_192_vaes_avx512; - state->gcm256_enc = aes_gcm_enc_256_vaes_avx512; - state->gcm128_dec = aes_gcm_dec_128_vaes_avx512; - state->gcm192_dec = aes_gcm_dec_192_vaes_avx512; - state->gcm256_dec = aes_gcm_dec_256_vaes_avx512; - state->gcm128_init = aes_gcm_init_128_vaes_avx512; - state->gcm192_init = aes_gcm_init_192_vaes_avx512; - state->gcm256_init = aes_gcm_init_256_vaes_avx512; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_vaes_avx512; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_vaes_avx512; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_vaes_avx512; - state->gcm128_enc_update = aes_gcm_enc_128_update_vaes_avx512; - state->gcm192_enc_update = aes_gcm_enc_192_update_vaes_avx512; - state->gcm256_enc_update = aes_gcm_enc_256_update_vaes_avx512; - state->gcm128_dec_update = aes_gcm_dec_128_update_vaes_avx512; - state->gcm192_dec_update = aes_gcm_dec_192_update_vaes_avx512; - state->gcm256_dec_update = aes_gcm_dec_256_update_vaes_avx512; - state->gcm128_enc_finalize = - aes_gcm_enc_128_finalize_vaes_avx512; - state->gcm192_enc_finalize = - aes_gcm_enc_192_finalize_vaes_avx512; - state->gcm256_enc_finalize = - aes_gcm_enc_256_finalize_vaes_avx512; - state->gcm128_dec_finalize = - aes_gcm_dec_128_finalize_vaes_avx512; - state->gcm192_dec_finalize = - aes_gcm_dec_192_finalize_vaes_avx512; - state->gcm256_dec_finalize = - aes_gcm_dec_256_finalize_vaes_avx512; - state->gcm128_precomp = aes_gcm_precomp_128_vaes_avx512; - state->gcm192_precomp = aes_gcm_precomp_192_vaes_avx512; - state->gcm256_precomp = aes_gcm_precomp_256_vaes_avx512; - state->gcm128_pre = aes_gcm_pre_128_vaes_avx512; - state->gcm192_pre = aes_gcm_pre_192_vaes_avx512; - state->gcm256_pre = aes_gcm_pre_256_vaes_avx512; - state->ghash = ghash_vaes_avx512; - state->ghash_pre = ghash_pre_vaes_avx512; - - submit_job_aes_gcm_enc_avx512 = vaes_submit_gcm_enc_avx512; - submit_job_aes_gcm_dec_avx512 = vaes_submit_gcm_dec_avx512; - - state->gmac128_init = imb_aes_gmac_init_128_vaes_avx512; - state->gmac192_init = imb_aes_gmac_init_192_vaes_avx512; - state->gmac256_init = imb_aes_gmac_init_256_vaes_avx512; - state->gmac128_update = imb_aes_gmac_update_128_vaes_avx512; - state->gmac192_update = imb_aes_gmac_update_192_vaes_avx512; - state->gmac256_update = imb_aes_gmac_update_256_vaes_avx512; - state->gmac128_finalize = imb_aes_gmac_finalize_128_vaes_avx512; - state->gmac192_finalize = imb_aes_gmac_finalize_192_vaes_avx512; - state->gmac256_finalize = imb_aes_gmac_finalize_256_vaes_avx512; - - submit_job_snow3g_uia2_avx512_ptr = - submit_job_snow3g_uia2_vaes_avx512; - flush_job_snow3g_uia2_avx512_ptr = - flush_job_snow3g_uia2_vaes_avx512; - } else { - state->gcm128_enc = aes_gcm_enc_128_avx512; - state->gcm192_enc = aes_gcm_enc_192_avx512; - state->gcm256_enc = aes_gcm_enc_256_avx512; - state->gcm128_dec = aes_gcm_dec_128_avx512; - state->gcm192_dec = aes_gcm_dec_192_avx512; - state->gcm256_dec = aes_gcm_dec_256_avx512; - state->gcm128_init = aes_gcm_init_128_avx512; - state->gcm192_init = aes_gcm_init_192_avx512; - state->gcm256_init = aes_gcm_init_256_avx512; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx512; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx512; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx512; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx512; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx512; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx512; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx512; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx512; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx512; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx512; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx512; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx512; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx512; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx512; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx512; - state->gcm128_precomp = aes_gcm_precomp_128_avx512; - state->gcm192_precomp = aes_gcm_precomp_192_avx512; - state->gcm256_precomp = aes_gcm_precomp_256_avx512; - state->gcm128_pre = aes_gcm_pre_128_avx512; - state->gcm192_pre = aes_gcm_pre_192_avx512; - state->gcm256_pre = aes_gcm_pre_256_avx512; - state->ghash = ghash_avx512; - state->ghash_pre = ghash_pre_avx_gen2; - - state->gmac128_init = imb_aes_gmac_init_128_avx512; - state->gmac192_init = imb_aes_gmac_init_192_avx512; - state->gmac256_init = imb_aes_gmac_init_256_avx512; - state->gmac128_update = imb_aes_gmac_update_128_avx512; - state->gmac192_update = imb_aes_gmac_update_192_avx512; - state->gmac256_update = imb_aes_gmac_update_256_avx512; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx512; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx512; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx512; - } - + return IMB_GET_NEXT_JOB(state); } -void -init_mb_mgr_avx512(IMB_MGR *state) +IMB_JOB *get_completed_job_avx512(IMB_MGR *state) { - init_mb_mgr_avx512_internal(state, 1); + return IMB_GET_COMPLETED_JOB(state); } - -#include "mb_mgr_code.h" diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c new file mode 100644 index 00000000..90a7c391 --- /dev/null +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -0,0 +1,687 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX512 + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/gcm.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_avx_type1.h" /* AESNI */ +#include "include/arch_avx2_type1.h" /* MD5 */ +#include "include/arch_avx512_type1.h" +#include "include/arch_avx512_type2.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx512_t1 +#define FLUSH_JOB flush_job_avx512_t1 +#define QUEUE_SIZE queue_size_avx512_t1 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512_t1 +#define GET_NEXT_JOB get_next_job_avx512_t1 +#define GET_COMPLETED_JOB get_completed_job_avx512_t1 +#define GET_NEXT_BURST get_next_burst_avx512_t1 +#define SUBMIT_BURST submit_burst_avx512_t1 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512_t1 +#define FLUSH_BURST flush_burst_avx512_t1 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512_t1 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512_t1 +#define SUBMIT_HASH_BURST submit_hash_burst_avx512_t1 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512_t1 + + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 + +/* AES-GCM */ +#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 +#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 +#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 +#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 +#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 +#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 + +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx512 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx512 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx512 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx512 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx512 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx512 + +static IMB_JOB * +submit_job_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_128(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_192(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_DEC_IV_256(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +static IMB_JOB * +submit_job_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_128(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_192(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_ENC_IV_256(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_gcm_dec_avx512 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx512 + +/* AES-CBC */ +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx512 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx512 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx512 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx512 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx512 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx512 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx + +/* AES-CTR */ +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx + +#define AES_CNTR_128 aes_cntr_128_avx +#define AES_CNTR_192 aes_cntr_192_avx +#define AES_CNTR_256 aes_cntr_256_avx + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx512 +#define AES_CFB_256_ONE aes_cfb_256_one_avx512 + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx512 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx512 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx512 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx512 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx512 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx512 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx512 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx512 + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 +#define FLUSH_JOB_HMAC flush_job_hmac_avx512 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +/* DES & 3DES */ +#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 +#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 + +#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 +#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 + +#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 +#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 + +#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 +#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 + +/* DES-DOCSIS */ +#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 +#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 +#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx512 +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx512 +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx512 +#define POLY1305_MAC poly1305_mac_plain_avx512 + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_no_gfni_avx512 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_no_gfni_avx512 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_no_gfni_avx512 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_no_gfni_avx512 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_no_gfni_avx512 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_no_gfni_avx512 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_avx512 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_avx512 + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB *submit_snow3g_uea2_job_avx512(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_avx512(snow3g_uea2_ooo, job); +} + +static IMB_JOB *flush_snow3g_uea2_job_avx512(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_avx512(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx512 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx512 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx512 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx512 + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, + const uint64_t key_size); +__forceinline +IMB_JOB * +FLUSH_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, + const uint64_t key_size); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, + const uint64_t key_size); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS128_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS256_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); + +static IMB_JOB * +submit_aes_docsis128_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job) +{ + (void) state; + + if (job->msg_len_to_hash_in_bytes == 0) { + if (job->msg_len_to_cipher_in_bytes == 0) { + /* NO cipher, NO CRC32 */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } + + /* Cipher, NO CRC32 */ + return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); + } + + /* Cipher + CRC32 // CRC32 */ + aes_docsis128_dec_crc32_avx512(job); + + return job; +} + +static IMB_JOB * +submit_aes_docsis256_dec_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job) +{ + (void) state; + + if (job->msg_len_to_hash_in_bytes == 0) { + if (job->msg_len_to_cipher_in_bytes == 0) { + /* NO cipher, NO CRC32 */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } + + /* Cipher, NO CRC32 */ + return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); + } + + /* Cipher + CRC32 // CRC32 */ + aes_docsis256_dec_crc32_avx512(job); + + return job; +} + +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_aes_docsis128_enc_crc32_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_aes_docsis256_enc_crc32_avx512 +#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_aes_docsis128_enc_crc32_avx512 +#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_aes_docsis256_enc_crc32_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_aes_docsis128_dec_crc32_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_aes_docsis256_dec_crc32_avx512 + +/* ====================================================================== */ + +static void +reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ + ooo_mgr_des_reset(state->des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_dec_ooo, AVX512_NUM_DES_LANES); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 16); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX512_NUM_SHA256_LANES); + + /* Init HMAC/SHA256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX512_NUM_SHA256_LANES); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX512_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX512_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); + + /* Init SNOW3G out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 16); + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 16); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX512_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX512_NUM_SHA256_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX512_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX512_NUM_SHA512_LANES); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX512 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX512) != IMB_CPUFLAGS_AVX512) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX512; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_avx512; + state->keyexp_192 = aes_keyexp_192_avx512; + state->keyexp_256 = aes_keyexp_256_avx512; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx512; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx512; + state->des_key_sched = des_key_schedule; + state->sha1_one_block = sha1_one_block_avx512; + state->sha1 = sha1_avx512; + state->sha224_one_block = sha224_one_block_avx512; + state->sha224 = sha224_avx512; + state->sha256_one_block = sha256_one_block_avx512; + state->sha256 = sha256_avx512; + state->sha384_one_block = sha384_one_block_avx512; + state->sha384 = sha384_avx512; + state->sha512_one_block = sha512_one_block_avx512; + state->sha512 = sha512_avx512; + state->md5_one_block = md5_one_block_avx512; + + state->aes128_cfb_one = aes_cfb_128_one_avx512; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx512; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx512; + state->eia3_n_buffer = zuc_eia3_n_buffer_avx512; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx512; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx512; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx512; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx512; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx512; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx512; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx512; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx512; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx512; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx512; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx512; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + + state->chacha20_poly1305_init = init_chacha20_poly1305_avx512; + state->chacha20_poly1305_enc_update = + update_enc_chacha20_poly1305_avx512; + state->chacha20_poly1305_dec_update = + update_dec_chacha20_poly1305_avx512; + state->chacha20_poly1305_finalize = + finalize_chacha20_poly1305_avx512; + + state->gcm128_enc = aes_gcm_enc_128_avx512; + state->gcm192_enc = aes_gcm_enc_192_avx512; + state->gcm256_enc = aes_gcm_enc_256_avx512; + state->gcm128_dec = aes_gcm_dec_128_avx512; + state->gcm192_dec = aes_gcm_dec_192_avx512; + state->gcm256_dec = aes_gcm_dec_256_avx512; + state->gcm128_init = aes_gcm_init_128_avx512; + state->gcm192_init = aes_gcm_init_192_avx512; + state->gcm256_init = aes_gcm_init_256_avx512; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx512; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx512; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx512; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx512; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx512; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx512; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx512; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx512; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_avx512; + state->gcm128_pre = aes_gcm_pre_128_avx512; + state->gcm192_pre = aes_gcm_pre_192_avx512; + state->gcm256_pre = aes_gcm_pre_256_avx512; + + state->ghash = ghash_avx512; + state->ghash_pre = ghash_pre_avx_gen2; + + state->gmac128_init = imb_aes_gmac_init_128_avx512; + state->gmac192_init = imb_aes_gmac_init_192_avx512; + state->gmac256_init = imb_aes_gmac_init_256_avx512; + state->gmac128_update = imb_aes_gmac_update_128_avx512; + state->gmac192_update = imb_aes_gmac_update_192_avx512; + state->gmac256_update = imb_aes_gmac_update_256_avx512; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx512; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx512; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx512; +} + +#include "mb_mgr_code.h" diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c new file mode 100644 index 00000000..0017a215 --- /dev/null +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -0,0 +1,723 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX512 + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/gcm.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_avx_type1.h" /* AESNI */ +#include "include/arch_avx2_type1.h" /* MD5 */ +#include "include/arch_avx512_type1.h" +#include "include/arch_avx512_type2.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx512_t2 +#define FLUSH_JOB flush_job_avx512_t2 +#define QUEUE_SIZE queue_size_avx512_t2 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx512_t2 +#define GET_NEXT_JOB get_next_job_avx512_t2 +#define GET_COMPLETED_JOB get_completed_job_avx512_t2 +#define SUBMIT_BURST submit_burst_avx512_t2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512_t2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512_t2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512_t2 +#define SUBMIT_HASH_BURST submit_hash_burst_avx512_t2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512_t2 +#define GET_NEXT_BURST get_next_burst_avx512_t2 +#define SUBMIT_BURST submit_burst_avx512_t2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx512_t2 +#define FLUSH_BURST flush_burst_avx512_t2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx512_t2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx512_t2 +#define SUBMIT_HASH_BURST submit_hash_burst_avx512_t2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx512_t2 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX512_T2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512_T2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512_T2 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512_T2 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512_T2 + +/* AES-GCM */ +#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 +#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 +#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 +#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 +#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 +#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 + +#define AES_GCM_DEC_IV_128_VAES aes_gcm_dec_var_iv_128_vaes_avx512 +#define AES_GCM_ENC_IV_128_VAES aes_gcm_enc_var_iv_128_vaes_avx512 +#define AES_GCM_DEC_IV_192_VAES aes_gcm_dec_var_iv_192_vaes_avx512 +#define AES_GCM_ENC_IV_192_VAES aes_gcm_enc_var_iv_192_vaes_avx512 +#define AES_GCM_DEC_IV_256_VAES aes_gcm_dec_var_iv_256_vaes_avx512 +#define AES_GCM_ENC_IV_256_VAES aes_gcm_enc_var_iv_256_vaes_avx512 + +static IMB_JOB * +vaes_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) + AES_GCM_DEC_IV_128_VAES(job->dec_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->key_len_in_bytes) + AES_GCM_DEC_IV_192_VAES(job->dec_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_DEC_IV_256_VAES(job->dec_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +static IMB_JOB * +vaes_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) + AES_GCM_ENC_IV_128_VAES(job->enc_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else if (24 == job->key_len_in_bytes) + AES_GCM_ENC_IV_192_VAES(job->enc_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + else /* assume 32 bytes */ + AES_GCM_ENC_IV_256_VAES(job->enc_keys, &ctx, + job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +#define SUBMIT_JOB_AES_GCM_DEC vaes_submit_gcm_dec_avx512 +#define SUBMIT_JOB_AES_GCM_ENC vaes_submit_gcm_enc_avx512 + +/* AES-CBC */ +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_vaes_avx512 +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_vaes_avx512 +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_vaes_avx512 + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_vaes_avx512 +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_vaes_avx512 +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_vaes_avx512 + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_vaes_avx512 +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_vaes_avx512 +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_vaes_avx512 + +#define AES_CBC_DEC_128 aes_cbc_dec_128_vaes_avx512 +#define AES_CBC_DEC_192 aes_cbc_dec_192_vaes_avx512 +#define AES_CBC_DEC_256 aes_cbc_dec_256_vaes_avx512 + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_vaes_avx512 +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_vaes_avx512 +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_vaes_avx512 +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_vaes_avx512 + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx512 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx512 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx512 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx512 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx512 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx512 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx512 +#define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx512 +#define AES_ECB_ENC_256 aes_ecb_enc_256_vaes_avx512 +#define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx512 +#define AES_ECB_DEC_192 aes_ecb_dec_192_vaes_avx512 +#define AES_ECB_DEC_256 aes_ecb_dec_256_vaes_avx512 + +/* AES-CTR */ +#define SUBMIT_JOB_AES_CNTR vaes_submit_cntr_avx512 +#define SUBMIT_JOB_AES_CNTR_BIT vaes_submit_cntr_bit_avx512 + +#define AES_CNTR_128 aes_cntr_128_avx +#define AES_CNTR_192 aes_cntr_192_avx +#define AES_CNTR_256 aes_cntr_256_avx + +static IMB_JOB * vaes_submit_cntr_avx512(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + aes_cntr_128_submit_vaes_avx512(job); + else if (24 == job->key_len_in_bytes) + aes_cntr_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_256_submit_vaes_avx512(job); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +static IMB_JOB * vaes_submit_cntr_bit_avx512(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + aes_cntr_bit_128_submit_vaes_avx512(job); + else if (24 == job->key_len_in_bytes) + aes_cntr_bit_192_submit_vaes_avx512(job); + else /* assume 32 bytes */ + aes_cntr_bit_256_submit_vaes_avx512(job); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_vaes_avx512 +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_vaes_avx512 + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_vaes_avx512 +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_vaes_avx512 + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_vaes_avx512 +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_vaes_avx512 + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_vaes_avx512 +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_vaes_avx512 + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_vaes_avx512 +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_vaes_avx512 + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx512 +#define AES_CFB_256_ONE aes_cfb_256_one_avx512 + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_vaes_avx512 +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_vaes_avx512 + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_vaes_avx512 +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_vaes_avx512 +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_vaes_avx512 +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_vaes_avx512 + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx512 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx512 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx512 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx512 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx512 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx512 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx512 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx512 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx512 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx512 + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_avx512 +#define FLUSH_JOB_HMAC flush_job_hmac_avx512 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx512 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx512 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx512 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx512 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx512 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx512 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx512 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx512 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +/* DES & 3DES */ +#define SUBMIT_JOB_DES_CBC_ENC submit_job_des_cbc_enc_avx512 +#define FLUSH_JOB_DES_CBC_ENC flush_job_des_cbc_enc_avx512 + +#define SUBMIT_JOB_DES_CBC_DEC submit_job_des_cbc_dec_avx512 +#define FLUSH_JOB_DES_CBC_DEC flush_job_des_cbc_dec_avx512 + +#define SUBMIT_JOB_3DES_CBC_ENC submit_job_3des_cbc_enc_avx512 +#define FLUSH_JOB_3DES_CBC_ENC flush_job_3des_cbc_enc_avx512 + +#define SUBMIT_JOB_3DES_CBC_DEC submit_job_3des_cbc_dec_avx512 +#define FLUSH_JOB_3DES_CBC_DEC flush_job_3des_cbc_dec_avx512 + +/* DES-DOCSIS */ +#define SUBMIT_JOB_DOCSIS_DES_ENC submit_job_docsis_des_enc_avx512 +#define FLUSH_JOB_DOCSIS_DES_ENC flush_job_docsis_des_enc_avx512 + +#define SUBMIT_JOB_DOCSIS_DES_DEC submit_job_docsis_des_dec_avx512 +#define FLUSH_JOB_DOCSIS_DES_DEC flush_job_docsis_des_dec_avx512 + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx512 +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx512 +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx512 +#define POLY1305_MAC poly1305_mac_fma_avx512 + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_gfni_avx512 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_gfni_avx512 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_gfni_avx512 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_gfni_avx512 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_gfni_avx512 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_gfni_avx512 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_gfni_avx512 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_gfni_avx512 + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB *submit_snow3g_uea2_job_vaes_avx512(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo, job); +} + +static IMB_JOB *flush_snow3g_uea2_job_vaes_avx512(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_vaes_avx512(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_vaes_avx512 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_vaes_avx512 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_vaes_avx512 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_vaes_avx512 + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, + const uint64_t key_size); +__forceinline +IMB_JOB * +FLUSH_JOB_DOCSIS_SEC_CRC_ENC(MB_MGR_DOCSIS_AES_OOO *state, + const uint64_t key_size); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, + const uint64_t key_size); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS128_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); + +__forceinline +IMB_JOB * +SUBMIT_JOB_DOCSIS256_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job); + +static IMB_JOB * +submit_job_docsis128_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job) +{ + (void) state; + + if (job->msg_len_to_hash_in_bytes == 0) { + if (job->msg_len_to_cipher_in_bytes == 0) { + /* NO cipher, NO CRC32 */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } + + /* Cipher, NO CRC32 */ + return SUBMIT_JOB_DOCSIS128_SEC_DEC(state, job); + } + + /* Cipher + CRC32 // CRC32 */ + aes_docsis128_dec_crc32_vaes_avx512(job); + + return job; +} + +static IMB_JOB * +submit_job_docsis256_sec_crc_dec_vaes_avx512(MB_MGR_DOCSIS_AES_OOO *state, + IMB_JOB *job) +{ + (void) state; + + if (job->msg_len_to_hash_in_bytes == 0) { + if (job->msg_len_to_cipher_in_bytes == 0) { + /* NO cipher, NO CRC32 */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } + + /* Cipher, NO CRC32 */ + return SUBMIT_JOB_DOCSIS256_SEC_DEC(state, job); + } + + /* Cipher + CRC32 // CRC32 */ + aes_docsis256_dec_crc32_vaes_avx512(job); + + return job; +} + +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_ENC submit_job_aes_docsis128_enc_crc32_vaes_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_ENC submit_job_aes_docsis256_enc_crc32_vaes_avx512 +#define FLUSH_JOB_DOCSIS128_SEC_CRC_ENC flush_job_aes_docsis128_enc_crc32_vaes_avx512 +#define FLUSH_JOB_DOCSIS256_SEC_CRC_ENC flush_job_aes_docsis256_enc_crc32_vaes_avx512 +#define SUBMIT_JOB_DOCSIS128_SEC_CRC_DEC submit_job_docsis128_sec_crc_dec_vaes_avx512 +#define SUBMIT_JOB_DOCSIS256_SEC_CRC_DEC submit_job_docsis256_sec_crc_dec_vaes_avx512 + +/* ====================================================================== */ + +static void +reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 16); + ooo_mgr_aes_reset(state->aes192_ooo, 16); + ooo_mgr_aes_reset(state->aes256_ooo, 16); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 16); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 16); + + /* DES, 3DES and DOCSIS DES (DES CBC + DES CFB for partial block) */ + ooo_mgr_des_reset(state->des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->des3_dec_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_enc_ooo, AVX512_NUM_DES_LANES); + ooo_mgr_des_reset(state->docsis_des_dec_ooo, AVX512_NUM_DES_LANES); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 16); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 16); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX512_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX512_NUM_SHA256_LANES); + + /* Init HMAC/SHA256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX512_NUM_SHA256_LANES); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX512_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX512_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 16); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 16); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 16); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 16); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 16); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 12); + + /* Init SNOW3G out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 16); + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 16); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX512_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX512_NUM_SHA256_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX512_NUM_SHA256_LANES); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX512_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX512_NUM_SHA512_LANES); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX512 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX512_T2) != + IMB_CPUFLAGS_AVX512_T2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX512; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_avx512; + state->keyexp_192 = aes_keyexp_192_avx512; + state->keyexp_256 = aes_keyexp_256_avx512; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx512; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx512; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx512; + + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_avx512; + state->sha1 = sha1_avx512; + state->sha224_one_block = sha224_one_block_avx512; + state->sha224 = sha224_avx512; + state->sha256_one_block = sha256_one_block_avx512; + state->sha256 = sha256_avx512; + state->sha384_one_block = sha384_one_block_avx512; + state->sha384 = sha384_avx512; + state->sha512_one_block = sha512_one_block_avx512; + state->sha512 = sha512_avx512; + state->md5_one_block = md5_one_block_avx512; + + state->aes128_cfb_one = aes_cfb_128_one_avx512; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx512; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx512; + state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_avx512; + state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_avx512; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx512; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx512; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx512; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx512; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx512; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx512; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx512; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx512; +#ifndef _WIN32 + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_vaes_avx512; +#else + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx512; +#endif + state->snow3g_init_key_sched = snow3g_init_key_sched_avx512; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx512; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx512; + state->crc16_x25 = crc16_x25_avx512; + state->crc32_sctp = crc32_sctp_avx512; + state->crc24_lte_a = crc24_lte_a_avx512; + state->crc24_lte_b = crc24_lte_b_avx512; + state->crc16_fp_data = crc16_fp_data_avx512; + state->crc11_fp_header = crc11_fp_header_avx512; + state->crc7_fp_header = crc7_fp_header_avx512; + state->crc10_iuup_data = crc10_iuup_data_avx512; + state->crc6_iuup_header = crc6_iuup_header_avx512; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx512; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx512; + + state->chacha20_poly1305_init = init_chacha20_poly1305_fma_avx512; + state->chacha20_poly1305_enc_update = + update_enc_chacha20_poly1305_fma_avx512; + state->chacha20_poly1305_dec_update = + update_dec_chacha20_poly1305_fma_avx512; + state->chacha20_poly1305_finalize = + finalize_chacha20_poly1305_fma_avx512; + + state->gcm128_enc = aes_gcm_enc_128_vaes_avx512; + state->gcm192_enc = aes_gcm_enc_192_vaes_avx512; + state->gcm256_enc = aes_gcm_enc_256_vaes_avx512; + state->gcm128_dec = aes_gcm_dec_128_vaes_avx512; + state->gcm192_dec = aes_gcm_dec_192_vaes_avx512; + state->gcm256_dec = aes_gcm_dec_256_vaes_avx512; + state->gcm128_init = aes_gcm_init_128_vaes_avx512; + state->gcm192_init = aes_gcm_init_192_vaes_avx512; + state->gcm256_init = aes_gcm_init_256_vaes_avx512; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_vaes_avx512; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_vaes_avx512; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_vaes_avx512; + state->gcm128_enc_update = aes_gcm_enc_128_update_vaes_avx512; + state->gcm192_enc_update = aes_gcm_enc_192_update_vaes_avx512; + state->gcm256_enc_update = aes_gcm_enc_256_update_vaes_avx512; + state->gcm128_dec_update = aes_gcm_dec_128_update_vaes_avx512; + state->gcm192_dec_update = aes_gcm_dec_192_update_vaes_avx512; + state->gcm256_dec_update = aes_gcm_dec_256_update_vaes_avx512; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_vaes_avx512; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_vaes_avx512; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_vaes_avx512; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_vaes_avx512; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_vaes_avx512; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_vaes_avx512; + state->gcm128_precomp = aes_gcm_precomp_128_vaes_avx512; + state->gcm192_precomp = aes_gcm_precomp_192_vaes_avx512; + state->gcm256_precomp = aes_gcm_precomp_256_vaes_avx512; + state->gcm128_pre = aes_gcm_pre_128_vaes_avx512; + state->gcm192_pre = aes_gcm_pre_192_vaes_avx512; + state->gcm256_pre = aes_gcm_pre_256_vaes_avx512; + + state->ghash = ghash_vaes_avx512; + state->ghash_pre = ghash_pre_vaes_avx512; + + state->gmac128_init = imb_aes_gmac_init_128_vaes_avx512; + state->gmac192_init = imb_aes_gmac_init_192_vaes_avx512; + state->gmac256_init = imb_aes_gmac_init_256_vaes_avx512; + state->gmac128_update = imb_aes_gmac_update_128_vaes_avx512; + state->gmac192_update = imb_aes_gmac_update_192_vaes_avx512; + state->gmac256_update = imb_aes_gmac_update_256_vaes_avx512; + state->gmac128_finalize = imb_aes_gmac_finalize_128_vaes_avx512; + state->gmac192_finalize = imb_aes_gmac_finalize_192_vaes_avx512; + state->gmac256_finalize = imb_aes_gmac_finalize_256_vaes_avx512; +} + +#include "mb_mgr_code.h" diff --git a/lib/include/arch_avx512_type2.h b/lib/include/arch_avx512_type2.h index cdc4516b..2ed2cf4b 100644 --- a/lib/include/arch_avx512_type2.h +++ b/lib/include/arch_avx512_type2.h @@ -27,7 +27,6 @@ /* AVX512 + VAES + VPCLMULQDQ + GFNI + FMA */ - #ifndef IMB_ASM_AVX512_T2_H #define IMB_ASM_AVX512_T2_H @@ -163,6 +162,8 @@ IMB_JOB *flush_job_aes256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state); void poly1305_mac_fma_avx512(IMB_JOB *job); uint32_t ethernet_fcs_avx512(const void *msg, const uint64_t len); +uint32_t ethernet_fcs_avx512_local(const void *msg, const uint64_t len, + const void *tag_ouput); uint32_t crc16_x25_avx512(const void *msg, const uint64_t len); uint32_t crc32_sctp_avx512(const void *msg, const uint64_t len); uint32_t crc24_lte_a_avx512(const void *msg, const uint64_t len); diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index c6827b2d..4f41568d 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -231,6 +231,8 @@ void aes_cmac_256_subkey_gen_avx(const void *key_exp, void aes128_cbc_mac_x8(AES_ARGS *args, uint64_t len); uint32_t ethernet_fcs_avx(const void *msg, const uint64_t len); +uint32_t ethernet_fcs_avx_local(const void *msg, const uint64_t len, + const void *tag_ouput); uint32_t crc16_x25_avx(const void *msg, const uint64_t len); uint32_t crc32_sctp_avx(const void *msg, const uint64_t len); uint32_t crc24_lte_a_avx(const void *msg, const uint64_t len); diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 24c98c9a..ee832af6 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -421,6 +421,10 @@ IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void +init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void +init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t get_next_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -429,7 +433,9 @@ get_next_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -get_next_burst_avx512(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +get_next_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +get_next_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -438,7 +444,9 @@ submit_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx512(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +submit_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_nocheck_sse(IMB_MGR *state, const uint32_t n_jobs, @@ -450,8 +458,11 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx512(IMB_MGR *state, const uint32_t n_jobs, - IMB_JOB **jobs); +submit_burst_nocheck_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_sse(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); @@ -460,7 +471,9 @@ flush_burst_avx(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -flush_burst_avx512(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +flush_burst_avx512_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_avx512_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, @@ -481,11 +494,17 @@ submit_cipher_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, @@ -506,11 +525,17 @@ submit_cipher_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_hash_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -524,9 +549,13 @@ submit_hash_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -540,7 +569,28 @@ submit_hash_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_nocheck_avx512(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); + +/* AVX512 TYPE1 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx512_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx512_t1(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx512_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512_t1(IMB_MGR *state); + +/* AVX512 TYPE2 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx512_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx512_t2(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx512_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx512_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx512_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx512_t2(IMB_MGR *state); + #endif /* IMB_IPSEC_MB_INTERNAL_H */ diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 15cf5647..39c1c49f 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1040,6 +1040,9 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_CPUFLAGS_AVX2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \ IMB_FEATURE_BMI2) #define IMB_CPUFLAGS_AVX512 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_AVX512_SKX) +#define IMB_CPUFLAGS_AVX512_T2 (IMB_CPUFLAGS_AVX512 | IMB_FEATURE_VAES | \ + IMB_FEATURE_VPCLMULQDQ | IMB_FEATURE_GFNI | \ + IMB_FEATURE_AVX512_IFMA | IMB_FEATURE_SHANI) /* TOP LEVEL (IMB_MGR) Data structure fields */ diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index ba157f31..b43b5a0e 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -548,39 +548,59 @@ EXPORTS get_next_burst_sse_no_aesni @522 get_next_burst_avx @523 get_next_burst_avx2 @524 - get_next_burst_avx512 @525 - submit_burst_sse @526 - submit_burst_sse_no_aesni @527 - submit_burst_avx @528 - submit_burst_avx2 @529 - submit_burst_avx512 @530 - submit_burst_nocheck_sse @531 - submit_burst_nocheck_sse_no_aesni @532 - submit_burst_nocheck_avx @533 - submit_burst_nocheck_avx2 @534 - submit_burst_nocheck_avx512 @535 - flush_burst_sse @536 - flush_burst_sse_no_aesni @537 - flush_burst_avx @538 - flush_burst_avx2 @539 - flush_burst_avx512 @540 - submit_cipher_burst_sse @541 - submit_cipher_burst_sse_no_aesni @542 - submit_cipher_burst_avx @543 - submit_cipher_burst_avx2 @544 - submit_cipher_burst_avx512 @545 - submit_cipher_burst_nocheck_sse @546 - submit_cipher_burst_nocheck_sse_no_aesni @547 - submit_cipher_burst_nocheck_avx @548 - submit_cipher_burst_nocheck_avx2 @549 - submit_cipher_burst_nocheck_avx512 @550 - submit_hash_burst_sse @551 - submit_hash_burst_sse_no_aesni @552 - submit_hash_burst_avx @553 - submit_hash_burst_avx2 @554 - submit_hash_burst_avx512 @555 - submit_hash_burst_nocheck_sse @556 - submit_hash_burst_nocheck_sse_no_aesni @557 - submit_hash_burst_nocheck_avx @558 - submit_hash_burst_nocheck_avx2 @559 - submit_hash_burst_nocheck_avx512 @560 + get_next_burst_avx512_t1 @525 + get_next_burst_avx512_t2 @526 + submit_burst_sse @527 + submit_burst_sse_no_aesni @528 + submit_burst_avx @529 + submit_burst_avx2 @530 + submit_burst_avx512_t1 @531 + submit_burst_avx512_t2 @532 + submit_burst_nocheck_sse @533 + submit_burst_nocheck_sse_no_aesni @534 + submit_burst_nocheck_avx @535 + submit_burst_nocheck_avx2 @536 + submit_burst_nocheck_avx512_t1 @537 + submit_burst_nocheck_avx512_t2 @538 + flush_burst_sse @539 + flush_burst_sse_no_aesni @540 + flush_burst_avx @541 + flush_burst_avx2 @542 + flush_burst_avx512_t1 @543 + flush_burst_avx512_t2 @544 + submit_cipher_burst_sse @545 + submit_cipher_burst_sse_no_aesni @546 + submit_cipher_burst_avx @547 + submit_cipher_burst_avx2 @548 + submit_cipher_burst_avx512_t1 @549 + submit_cipher_burst_avx512_t2 @550 + submit_cipher_burst_nocheck_sse @551 + submit_cipher_burst_nocheck_sse_no_aesni @552 + submit_cipher_burst_nocheck_avx @553 + submit_cipher_burst_nocheck_avx2 @554 + submit_cipher_burst_nocheck_avx512_t1 @555 + submit_cipher_burst_nocheck_avx512_t2 @556 + submit_hash_burst_sse @557 + submit_hash_burst_sse_no_aesni @558 + submit_hash_burst_avx @559 + submit_hash_burst_avx2 @560 + submit_hash_burst_avx512_t1 @561 + submit_hash_burst_avx512_t2 @562 + submit_hash_burst_nocheck_sse @563 + submit_hash_burst_nocheck_sse_no_aesni @564 + submit_hash_burst_nocheck_avx @565 + submit_hash_burst_nocheck_avx2 @566 + submit_hash_burst_nocheck_avx512_t1 @567 + submit_hash_burst_nocheck_avx512_t2 @568 + flush_job_avx512_t1 @569 + flush_job_avx512_t2 @570 + queue_size_avx512_t1 @571 + queue_size_avx512_t2 @572 + submit_job_avx512_t1 @573 + submit_job_avx512_t2 @574 + submit_job_nocheck_avx512_t1 @575 + submit_job_nocheck_avx512_t2 @576 + get_next_job_avx512_t1 @577 + get_next_job_avx512_t2 @578 + get_completed_job_avx512_t1 @579 + get_completed_job_avx512_t2 @580 diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 66348269..d6bfc2a3 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -448,6 +448,8 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_avx.obj \ $(OBJ_DIR)\mb_mgr_avx2.obj \ $(OBJ_DIR)\mb_mgr_avx512.obj \ + $(OBJ_DIR)\mb_mgr_avx512_t1.obj \ + $(OBJ_DIR)\mb_mgr_avx512_t2.obj \ $(OBJ_DIR)\mb_mgr_des_avx512.obj \ $(OBJ_DIR)\mb_mgr_sse.obj \ $(OBJ_DIR)\alloc.obj \ -- GitLab From dd6f6359e848ebbcfcc715ee362585ffa16ce58d Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Mon, 8 Aug 2022 13:59:22 +0100 Subject: [PATCH 291/369] CMAC bug fix --- test/acvp_app_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 8d9f09d2..74d05f51 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -572,7 +572,7 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) } if (tc->verify == 1) { - if (memcmp(res_tag, tc->mac, tc->mac_len != 0)) { + if (memcmp(res_tag, tc->mac, tc->mac_len) != 0) { if (verbose) { hexdump(stdout, "result tag: ", res_tag, (tc->mac_len)); -- GitLab From e5e459a62bb6f9f25ed7d6c50f5f7e0c0856be7c Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Mon, 8 Aug 2022 16:00:59 +0100 Subject: [PATCH 292/369] test: [ACVP] support for SHA-1 --- test/acvp_app_main.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 74d05f51..6cf36324 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -785,6 +785,64 @@ static int aes_hmac_sha224_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_sha1_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HASH_TC *tc; + IMB_JOB *job = NULL; + unsigned len; + uint8_t *m; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hash; + + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) { + m = malloc(tc->msg_len * 3); + len = tc->msg_len * 3; + + if (m == NULL) { + printf("Can't allocate buffer memory\n"); + return EXIT_FAILURE; + } + memcpy(m, tc->m1, tc->msg_len); + memcpy(m + tc->msg_len, tc->m2, tc->msg_len); + memcpy(m + tc->msg_len * 2, tc->m3, tc->msg_len); + } else { + m = tc->msg; + len = tc->msg_len; + } + + if (direct_api == 1) { + IMB_SHA1(mb_mgr, m, len, tc->md); + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_SHA_1; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->src = m; + job->msg_len_to_hash_in_bytes = len; + job->auth_tag_output_len_in_bytes = + IMB_SHA1_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->md; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) + free(m); + tc->md_len = IMB_SHA1_DIGEST_SIZE_IN_BYTES; + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -920,6 +978,10 @@ int main(int argc, char **argv) &aes_hmac_sha224_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA1, + &aes_sha1_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From ed3036f4856c493b29d1c2ab8c66ec7ed3eb5fd0 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Thu, 11 Aug 2022 14:36:19 +0100 Subject: [PATCH 293/369] test: [ACVP] support for HMAC-SHA-384 --- test/acvp_app_main.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 6cf36324..c4e28750 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -785,6 +785,72 @@ static int aes_hmac_sha224_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_hmac_sha384_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HMAC_TC *tc; + IMB_JOB *job = NULL; + uint32_t i = 0; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint8_t key[IMB_SHA_384_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_384_BLOCK_SIZE]; + uint32_t key_len = 0; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hmac; + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (tc->key_len <= IMB_SHA_384_BLOCK_SIZE) { + memcpy(key, tc->key, tc->key_len); + key_len = (uint32_t) tc->key_len; + } else { + IMB_SHA384(mb_mgr, tc->key, tc->key_len, key); + key_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA384_ONE_BLOCK(mb_mgr, buf, opad_hash); + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + /* + * The library only supports 24 or 48-byte tags and therefore, + * we are outputting 48 bytes always + */ + job->auth_tag_output_len_in_bytes = IMB_SHA384_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + static int aes_sha1_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; @@ -978,6 +1044,10 @@ int main(int argc, char **argv) &aes_hmac_sha224_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_384, + &aes_hmac_sha384_handler) != ACVP_SUCCESS) + goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA1, &aes_sha1_handler) != ACVP_SUCCESS) goto exit; -- GitLab From fee16d2e9d174f50e71ae084c94e3b932a4c7738 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Thu, 4 Aug 2022 13:36:53 +0100 Subject: [PATCH 294/369] test: [ACVP] support for HMAC-SHA-512 --- test/acvp_app_main.c | 70 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index c4e28750..ada8c611 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -851,6 +851,72 @@ static int aes_hmac_sha384_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_hmac_sha512_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HMAC_TC *tc; + IMB_JOB *job = NULL; + uint32_t i = 0; + DECLARE_ALIGNED(uint8_t ipad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t opad_hash[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint8_t key[IMB_SHA_512_BLOCK_SIZE]; + uint8_t buf[IMB_SHA_512_BLOCK_SIZE]; + uint32_t key_len = 0; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hmac; + + /* prepare the key */ + memset(key, 0, sizeof(key)); + if (tc->key_len <= IMB_SHA_512_BLOCK_SIZE) { + memcpy(key, tc->key, tc->key_len); + key_len = (uint32_t) tc->key_len; + } else { + IMB_SHA512(mb_mgr, tc->key, tc->key_len, key); + key_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; + } + + /* compute ipad hash */ + memset(buf, 0x36, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, ipad_hash); + + /* compute opad hash */ + memset(buf, 0x5c, sizeof(buf)); + for (i = 0; i < key_len; i++) + buf[i] ^= key[i]; + IMB_SHA512_ONE_BLOCK(mb_mgr, buf, opad_hash); + + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->u.HMAC._hashed_auth_key_xor_ipad = ipad_hash; + job->u.HMAC._hashed_auth_key_xor_opad = opad_hash; + job->src = tc->msg; + job->msg_len_to_hash_in_bytes = tc->msg_len; + /* + * The library only supports 32 or 64-byte tags and therefore, + * we are outputting 64 bytes always + */ + job->auth_tag_output_len_in_bytes = IMB_SHA512_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->mac; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + static int aes_sha1_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; @@ -1048,6 +1114,10 @@ int main(int argc, char **argv) &aes_hmac_sha384_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_512, + &aes_hmac_sha512_handler) != ACVP_SUCCESS) + goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA1, &aes_sha1_handler) != ACVP_SUCCESS) goto exit; -- GitLab From 634a9964ab0cda90c9c261e7be1404bb6eb9e8ff Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 10 Aug 2022 11:35:45 +0100 Subject: [PATCH 295/369] test: [ACVP] support for SHA2-224 --- test/acvp_app_main.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index ada8c611..8e0f8c08 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -975,6 +975,64 @@ static int aes_sha1_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_sha2_224_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HASH_TC *tc; + IMB_JOB *job = NULL; + unsigned len; + uint8_t *m; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hash; + + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) { + m = malloc(tc->msg_len * 3); + len = tc->msg_len * 3; + + if (m == NULL) { + printf("Can't allocate buffer memory\n"); + return EXIT_FAILURE; + } + memcpy(m, tc->m1, tc->msg_len); + memcpy(m + tc->msg_len, tc->m2, tc->msg_len); + memcpy(m + tc->msg_len * 2, tc->m3, tc->msg_len); + } else { + m = tc->msg; + len = tc->msg_len; + } + + if (direct_api == 1) { + IMB_SHA224(mb_mgr, m, len, tc->md); + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_SHA_224; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->src = m; + job->msg_len_to_hash_in_bytes = len; + job->auth_tag_output_len_in_bytes = + IMB_SHA224_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->md; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) + free(m); + tc->md_len = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -1122,6 +1180,10 @@ int main(int argc, char **argv) &aes_sha1_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA224, + &aes_sha2_224_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 359c72094b5023278cb289ca405f94cb1bd25aab Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 10 Aug 2022 15:21:49 +0100 Subject: [PATCH 296/369] test: [ACVP] support for SHA-256 --- test/acvp_app_main.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 8e0f8c08..d0b1c371 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1033,6 +1033,64 @@ static int aes_sha2_224_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_sha2_256_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HASH_TC *tc; + IMB_JOB *job = NULL; + unsigned len; + uint8_t *m; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hash; + + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) { + m = malloc(tc->msg_len * 3); + len = tc->msg_len * 3; + + if (m == NULL) { + printf("Can't allocate buffer memory\n"); + return EXIT_FAILURE; + } + memcpy(m, tc->m1, tc->msg_len); + memcpy(m + tc->msg_len, tc->m2, tc->msg_len); + memcpy(m + tc->msg_len * 2, tc->m3, tc->msg_len); + } else { + m = tc->msg; + len = tc->msg_len; + } + + if (direct_api == 1) { + IMB_SHA256(mb_mgr, m, len, tc->md); + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_SHA_256; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->src = m; + job->msg_len_to_hash_in_bytes = len; + job->auth_tag_output_len_in_bytes = + IMB_SHA256_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->md; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) + free(m); + tc->md_len = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -1184,6 +1242,10 @@ int main(int argc, char **argv) &aes_sha2_224_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA256, + &aes_sha2_256_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 3babcce880b96bc26363cf2bf35babca1184acf1 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 17 Aug 2022 15:48:06 +0100 Subject: [PATCH 297/369] test: [ACVP] support for SHA2-384 --- test/acvp_app_main.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index d0b1c371..492e1f74 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1091,6 +1091,64 @@ static int aes_sha2_256_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_sha2_384_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HASH_TC *tc; + IMB_JOB *job = NULL; + unsigned len; + uint8_t *m; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hash; + + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) { + m = malloc(tc->msg_len * 3); + len = tc->msg_len * 3; + + if (m == NULL) { + printf("Can't allocate buffer memory\n"); + return EXIT_FAILURE; + } + memcpy(m, tc->m1, tc->msg_len); + memcpy(m + tc->msg_len, tc->m2, tc->msg_len); + memcpy(m + tc->msg_len * 2, tc->m3, tc->msg_len); + } else { + m = tc->msg; + len = tc->msg_len; + } + + if (direct_api == 1) { + IMB_SHA384(mb_mgr, m, len, tc->md); + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_SHA_384; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->src = m; + job->msg_len_to_hash_in_bytes = len; + job->auth_tag_output_len_in_bytes = + IMB_SHA384_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->md; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) + free(m); + tc->md_len = IMB_SHA384_DIGEST_SIZE_IN_BYTES; + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -1246,6 +1304,10 @@ int main(int argc, char **argv) &aes_sha2_256_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA384, + &aes_sha2_384_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 3e47f32b42937b4cc512320c24d53ac9765f95c7 Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Wed, 17 Aug 2022 16:33:06 +0100 Subject: [PATCH 298/369] test: [ACVP] support for SHA2-512 --- test/acvp_app_main.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 492e1f74..b88930e9 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1149,6 +1149,64 @@ static int aes_sha2_384_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } +static int aes_sha2_512_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_HASH_TC *tc; + IMB_JOB *job = NULL; + unsigned len; + uint8_t *m; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.hash; + + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) { + m = malloc(tc->msg_len * 3); + len = tc->msg_len * 3; + + if (m == NULL) { + printf("Can't allocate buffer memory\n"); + return EXIT_FAILURE; + } + memcpy(m, tc->m1, tc->msg_len); + memcpy(m + tc->msg_len, tc->m2, tc->msg_len); + memcpy(m + tc->msg_len * 2, tc->m3, tc->msg_len); + } else { + m = tc->msg; + len = tc->msg_len; + } + + if (direct_api == 1) { + IMB_SHA512(mb_mgr, m, len, tc->md); + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = IMB_AUTH_SHA_512; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->src = m; + job->msg_len_to_hash_in_bytes = len; + job->auth_tag_output_len_in_bytes = + IMB_SHA512_DIGEST_SIZE_IN_BYTES; + job->auth_tag_output = tc->md; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (tc->test_type == ACVP_HASH_TEST_TYPE_MCT) + free(m); + tc->md_len = IMB_SHA512_DIGEST_SIZE_IN_BYTES; + return EXIT_SUCCESS; +} + static void usage(const char *app_name) { fprintf(stderr, "Usage: %s --req FILENAME --resp FILENAME [opt args], " @@ -1308,6 +1366,10 @@ int main(int argc, char **argv) &aes_sha2_384_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA512, + &aes_sha2_512_handler) != ACVP_SUCCESS) + goto exit; + /* Allocate and initialize MB_MGR */ if (test_arch == IMB_ARCH_NOAESNI) mb_mgr = alloc_mb_mgr(IMB_FLAG_AESNI_OFF); -- GitLab From 66a5ea2e469b00d74dd9183c0a08088f7c1ef11a Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 23 Aug 2022 11:40:57 +0100 Subject: [PATCH 299/369] lib: [ZUC-EIA3] allow un-aligned digest load/stores --- lib/avx/zuc_x4_avx.asm | 16 ++++++++-------- lib/sse_t1/zuc_x4_sse.asm | 18 +++++++++--------- test/Makefile | 4 ++++ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx/zuc_x4_avx.asm index dbe8336e..574e54ea 100644 --- a/lib/avx/zuc_x4_avx.asm +++ b/lib/avx/zuc_x4_avx.asm @@ -959,24 +959,24 @@ align 64 %endrep %if %%TAG_SIZE == 4 - vmovdqa [%%TAGS], %%KSTR1 + vmovdqu [%%TAGS], %%KSTR1 REORDER_LFSR pState, 1 %elif %%TAG_SIZE == 8 ; Transpose the keystream and store the 8 bytes per buffer consecutively, ; being the initial tag for each buffer vpunpckldq %%XTMP1, %%KSTR1, %%KSTR2 vpunpckhdq %%XTMP2, %%KSTR1, %%KSTR2 - vmovdqa [%%TAGS], %%XTMP1 - vmovdqa [%%TAGS + 16], %%XTMP2 + vmovdqu [%%TAGS], %%XTMP1 + vmovdqu [%%TAGS + 16], %%XTMP2 REORDER_LFSR pState, 2 %elif %%TAG_SIZE == 16 ; Transpose the keystream and store the 16 bytes per buffer consecutively, ; being the initial tag for each buffer TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 - vmovdqa [%%TAGS], %%KSTR1 - vmovdqa [%%TAGS + 16], %%KSTR2 - vmovdqa [%%TAGS + 16*2], %%KSTR3 - vmovdqa [%%TAGS + 16*3], %%KSTR4 + vmovdqu [%%TAGS], %%KSTR1 + vmovdqu [%%TAGS + 16], %%KSTR2 + vmovdqu [%%TAGS + 16*2], %%KSTR3 + vmovdqu [%%TAGS + 16*3], %%KSTR4 REORDER_LFSR pState, 4 %endif FUNC_RESTORE @@ -1921,7 +1921,7 @@ remainder_key_sz_128: xor [%%T], %%TMP %else ;; %%TAG_SZ == 16 vpxor %%XDIGEST, [%%T] - vmovdqa [%%T], %%XDIGEST + vmovdqu [%%T], %%XDIGEST %endif %endmacro diff --git a/lib/sse_t1/zuc_x4_sse.asm b/lib/sse_t1/zuc_x4_sse.asm index 76abf2df..55aaf3fe 100644 --- a/lib/sse_t1/zuc_x4_sse.asm +++ b/lib/sse_t1/zuc_x4_sse.asm @@ -1070,7 +1070,7 @@ mksection .text %endrep %if %%TAG_SIZE == 4 - movdqa [%%TAGS], %%KSTR1 + movdqu [%%TAGS], %%KSTR1 REORDER_LFSR pState, 1 %elif %%TAG_SIZE == 8 ; Transpose the keystream and store the 8 bytes per buffer consecutively, @@ -1078,17 +1078,17 @@ mksection .text movdqa %%XTMP1, %%KSTR1 punpckldq %%XTMP1, %%KSTR2 punpckhdq %%KSTR1, %%KSTR2 - movdqa [%%TAGS], %%XTMP1 - movdqa [%%TAGS + 16], %%KSTR1 + movdqu [%%TAGS], %%XTMP1 + movdqu [%%TAGS + 16], %%KSTR1 REORDER_LFSR pState, 2 %elif %%TAG_SIZE == 16 ; Transpose the keystream and store the 16 bytes per buffer consecutively, ; being the initial tag for each buffer TRANSPOSE4_U32 %%KSTR1, %%KSTR2, %%KSTR3, %%KSTR4, %%XTMP5, %%XTMP6 - movdqa [%%TAGS], %%KSTR1 - movdqa [%%TAGS + 16], %%KSTR2 - movdqa [%%TAGS + 16*2], %%KSTR3 - movdqa [%%TAGS + 16*3], %%KSTR4 + movdqu [%%TAGS], %%KSTR1 + movdqu [%%TAGS + 16], %%KSTR2 + movdqu [%%TAGS + 16*2], %%KSTR3 + movdqu [%%TAGS + 16*3], %%KSTR4 REORDER_LFSR pState, 4 %endif @@ -2053,9 +2053,9 @@ remainder_key_sz_128: movq %%TMP, %%XTMP6 xor [%%T], %%TMP %else ;; %%TAG_SZ == 16 - movdqa %%XTMP1, [%%T] + movdqu %%XTMP1, [%%T] pxor %%XTMP1, %%XTMP6 - movdqa [%%T], %%XTMP1 + movdqu [%%T], %%XTMP1 %endif ;; Copy last 16 bytes of KS to the front diff --git a/test/Makefile b/test/Makefile index b03700a5..72c0dc84 100644 --- a/test/Makefile +++ b/test/Makefile @@ -168,11 +168,15 @@ else endif $(FUZZ_APP): $(FUZZ_SOURCES) +ifeq ($(MINGW),0) ifneq (, $(shell which clang)) clang $(FUZZ_CFLAGS) $(FUZZ_SOURCES) $(LDLIBS) -o $@ else @echo "Clang is not installed. $(FUZZ_APP) is not built" endif +else + @echo "Fuzzing not supported on MinGW build. $(FUZZ_APP) is not built" +endif .PHONY: clean -- GitLab From cf344a7a3496b74561b68698de9b8e654090bdf8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 13:02:28 +0000 Subject: [PATCH 300/369] lib: [self-test] new self test module added with AES-CBC and AES-CTR cipher tests --- lib/Makefile | 3 +- lib/include/arch_x86_64.h | 11 ++ lib/win_x64.mak | 3 +- lib/x86_64/self_test.c | 351 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 lib/x86_64/self_test.c diff --git a/lib/Makefile b/lib/Makefile index 95264b18..149b1e26 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -282,7 +282,8 @@ c_lib_objs := \ snow3g_iv.o \ mb_mgr_auto.o \ error.o \ - ooo_mgr_reset.o + ooo_mgr_reset.o \ + self_test.o ifeq ($(AESNI_EMU), y) c_lib_objs := $(c_lib_objs) \ diff --git a/lib/include/arch_x86_64.h b/lib/include/arch_x86_64.h index 87b9a8f2..30abfc14 100644 --- a/lib/include/arch_x86_64.h +++ b/lib/include/arch_x86_64.h @@ -121,4 +121,15 @@ IMB_DLL_LOCAL void docsis_des_dec_basic(const void *input, void *output, const int size, const uint64_t *ks, const uint64_t *ivec); +/** + * @brief Runs self test on selected CAVP algorithms + * + * @param p_mgr initialized MB manager structure + * + * @return Self test status + * @retval 0 self test failed + * @retval 1 seld test passed + */ +IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr); + #endif /* IMB_ARCH_X86_64_H */ diff --git a/lib/win_x64.mak b/lib/win_x64.mak index d6bfc2a3..f755978b 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -350,7 +350,8 @@ lib_objs1 = \ $(OBJ_DIR)\error.obj \ $(OBJ_DIR)\memcpy_sse.obj \ $(OBJ_DIR)\memcpy_avx.obj \ - $(OBJ_DIR)\ooo_mgr_reset.obj + $(OBJ_DIR)\ooo_mgr_reset.obj \ + $(OBJ_DIR)\self_test.obj lib_objs2 = \ $(OBJ_DIR)\mb_mgr_aes192_cbc_enc_flush_avx.obj \ diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c new file mode 100644 index 00000000..6f32e7ba --- /dev/null +++ b/lib/x86_64/self_test.c @@ -0,0 +1,351 @@ +/******************************************************************************* + Copyright (c) 2012-2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + + +#include +#include + +#include "intel-ipsec-mb.h" +#include "arch_x86_64.h" + +struct self_test_cipher_vector { + IMB_CIPHER_MODE cipher_mode; + const uint8_t *cipher_key; + size_t cipher_key_size; /* key size in bytes */ + const uint8_t *cipher_iv; /* initialization vector */ + size_t cipher_iv_size; + const uint8_t *plain_text; + size_t plain_text_size; + const uint8_t *cipher_text; +}; + +/* + * AES-CBC Test vectors from + * http://csrc.nist.gov/publications/fips/fips197/fips-197.pdf + */ + +static const uint8_t aes_cbc_128_key[] = { + 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, + 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c +}; +static const uint8_t aes_cbc_128_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const uint8_t aes_cbc_128_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; + +static const uint8_t aes_cbc_128_cipher_text[] = { + 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, + 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, + 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, + 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, + 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 +}; + +static const uint8_t aes_cbc_192_key[] = { + 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, + 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b +}; +static const uint8_t aes_cbc_192_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const uint8_t aes_cbc_192_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t aes_cbc_192_cipher_text[] = { + 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, + 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, + 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, + 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, + 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd +}; + +static const uint8_t aes_cbc_256_key[] = { + 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, + 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, + 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 +}; +static const uint8_t aes_cbc_256_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const uint8_t aes_cbc_256_plain_text[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, + 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, + 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, + 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 +}; +static const uint8_t aes_cbc_256_cipher_text[] = { + 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, + 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, + 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, + 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, + 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b +}; + +/* + * Test Vector from + * https://tools.ietf.org/html/rfc3686 + */ + +static const uint8_t aes_ctr_128_key[] = { + 0xAE, 0x68, 0x52, 0xF8, 0x12, 0x10, 0x67, 0xCC, + 0x4B, 0xF7, 0xA5, 0x76, 0x55, 0x77, 0xF3, 0x9E, +}; +static const uint8_t aes_ctr_128_iv[] = { + 0x00, 0x00, 0x00, 0x30, /* nonce */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; +static const uint8_t aes_ctr_128_plain_text[] = { + 0x53, 0x69, 0x6E, 0x67, 0x6C, 0x65, 0x20, 0x62, + 0x6C, 0x6F, 0x63, 0x6B, 0x20, 0x6D, 0x73, 0x67, +}; +static const uint8_t aes_ctr_128_cipher_text[] = { + 0xE4, 0x09, 0x5D, 0x4F, 0xB7, 0xA7, 0xB3, 0x79, + 0x2D, 0x61, 0x75, 0xA3, 0x26, 0x13, 0x11, 0xB8, +}; + +static const uint8_t aes_ctr_192_key[] = { + 0x16, 0xAF, 0x5B, 0x14, 0x5F, 0xC9, 0xF5, 0x79, + 0xC1, 0x75, 0xF9, 0x3E, 0x3B, 0xFB, 0x0E, 0xED, + 0x86, 0x3D, 0x06, 0xCC, 0xFD, 0xB7, 0x85, 0x15, +}; +static const uint8_t aes_ctr_192_iv[] = { + 0x00, 0x00, 0x00, 0x48, /* nonce */ + 0x36, 0x73, 0x3C, 0x14, 0x7D, 0x6D, 0x93, 0xCB, +}; +static const uint8_t aes_ctr_192_plain_text[] = { + 0x53, 0x69, 0x6E, 0x67, 0x6C, 0x65, 0x20, 0x62, + 0x6C, 0x6F, 0x63, 0x6B, 0x20, 0x6D, 0x73, 0x67, +}; +static const uint8_t aes_ctr_192_cipher_text[] = { + 0x4B, 0x55, 0x38, 0x4F, 0xE2, 0x59, 0xC9, 0xC8, + 0x4E, 0x79, 0x35, 0xA0, 0x03, 0xCB, 0xE9, 0x28, +}; + +static const uint8_t aes_ctr_256_key[] = { + 0x77, 0x6B, 0xEF, 0xF2, 0x85, 0x1D, 0xB0, 0x6F, + 0x4C, 0x8A, 0x05, 0x42, 0xC8, 0x69, 0x6F, 0x6C, + 0x6A, 0x81, 0xAF, 0x1E, 0xEC, 0x96, 0xB4, 0xD3, + 0x7F, 0xC1, 0xD6, 0x89, 0xE6, 0xC1, 0xC1, 0x04, +}; +static const uint8_t aes_ctr_256_iv[] = { + 0x00, 0x00, 0x00, 0x60, /* nonce */ + 0xDB, 0x56, 0x72, 0xC9, 0x7A, 0xA8, 0xF0, 0xB2, +}; +static const uint8_t aes_ctr_256_plain_text[] = { + 0x53, 0x69, 0x6E, 0x67, 0x6C, 0x65, 0x20, 0x62, + 0x6C, 0x6F, 0x63, 0x6B, 0x20, 0x6D, 0x73, 0x67, +}; +static const uint8_t aes_ctr_256_cipher_text[] = { + 0x14, 0x5A, 0xD0, 0x1D, 0xBF, 0x82, 0x4E, 0xC7, + 0x56, 0x08, 0x63, 0xDC, 0x71, 0xE3, 0xE0, 0xC0, +}; + + +#define ADD_CIPHER_VECTOR(_cmode,_key,_iv,_plain,_cipher) \ + {_cmode, _key, sizeof(_key), _iv, sizeof(_iv), \ + _plain, sizeof(_plain), _cipher} + +struct self_test_cipher_vector cipher_vectors[] = { + ADD_CIPHER_VECTOR(IMB_CIPHER_CBC, aes_cbc_128_key, aes_cbc_128_iv, + aes_cbc_128_plain_text, aes_cbc_128_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_CBC, aes_cbc_192_key, aes_cbc_192_iv, + aes_cbc_192_plain_text, aes_cbc_192_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_CBC, aes_cbc_256_key, aes_cbc_256_iv, + aes_cbc_256_plain_text, aes_cbc_256_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_CNTR, aes_ctr_128_key, aes_ctr_128_iv, + aes_ctr_128_plain_text, aes_ctr_128_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_CNTR, aes_ctr_192_key, aes_ctr_192_iv, + aes_ctr_192_plain_text, aes_ctr_192_cipher_text), + ADD_CIPHER_VECTOR(IMB_CIPHER_CNTR, aes_ctr_256_key, aes_ctr_256_iv, + aes_ctr_256_plain_text, aes_ctr_256_cipher_text), +}; + +static int process_job(IMB_MGR *p_mgr, IMB_JOB **p_job) +{ + IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); + + if (!job) { + const int err = imb_get_errno(p_mgr); + + /* check for error */ + if (err != 0) + return 0; + + /* if no error then flush to get one */ + job = IMB_FLUSH_JOB(p_mgr); + /* if flush returns nothing then it's an error */ + if (!job) + return 0; + } + + /* if returned job is not complete then it's an error */ + if (job->status != IMB_STATUS_COMPLETED) + return 0; + + *p_job = job; + return 1; +} + +static int self_test_ciphers(IMB_MGR *p_mgr) +{ + uint8_t scratch[256]; + DECLARE_ALIGNED(uint32_t expkey_enc[4*15], 16); + DECLARE_ALIGNED(uint32_t expkey_dec[4*15], 16); + unsigned i; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (i = 0; i < IMB_DIM(cipher_vectors); i++) { + struct self_test_cipher_vector *v = &cipher_vectors[i]; + + IMB_ASSERT(v->plain_text_size <= sizeof(scratch)); + + /* message too long */ + if (v->plain_text_size > sizeof(scratch)) + return 0; + + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES_KEYEXP_128(p_mgr, v->cipher_key, + expkey_enc, expkey_dec); + break; + case IMB_KEY_192_BYTES: + IMB_AES_KEYEXP_192(p_mgr, v->cipher_key, + expkey_enc, expkey_dec); + break; + case IMB_KEY_256_BYTES: + IMB_AES_KEYEXP_256(p_mgr, v->cipher_key, + expkey_enc, expkey_dec); + break; + default: + /* invalid key size */ + return 0; + } + + /* test encrypt direction */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = v->plain_text; + job->dst = scratch; + job->cipher_mode = v->cipher_mode; + job->enc_keys = expkey_enc; + if (v->cipher_mode != IMB_CIPHER_CNTR) + job->dec_keys = expkey_dec; + job->key_len_in_bytes = v->cipher_key_size; + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + + memset(scratch, 0, sizeof(scratch)); + + /* submit job and get it processed */ + if (!process_job(p_mgr, &job)) + return 0; + + /* check for cipher text mismatch */ + if (memcmp(scratch, v->cipher_text, v->plain_text_size)) + return 0; + + /* test decrypt direction */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_NULL; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = v->cipher_text; + job->dst = scratch; + job->cipher_mode = v->cipher_mode; + job->dec_keys = expkey_dec; + if (v->cipher_mode == IMB_CIPHER_CNTR) + job->enc_keys = expkey_enc; + job->key_len_in_bytes = v->cipher_key_size; + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + + memset(scratch, 0, sizeof(scratch)); + + /* submit job and get it processed */ + if (!process_job(p_mgr, &job)) + return 0; + + /* check for plain text mismatch */ + if (memcmp(scratch, v->plain_text, v->plain_text_size)) + return 0; + + } /* for(cipher_vectors) */ + + return 1; +} + +IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) +{ + if (!self_test_ciphers(p_mgr)) + return 0; + + return 1; +} -- GitLab From 7228bacfa6f835b0c652461855d996d754aa2d17 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 13:06:39 +0000 Subject: [PATCH 301/369] lib: [self-test] new self test error code added --- lib/include/error.inc | 3 ++- lib/intel-ipsec-mb.h | 1 + lib/x86_64/error.c | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/include/error.inc b/lib/include/error.inc index 48afb5c1..54df265f 100644 --- a/lib/include/error.inc +++ b/lib/include/error.inc @@ -96,8 +96,9 @@ SET_ERRNO_TYPES \ IMB_ERR_NULL_JOB, \ IMB_ERR_QUEUE_SPACE, \ IMB_ERR_NULL_BURST, \ + IMB_ERR_BURST_SIZE, \ IMB_ERR_BURST_OOO, \ - IMB_ERR_BURST_SIZE + IMB_ERR_SELFTEST ;; Reset global imb_errno to 0 %macro IMB_ERR_CHECK_RESET 0 diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 39c1c49f..5192f8ea 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -228,6 +228,7 @@ typedef enum { IMB_ERR_NULL_BURST, IMB_ERR_BURST_SIZE, IMB_ERR_BURST_OOO, + IMB_ERR_SELFTEST, /* add new error types above this comment */ IMB_ERR_MAX /* don't move this one */ } IMB_ERR; diff --git a/lib/x86_64/error.c b/lib/x86_64/error.c index 9fc210d0..19189093 100644 --- a/lib/x86_64/error.c +++ b/lib/x86_64/error.c @@ -90,8 +90,9 @@ IMB_DLL_LOCAL const int imb_errno_types[] = { IMB_ERR_NULL_JOB, IMB_ERR_QUEUE_SPACE, IMB_ERR_NULL_BURST, + IMB_ERR_BURST_SIZE, IMB_ERR_BURST_OOO, - IMB_ERR_BURST_SIZE + IMB_ERR_SELFTEST }; #ifdef DEBUG @@ -223,6 +224,8 @@ imb_get_strerror(int errnum) return "Invalid burst size"; case IMB_ERR_BURST_OOO: return "Burst jobs out of order"; + case IMB_ERR_SELFTEST: + return "Self-test failed"; default: return strerror(errnum); } -- GitLab From 676e0a9c42bad44591c9bc736cf6de9513210b37 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 13:07:43 +0000 Subject: [PATCH 302/369] lib: [self-test] self test operation added as part of MB MGR initialization - MB MGR initialization error check added to the test application --- lib/avx/mb_mgr_avx.c | 4 ++++ lib/avx2_t1/mb_mgr_avx2.c | 4 ++++ lib/avx512_t1/mb_mgr_avx512.c | 5 +++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 4 ++++ lib/sse_t1/mb_mgr_sse.c | 4 ++++ test/main.c | 7 +++++++ 6 files changed, 28 insertions(+) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 8a24bedc..df9a0de4 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -43,6 +43,7 @@ #include "include/cpu_feature.h" #include "include/aesni_emu.h" #include "include/error.h" +#include "include/arch_x86_64.h" /* self-test */ #include "include/arch_avx_type1.h" #include "include/ooo_mgr_reset.h" @@ -622,6 +623,9 @@ void init_mb_mgr_avx(IMB_MGR *state) { init_mb_mgr_avx_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } #include "mb_mgr_code.h" diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 24925f5b..85391b9a 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -46,6 +46,7 @@ #include "include/aesni_emu.h" #include "include/error.h" +#include "include/arch_x86_64.h" /* self-test */ #include "include/arch_sse_type1.h" /* poly1305 */ #include "include/arch_avx_type1.h" #include "include/arch_avx2_type1.h" @@ -612,6 +613,9 @@ void init_mb_mgr_avx2(IMB_MGR *state) { init_mb_mgr_avx2_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } #include "mb_mgr_code.h" diff --git a/lib/avx512_t1/mb_mgr_avx512.c b/lib/avx512_t1/mb_mgr_avx512.c index 3fb8783b..d83bd29b 100644 --- a/lib/avx512_t1/mb_mgr_avx512.c +++ b/lib/avx512_t1/mb_mgr_avx512.c @@ -30,6 +30,8 @@ #include "include/error.h" #include "include/cpu_feature.h" #include "include/aesni_emu.h" +#include "include/error.h" +#include "include/arch_x86_64.h" /* self-test */ IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs) @@ -63,6 +65,9 @@ void init_mb_mgr_avx512(IMB_MGR *state) { init_mb_mgr_avx512_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } IMB_JOB *submit_job_avx512(IMB_MGR *state) diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 21f1697e..a9b7d1b6 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -41,6 +41,7 @@ #include "include/gcm.h" #include "include/noaesni.h" #include "include/error.h" +#include "include/arch_x86_64.h" /* self-test */ #include "include/arch_noaesni.h" #include "include/arch_sse_type1.h" @@ -631,6 +632,9 @@ void init_mb_mgr_sse_no_aesni(IMB_MGR *state) { init_mb_mgr_sse_no_aesni_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } #include "mb_mgr_code.h" diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index 4d507e5c..d33a6f37 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -47,6 +47,7 @@ #include "include/aesni_emu.h" #include "include/error.h" +#include "include/arch_x86_64.h" /* self-test */ #include "include/arch_sse_type1.h" #include "include/arch_sse_type2.h" #include "include/arch_sse_type3.h" @@ -945,6 +946,9 @@ void init_mb_mgr_sse(IMB_MGR *state) { init_mb_mgr_sse_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } #include "mb_mgr_code.h" diff --git a/test/main.c b/test/main.c index 22f54866..5ada502a 100644 --- a/test/main.c +++ b/test/main.c @@ -442,6 +442,13 @@ main(int argc, char **argv) break; } + if (imb_get_errno(p_mgr) != 0) { + printf("Error initializing MB_MGR structure! %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + free_mb_mgr(p_mgr); + return EXIT_FAILURE; + } + print_tested_arch(p_mgr->features, atype); for (test_idx = 0; test_idx < DIM(tests); test_idx++) { -- GitLab From 23db56c4efa7547bf0c37d72733e156a04e7d70b Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 14:15:00 +0000 Subject: [PATCH 303/369] lib: [self-test] add SHA1, SHA224, SHA256, SHA384 and SHA512 to the test module --- lib/x86_64/self_test.c | 216 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 190 insertions(+), 26 deletions(-) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 6f32e7ba..e3c15dfc 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -32,6 +32,38 @@ #include "intel-ipsec-mb.h" #include "arch_x86_64.h" +static int process_job(IMB_MGR *p_mgr, IMB_JOB **p_job) +{ + IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); + + if (!job) { + const int err = imb_get_errno(p_mgr); + + /* check for error */ + if (err != 0) + return 0; + + /* if no error then flush to get one */ + job = IMB_FLUSH_JOB(p_mgr); + /* if flush returns nothing then it's an error */ + if (!job) + return 0; + } + + /* if returned job is not complete then it's an error */ + if (job->status != IMB_STATUS_COMPLETED) + return 0; + + *p_job = job; + return 1; +} + +/* + * ============================================================================= + * CIPHER SELF-TEST + * ============================================================================= + */ + struct self_test_cipher_vector { IMB_CIPHER_MODE cipher_mode; const uint8_t *cipher_key; @@ -218,32 +250,6 @@ struct self_test_cipher_vector cipher_vectors[] = { aes_ctr_256_plain_text, aes_ctr_256_cipher_text), }; -static int process_job(IMB_MGR *p_mgr, IMB_JOB **p_job) -{ - IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); - - if (!job) { - const int err = imb_get_errno(p_mgr); - - /* check for error */ - if (err != 0) - return 0; - - /* if no error then flush to get one */ - job = IMB_FLUSH_JOB(p_mgr); - /* if flush returns nothing then it's an error */ - if (!job) - return 0; - } - - /* if returned job is not complete then it's an error */ - if (job->status != IMB_STATUS_COMPLETED) - return 0; - - *p_job = job; - return 1; -} - static int self_test_ciphers(IMB_MGR *p_mgr) { uint8_t scratch[256]; @@ -342,10 +348,168 @@ static int self_test_ciphers(IMB_MGR *p_mgr) return 1; } +/* + * ============================================================================= + * HASH SELF-TEST + * ============================================================================= + */ + +struct self_test_hash_vector { + IMB_HASH_ALG hash_mode; + const uint8_t *hash_key; + size_t hash_key_size; /* key size in bytes */ + const uint8_t *hash_iv; /* initialization vector */ + size_t hash_iv_size; + const uint8_t *message; + size_t message_size; + const uint8_t *tag; + size_t tag_size; +}; + +/* + * Test vectors come from this NIST document: + * + * https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + */ + +const uint8_t sha_message[] = { + 0x61, 0x62, 0x63, 0x64, 0x62, 0x63, 0x64, 0x65, + 0x63, 0x64, 0x65, 0x66, 0x64, 0x65, 0x66, 0x67, + 0x65, 0x66, 0x67, 0x68, 0x66, 0x67, 0x68, 0x69, + 0x67, 0x68, 0x69, 0x6a, 0x68, 0x69, 0x6a, 0x6b, + 0x69, 0x6a, 0x6b, 0x6c, 0x6a, 0x6b, 0x6c, 0x6d, + 0x6b, 0x6c, 0x6d, 0x6e, 0x6c, 0x6d, 0x6e, 0x6f, + 0x6d, 0x6e, 0x6f, 0x70, 0x6e, 0x6f, 0x70, 0x71 +}; + +const uint8_t sha1_digest[] = { + 0x84, 0x98, 0x3e, 0x44, + 0x1c, 0x3b, 0xd2, 0x6e, + 0xba, 0xae, 0x4a, 0xa1, + 0xf9, 0x51, 0x29, 0xe5, + 0xe5, 0x46, 0x70, 0xf1 +}; + +const uint8_t sha224_digest[] = { + 0x75, 0x38, 0x8b, 0x16, + 0x51, 0x27, 0x76, 0xcc, + 0x5d, 0xba, 0x5d, 0xa1, + 0xfd, 0x89, 0x01, 0x50, + 0xb0, 0xc6, 0x45, 0x5c, + 0xb4, 0xf5, 0x8b, 0x19, + 0x52, 0x52, 0x25, 0x25 +}; + +const uint8_t sha256_digest[] = { + 0x24, 0x8d, 0x6a, 0x61, + 0xd2, 0x06, 0x38, 0xb8, + 0xe5, 0xc0, 0x26, 0x93, + 0x0c, 0x3e, 0x60, 0x39, + 0xa3, 0x3c, 0xe4, 0x59, + 0x64, 0xff, 0x21, 0x67, + 0xf6, 0xec, 0xed, 0xd4, + 0x19, 0xdb, 0x06, 0xc1 +}; + +const uint8_t sha384_digest[] = { + 0x33, 0x91, 0xfd, 0xdd, 0xfc, 0x8d, 0xc7, 0x39, + 0x37, 0x07, 0xa6, 0x5b, 0x1b, 0x47, 0x09, 0x39, + 0x7c, 0xf8, 0xb1, 0xd1, 0x62, 0xaf, 0x05, 0xab, + 0xfe, 0x8f, 0x45, 0x0d, 0xe5, 0xf3, 0x6b, 0xc6, + 0xb0, 0x45, 0x5a, 0x85, 0x20, 0xbc, 0x4e, 0x6f, + 0x5f, 0xe9, 0x5b, 0x1f, 0xe3, 0xc8, 0x45, 0x2b +}; + +const uint8_t sha512_digest[] = { + 0x20, 0x4a, 0x8f, 0xc6, 0xdd, 0xa8, 0x2f, 0x0a, + 0x0c, 0xed, 0x7b, 0xeb, 0x8e, 0x08, 0xa4, 0x16, + 0x57, 0xc1, 0x6e, 0xf4, 0x68, 0xb2, 0x28, 0xa8, + 0x27, 0x9b, 0xe3, 0x31, 0xa7, 0x03, 0xc3, 0x35, + 0x96, 0xfd, 0x15, 0xc1, 0x3b, 0x1b, 0x07, 0xf9, + 0xaa, 0x1d, 0x3b, 0xea, 0x57, 0x78, 0x9c, 0xa0, + 0x31, 0xad, 0x85, 0xc7, 0xa7, 0x1d, 0xd7, 0x03, + 0x54, 0xec, 0x63, 0x12, 0x38, 0xca, 0x34, 0x45 +}; + +#define ADD_SHA_VECTOR(_hmode,_msg,_digest) \ + {_hmode, NULL, 0, NULL, 0, _msg, sizeof(_msg), \ + _digest, sizeof(_digest)} + +struct self_test_hash_vector hash_vectors[] = { + ADD_SHA_VECTOR(IMB_AUTH_SHA_1, sha_message, sha1_digest), + ADD_SHA_VECTOR(IMB_AUTH_SHA_224, sha_message, sha224_digest), + ADD_SHA_VECTOR(IMB_AUTH_SHA_256, sha_message, sha256_digest), + ADD_SHA_VECTOR(IMB_AUTH_SHA_384, sha_message, sha384_digest), + ADD_SHA_VECTOR(IMB_AUTH_SHA_512, sha_message, sha512_digest) +}; + +static int self_test_hash(IMB_MGR *p_mgr) +{ + uint8_t scratch[64]; + unsigned i; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (i = 0; i < IMB_DIM(hash_vectors); i++) { + struct self_test_hash_vector *v = &hash_vectors[i]; + + IMB_ASSERT(v->tag_size <= sizeof(scratch)); + + /* tag too long */ + if (v->tag_size > sizeof(scratch)) + return 0; + + /* test encrypt direction */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = v->hash_mode; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = v->message; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->message_size; + job->auth_tag_output = scratch; + job->auth_tag_output_len_in_bytes = v->tag_size; + + memset(scratch, 0, sizeof(scratch)); + + /* submit job and get it processed */ + if (!process_job(p_mgr, &job)) + return 0; + + /* check for TAG mismatch */ + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + + } /* for(hash_vectors) */ + + return 1; +} + +/* + * ============================================================================= + * AEAD SELF-TEST + * ============================================================================= + */ + +/* @todo */ + +/* + * ============================================================================= + * SELF-TEST INTERNAL API + * ============================================================================= + */ + IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) { if (!self_test_ciphers(p_mgr)) return 0; + if (!self_test_hash(p_mgr)) + return 0; + return 1; } -- GitLab From 648ef3fa599b02caeb258b3b1dd5bd9d19a5aed5 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 15:03:18 +0000 Subject: [PATCH 304/369] lib: [self-test] add HMAC-SHA1, HMAC-SHA224, HMAC-SHA256, HMAC-SHA384 and HMAC-SHA512 to the test module --- lib/x86_64/self_test.c | 246 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 243 insertions(+), 3 deletions(-) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index e3c15dfc..f07aa7d9 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -436,17 +436,185 @@ const uint8_t sha512_digest[] = { {_hmode, NULL, 0, NULL, 0, _msg, sizeof(_msg), \ _digest, sizeof(_digest)} +/* + * Test vector from https://csrc.nist.gov/csrc/media/publications/fips/198/ + * archive/2002-03-06/documents/fips-198a.pdf + */ + +static const uint8_t hmac_sha1_key[] = { + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0 +}; + +static const uint8_t hmac_sha1_message[] = { + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x23, + 0x34 +}; + +static const uint8_t hmac_sha1_digest[] = { + 0x9e, 0xa8, 0x86, 0xef, 0xe2, 0x68, 0xdb, 0xec, + 0xce, 0x42, 0x0c, 0x75 +}; + +/* + * Test vector from https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/hmac_sha224.pdf + */ +static const uint8_t hmac_sha224_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f +}; +static const uint8_t hmac_sha224_message[] = { + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, + 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, + 0x65, 0x6e +}; +static const uint8_t hmac_sha224_digest[] = { + 0xc7, 0x40, 0x5e, 0x3a, 0xe0, 0x58, 0xe8, 0xcd, + 0x30, 0xb0, 0x8b, 0x41, 0x40, 0x24, 0x85, 0x81, + 0xed, 0x17, 0x4c, 0xb3, 0x4e, 0x12, 0x24, 0xbc, + 0xc1, 0xef, 0xc8, 0x1b +}; + +/* + * Test vector from https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/hmac_sha256.pdf + */ +static const uint8_t hmac_sha256_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f +}; +static const uint8_t hmac_sha256_message[] = { + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, + 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, + 0x65, 0x6e +}; +static const uint8_t hmac_sha256_digest[] = { + 0x8b, 0xb9, 0xa1, 0xdb, 0x98, 0x06, 0xf2, 0x0d, + 0xf7, 0xf7, 0x7b, 0x82, 0x13, 0x8c, 0x79, 0x14, + 0xd1, 0x74, 0xd5, 0x9e, 0x13, 0xdc, 0x4d, 0x01, + 0x69, 0xc9, 0x05, 0x7b, 0x13, 0x3e, 0x1d, 0x62, +}; + +/* + * Test vector from https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/hmac_sha384.pdf + */ +static const uint8_t hmac_sha384_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f +}; +static const uint8_t hmac_sha384_message[] = { + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, + 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, + 0x65, 0x6e +}; +static const uint8_t hmac_sha384_digest[] = { + 0x63, 0xc5, 0xda, 0xa5, 0xe6, 0x51, 0x84, 0x7c, + 0xa8, 0x97, 0xc9, 0x58, 0x14, 0xab, 0x83, 0x0b, + 0xed, 0xed, 0xc7, 0xd2, 0x5e, 0x83, 0xee, 0xf9 +}; + +/* + * Test vector from https://csrc.nist.gov/csrc/media/projects/ + * cryptographic-standards-and-guidelines/documents/examples/hmac_sha512.pdf + */ +static const uint8_t hmac_sha512_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f +}; +static const uint8_t hmac_sha512_message[] = { + 0x53, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x20, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x20, 0x6b, 0x65, 0x79, 0x6c, 0x65, + 0x6e, 0x3d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6c, + 0x65, 0x6e +}; +static const uint8_t hmac_sha512_digest[] = { + 0xfc, 0x25, 0xe2, 0x40, 0x65, 0x8c, 0xa7, 0x85, + 0xb7, 0xa8, 0x11, 0xa8, 0xd3, 0xf7, 0xb4, 0xca, + 0x48, 0xcf, 0xa2, 0x6a, 0x8a, 0x36, 0x6b, 0xf2, + 0xcd, 0x1f, 0x83, 0x6b, 0x05, 0xfc, 0xb0, 0x24 +}; + +#define ADD_HMAC_SHA_VECTOR(_hmode,_key,_msg,_digest) \ + {_hmode, _key, sizeof(_key), NULL, 0, _msg, sizeof(_msg), \ + _digest, sizeof(_digest)} + struct self_test_hash_vector hash_vectors[] = { ADD_SHA_VECTOR(IMB_AUTH_SHA_1, sha_message, sha1_digest), ADD_SHA_VECTOR(IMB_AUTH_SHA_224, sha_message, sha224_digest), ADD_SHA_VECTOR(IMB_AUTH_SHA_256, sha_message, sha256_digest), ADD_SHA_VECTOR(IMB_AUTH_SHA_384, sha_message, sha384_digest), - ADD_SHA_VECTOR(IMB_AUTH_SHA_512, sha_message, sha512_digest) + ADD_SHA_VECTOR(IMB_AUTH_SHA_512, sha_message, sha512_digest), + ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_1, hmac_sha1_key, + hmac_sha1_message, hmac_sha1_digest), + ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_224, hmac_sha224_key, + hmac_sha224_message, hmac_sha224_digest), + ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_256, hmac_sha256_key, + hmac_sha256_message, hmac_sha256_digest), + ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_384, hmac_sha384_key, + hmac_sha384_message, hmac_sha384_digest), + ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_512, hmac_sha512_key, + hmac_sha512_message, hmac_sha512_digest), }; static int self_test_hash(IMB_MGR *p_mgr) { - uint8_t scratch[64]; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; unsigned i; while (IMB_FLUSH_JOB(p_mgr) != NULL) @@ -461,7 +629,7 @@ static int self_test_hash(IMB_MGR *p_mgr) if (v->tag_size > sizeof(scratch)) return 0; - /* test encrypt direction */ + /* test JOB API */ IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); job->hash_alg = v->hash_mode; @@ -474,6 +642,78 @@ static int self_test_hash(IMB_MGR *p_mgr) job->auth_tag_output = scratch; job->auth_tag_output_len_in_bytes = v->tag_size; + if (v->hash_mode == IMB_AUTH_HMAC_SHA_1) { + /* compute IPAD and OPAD */ + unsigned j; + + IMB_ASSERT(sizeof(scratch) >= IMB_SHA1_BLOCK_SIZE); + + memset(scratch, 0x36, IMB_SHA1_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + memset(scratch, 0x5c, IMB_SHA1_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + } + if (v->hash_mode == IMB_AUTH_HMAC_SHA_224 || + v->hash_mode == IMB_AUTH_HMAC_SHA_256) { + /* compute IPAD and OPAD */ + unsigned j; + + IMB_ASSERT(sizeof(scratch) >= IMB_SHA_256_BLOCK_SIZE); + + memset(scratch, 0x36, IMB_SHA_256_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + if (v->hash_mode == IMB_AUTH_HMAC_SHA_224) + IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + else + IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + memset(scratch, 0x5c, IMB_SHA_256_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + if (v->hash_mode == IMB_AUTH_HMAC_SHA_224) + IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_opad); + else + IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + } + if (v->hash_mode == IMB_AUTH_HMAC_SHA_384 || + v->hash_mode == IMB_AUTH_HMAC_SHA_512) { + /* compute IPAD and OPAD */ + unsigned j; + + IMB_ASSERT(sizeof(scratch) >= IMB_SHA_512_BLOCK_SIZE); + + memset(scratch, 0x36, IMB_SHA_512_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + if (v->hash_mode == IMB_AUTH_HMAC_SHA_384) + IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + else + IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + memset(scratch, 0x5c, IMB_SHA_512_BLOCK_SIZE); + for (j = 0; j < v->hash_key_size; j++) + scratch[j] ^= v->hash_key[j]; + if (v->hash_mode == IMB_AUTH_HMAC_SHA_384) + IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_opad); + else + IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + } + memset(scratch, 0, sizeof(scratch)); /* submit job and get it processed */ -- GitLab From 85b2e93d4e90b725e94ea5caac757a0dd7a48d69 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 15:25:05 +0000 Subject: [PATCH 305/369] lib: [self-test] add AES-CMAC-128 and AES-CMAC-256 to the test module --- lib/x86_64/self_test.c | 74 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 5 deletions(-) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index f07aa7d9..b63e7ccb 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -356,10 +356,8 @@ static int self_test_ciphers(IMB_MGR *p_mgr) struct self_test_hash_vector { IMB_HASH_ALG hash_mode; - const uint8_t *hash_key; + const uint8_t *hash_key; /* cmac, hmac */ size_t hash_key_size; /* key size in bytes */ - const uint8_t *hash_iv; /* initialization vector */ - size_t hash_iv_size; const uint8_t *message; size_t message_size; const uint8_t *tag; @@ -433,7 +431,7 @@ const uint8_t sha512_digest[] = { }; #define ADD_SHA_VECTOR(_hmode,_msg,_digest) \ - {_hmode, NULL, 0, NULL, 0, _msg, sizeof(_msg), \ + {_hmode, NULL, 0, _msg, sizeof(_msg), \ _digest, sizeof(_digest)} /* @@ -589,7 +587,44 @@ static const uint8_t hmac_sha512_digest[] = { }; #define ADD_HMAC_SHA_VECTOR(_hmode,_key,_msg,_digest) \ - {_hmode, _key, sizeof(_key), NULL, 0, _msg, sizeof(_msg), \ + {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ + _digest, sizeof(_digest)} + +/* + * 3GPP 33.401 C.2.1 Test Case 2 + */ +static const uint8_t aes_cmac_128_key[] = { + 0xd3, 0xc5, 0xd5, 0x92, 0x32, 0x7f, 0xb1, 0x1c, + 0x40, 0x35, 0xc6, 0x68, 0x0a, 0xf8, 0xc6, 0xd1 +}; + +static const uint8_t aes_cmac_128_tag[] = { + 0xb9, 0x37, 0x87, 0xe6 +}; + +static const uint8_t aes_cmac_128_message[] = { + 0x39, 0x8a, 0x59, 0xb4, 0xd4, 0x00, 0x00, 0x00, + 0x48, 0x45, 0x83, 0xd5, 0xaf, 0xe0, 0x82, 0xae +}; + +static const uint8_t aes_cmac_256_key[] = { + 0x60, 0x3D, 0xEB, 0x10, 0x15, 0xCA, 0x71, 0xBE, + 0x2B, 0x73, 0xAE, 0xF0, 0x85, 0x7D, 0x77, 0x81, + 0x1F, 0x35, 0x2C, 0x07, 0x3B, 0x61, 0x08, 0xD7, + 0x2D, 0x98, 0x10, 0xA3, 0x09, 0x14, 0xDF, 0xF4 +}; +static const uint8_t aes_cmac_256_message[] = { + 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, + 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57 +}; +static const uint8_t aes_cmac_256_tag[] = { + 0x15, 0x67, 0x27, 0xDC, 0x08, 0x78, 0x94, 0x4A, + 0x02, 0x3C, 0x1F, 0xE0, 0x3B, 0xAD, 0x6D, 0x93 +}; + +#define ADD_CMAC_VECTOR(_hmode,_key,_msg,_digest) \ + {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ _digest, sizeof(_digest)} struct self_test_hash_vector hash_vectors[] = { @@ -608,12 +643,22 @@ struct self_test_hash_vector hash_vectors[] = { hmac_sha384_message, hmac_sha384_digest), ADD_HMAC_SHA_VECTOR(IMB_AUTH_HMAC_SHA_512, hmac_sha512_key, hmac_sha512_message, hmac_sha512_digest), + ADD_CMAC_VECTOR(IMB_AUTH_AES_CMAC, aes_cmac_128_key, + aes_cmac_128_message, aes_cmac_128_tag), + ADD_CMAC_VECTOR(IMB_AUTH_AES_CMAC_256, aes_cmac_256_key, + aes_cmac_256_message, aes_cmac_256_tag), }; static int self_test_hash(IMB_MGR *p_mgr) { + /* hmac */ DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + /* cmac */ + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint32_t skey1[4], skey2[4]; + /* all */ uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; unsigned i; @@ -661,6 +706,7 @@ static int self_test_hash(IMB_MGR *p_mgr) job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; } + if (v->hash_mode == IMB_AUTH_HMAC_SHA_224 || v->hash_mode == IMB_AUTH_HMAC_SHA_256) { /* compute IPAD and OPAD */ @@ -687,6 +733,7 @@ static int self_test_hash(IMB_MGR *p_mgr) job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; } + if (v->hash_mode == IMB_AUTH_HMAC_SHA_384 || v->hash_mode == IMB_AUTH_HMAC_SHA_512) { /* compute IPAD and OPAD */ @@ -714,6 +761,23 @@ static int self_test_hash(IMB_MGR *p_mgr) job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; } + if (v->hash_mode == IMB_AUTH_AES_CMAC) { + IMB_AES_KEYEXP_128(p_mgr, v->hash_key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_128(p_mgr, expkey, skey1, skey2); + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + } + + if (v->hash_mode == IMB_AUTH_AES_CMAC_256) { + IMB_AES_KEYEXP_256(p_mgr, v->hash_key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_256(p_mgr, expkey, skey1, skey2); + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + } + + /* clear space where computed TAG is put into */ memset(scratch, 0, sizeof(scratch)); /* submit job and get it processed */ -- GitLab From eece829a11b7e86f60754b0b9649d5f47f59c81d Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 15:52:48 +0000 Subject: [PATCH 306/369] lib: [self-test] add AES-GMAC-128, AES-GMAC-192 and AES-GMAC-256 to the test module --- lib/x86_64/self_test.c | 130 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 7 deletions(-) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index b63e7ccb..94bcbb0e 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -356,12 +356,14 @@ static int self_test_ciphers(IMB_MGR *p_mgr) struct self_test_hash_vector { IMB_HASH_ALG hash_mode; - const uint8_t *hash_key; /* cmac, hmac */ + const uint8_t *hash_key; /* cmac, hmac, gmac */ size_t hash_key_size; /* key size in bytes */ const uint8_t *message; size_t message_size; const uint8_t *tag; size_t tag_size; + const uint8_t *hash_iv; /* gmac */ + size_t hash_iv_size; }; /* @@ -431,8 +433,8 @@ const uint8_t sha512_digest[] = { }; #define ADD_SHA_VECTOR(_hmode,_msg,_digest) \ - {_hmode, NULL, 0, _msg, sizeof(_msg), \ - _digest, sizeof(_digest)} + {_hmode, NULL, 0, _msg, sizeof(_msg), \ + _digest, sizeof(_digest), NULL, 0} /* * Test vector from https://csrc.nist.gov/csrc/media/publications/fips/198/ @@ -588,7 +590,7 @@ static const uint8_t hmac_sha512_digest[] = { #define ADD_HMAC_SHA_VECTOR(_hmode,_key,_msg,_digest) \ {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ - _digest, sizeof(_digest)} + _digest, sizeof(_digest), NULL, 0} /* * 3GPP 33.401 C.2.1 Test Case 2 @@ -623,9 +625,91 @@ static const uint8_t aes_cmac_256_tag[] = { 0x02, 0x3C, 0x1F, 0xE0, 0x3B, 0xAD, 0x6D, 0x93 }; -#define ADD_CMAC_VECTOR(_hmode,_key,_msg,_digest) \ - {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ - _digest, sizeof(_digest)} +#define ADD_CMAC_VECTOR(_hmode,_key,_msg,_digest) \ + {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ + _digest, sizeof(_digest), NULL, 0} + +/* + * GMAC vectors + */ +static const uint8_t aes_gmac_128_key[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F +}; +static const uint8_t aes_gmac_128_iv[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B +}; +static const uint8_t aes_gmac_128_message[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, +}; +static const uint8_t aes_gmac_128_tag[] = { + 0xC5, 0x3A, 0xF9, 0xE8 +}; + +static const uint8_t aes_gmac_192_key[] = { + 0xaa, 0x74, 0x0a, 0xbf, 0xad, 0xcd, 0xa7, 0x79, + 0x22, 0x0d, 0x3b, 0x40, 0x6c, 0x5d, 0x7e, 0xc0, + 0x9a, 0x77, 0xfe, 0x9d, 0x94, 0x10, 0x45, 0x39, +}; +static const uint8_t aes_gmac_192_iv[] = { + 0xab, 0x22, 0x65, 0xb4, 0xc1, 0x68, 0x95, 0x55, + 0x61, 0xf0, 0x43, 0x15 +}; +static const uint8_t aes_gmac_192_message[] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, +}; +static const uint8_t aes_gmac_192_tag[] = { + 0xCF, 0x82, 0x80, 0x64, 0x02, 0x46, 0xF4, 0xFB, + 0x33, 0xAE, 0x1D, 0x90, 0xEA, 0x48, 0x83, 0xDB +}; + +static const uint8_t aes_gmac_256_key[] = { + 0xb5, 0x48, 0xe4, 0x93, 0x4f, 0x5c, 0x64, 0xd3, + 0xc0, 0xf0, 0xb7, 0x8f, 0x7b, 0x4d, 0x88, 0x24, + 0xaa, 0xc4, 0x6b, 0x3c, 0x8d, 0x2c, 0xc3, 0x5e, + 0xe4, 0xbf, 0xb2, 0x54, 0xe4, 0xfc, 0xba, 0xf7, +}; +static const uint8_t aes_gmac_256_iv[] = { + 0x2e, 0xed, 0xe1, 0xdc, 0x64, 0x47, 0xc7, 0xaf, + 0xc4, 0x41, 0x53, 0x58, +}; +static const uint8_t aes_gmac_256_message[] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, + 0x01 +}; +static const uint8_t aes_gmac_256_tag[] = { + 0x77, 0x46, 0x0D, 0x6F, 0xB1, 0x87, 0xDB, 0xA9, + 0x46, 0xAD, 0xCD, 0xFB, 0xB7, 0xF9, 0x13, 0xA1 +}; + +#define ADD_GMAC_VECTOR(_hmode,_key,_iv,_msg,_tag) \ + {_hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ + _tag, sizeof(_tag), \ + _iv, sizeof(_iv)} struct self_test_hash_vector hash_vectors[] = { ADD_SHA_VECTOR(IMB_AUTH_SHA_1, sha_message, sha1_digest), @@ -647,6 +731,15 @@ struct self_test_hash_vector hash_vectors[] = { aes_cmac_128_message, aes_cmac_128_tag), ADD_CMAC_VECTOR(IMB_AUTH_AES_CMAC_256, aes_cmac_256_key, aes_cmac_256_message, aes_cmac_256_tag), + ADD_GMAC_VECTOR(IMB_AUTH_AES_GMAC_128, aes_gmac_128_key, + aes_gmac_128_iv, aes_gmac_128_message, + aes_gmac_128_tag), + ADD_GMAC_VECTOR(IMB_AUTH_AES_GMAC_192, aes_gmac_192_key, + aes_gmac_192_iv, aes_gmac_192_message, + aes_gmac_192_tag), + ADD_GMAC_VECTOR(IMB_AUTH_AES_GMAC_256, aes_gmac_256_key, + aes_gmac_256_iv, aes_gmac_256_message, + aes_gmac_256_tag), }; static int self_test_hash(IMB_MGR *p_mgr) @@ -658,6 +751,8 @@ static int self_test_hash(IMB_MGR *p_mgr) DECLARE_ALIGNED(uint32_t expkey[4*15], 16); DECLARE_ALIGNED(uint32_t dust[4*15], 16); uint32_t skey1[4], skey2[4]; + /* gmac */ + struct gcm_key_data gmac_key; /* all */ uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; unsigned i; @@ -777,6 +872,27 @@ static int self_test_hash(IMB_MGR *p_mgr) job->u.CMAC._skey2 = skey2; } + if (v->hash_mode == IMB_AUTH_AES_GMAC_128) { + IMB_AES128_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = v->hash_iv; + job->u.GMAC.iv_len_in_bytes = v->hash_iv_size; + } + + if (v->hash_mode == IMB_AUTH_AES_GMAC_192) { + IMB_AES192_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = v->hash_iv; + job->u.GMAC.iv_len_in_bytes = v->hash_iv_size; + } + + if (v->hash_mode == IMB_AUTH_AES_GMAC_256) { + IMB_AES256_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = v->hash_iv; + job->u.GMAC.iv_len_in_bytes = v->hash_iv_size; + } + /* clear space where computed TAG is put into */ memset(scratch, 0, sizeof(scratch)); -- GitLab From f8ec4e1540c03d6c215de179f33565d6fb3844fd Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 16:03:50 +0000 Subject: [PATCH 307/369] lib: [self-test] add direct API test for SHA1, SHA224, SHA256, SHA384 and SHA512 --- lib/x86_64/self_test.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 94bcbb0e..d8fd8a00 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -904,6 +904,40 @@ static int self_test_hash(IMB_MGR *p_mgr) if (memcmp(scratch, v->tag, v->tag_size)) return 0; + /* exercise direct API test if available */ + memset(scratch, 0, sizeof(scratch)); + + if (v->hash_mode == IMB_AUTH_SHA_1) { + memset(scratch, 0, sizeof(scratch)); + IMB_SHA1(p_mgr, v->message, v->message_size, scratch); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_SHA_224) { + memset(scratch, 0, sizeof(scratch)); + IMB_SHA224(p_mgr, v->message, v->message_size, scratch); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_SHA_256) { + memset(scratch, 0, sizeof(scratch)); + IMB_SHA256(p_mgr, v->message, v->message_size, scratch); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_SHA_384) { + memset(scratch, 0, sizeof(scratch)); + IMB_SHA384(p_mgr, v->message, v->message_size, scratch); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_SHA_512) { + memset(scratch, 0, sizeof(scratch)); + IMB_SHA512(p_mgr, v->message, v->message_size, scratch); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + } /* for(hash_vectors) */ return 1; -- GitLab From 336ac88df096f697ed2945bc5a07c7503b11d6ef Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 5 Aug 2022 16:11:02 +0000 Subject: [PATCH 308/369] lib: [self-test] add direct API test for AES-GMAC-128, AES-GMAC-192 and AES-GMAC-256 --- lib/x86_64/self_test.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index d8fd8a00..41fb49d7 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -937,6 +937,48 @@ static int self_test_hash(IMB_MGR *p_mgr) if (memcmp(scratch, v->tag, v->tag_size)) return 0; } + if (v->hash_mode == IMB_AUTH_AES_GMAC_128) { + struct gcm_context_data ctx; + + memset(scratch, 0, sizeof(scratch)); + IMB_AES128_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + IMB_AES128_GMAC_INIT(p_mgr, &gmac_key, &ctx, v->hash_iv, + v->hash_iv_size); + IMB_AES128_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + v->message, v->message_size); + IMB_AES128_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tag_size); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_AES_GMAC_192) { + struct gcm_context_data ctx; + + memset(scratch, 0, sizeof(scratch)); + IMB_AES192_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + IMB_AES192_GMAC_INIT(p_mgr, &gmac_key, &ctx, v->hash_iv, + v->hash_iv_size); + IMB_AES192_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + v->message, v->message_size); + IMB_AES192_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tag_size); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } + if (v->hash_mode == IMB_AUTH_AES_GMAC_256) { + struct gcm_context_data ctx; + + memset(scratch, 0, sizeof(scratch)); + IMB_AES256_GCM_PRE(p_mgr, v->hash_key, &gmac_key); + IMB_AES256_GMAC_INIT(p_mgr, &gmac_key, &ctx, v->hash_iv, + v->hash_iv_size); + IMB_AES256_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + v->message, v->message_size); + IMB_AES256_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tag_size); + if (memcmp(scratch, v->tag, v->tag_size)) + return 0; + } } /* for(hash_vectors) */ -- GitLab From 7dfa53c6aa5ec275cc073132c3f79966c7c06119 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 8 Aug 2022 10:30:11 +0000 Subject: [PATCH 309/369] lib: [self-test] add job and direct API tests for AES-GCM-128, AES-GCM-192 and AES-GCM-256 --- lib/x86_64/self_test.c | 368 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 367 insertions(+), 1 deletion(-) diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 41fb49d7..8c1cbc15 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -991,7 +991,370 @@ static int self_test_hash(IMB_MGR *p_mgr) * ============================================================================= */ -/* @todo */ +struct self_test_aead_vector { + IMB_HASH_ALG hash_mode; + IMB_CIPHER_MODE cipher_mode; + const uint8_t *cipher_key; + size_t cipher_key_size; + const uint8_t *cipher_iv; + size_t cipher_iv_size; + const uint8_t *aad; + size_t aad_size; + const uint8_t *plain_text; + size_t plain_text_size; + const uint8_t *cipher_text; + const uint8_t *tag; + size_t tag_size; +}; + +/* + * http://csrc.nist.gov/groups/STM/cavp/gcmtestvectors.zip + * gcmEncryptExtIV128.rsp + */ +static const uint8_t aes_gcm_128_key[] = { + 0xc9, 0x39, 0xcc, 0x13, 0x39, 0x7c, 0x1d, 0x37, + 0xde, 0x6a, 0xe0, 0xe1, 0xcb, 0x7c, 0x42, 0x3c +}; +static const uint8_t aes_gcm_128_iv[] = { + 0xb3, 0xd8, 0xcc, 0x01, 0x7c, 0xbb, 0x89, 0xb3, + 0x9e, 0x0f, 0x67, 0xe2 +}; +static const uint8_t aes_gcm_128_plain_text[] = { + 0xc3, 0xb3, 0xc4, 0x1f, 0x11, 0x3a, 0x31, 0xb7, + 0x3d, 0x9a, 0x5c, 0xd4, 0x32, 0x10, 0x30, 0x69 +}; +static const uint8_t aes_gcm_128_aad[] = { + 0x24, 0x82, 0x56, 0x02, 0xbd, 0x12, 0xa9, 0x84, + 0xe0, 0x09, 0x2d, 0x3e, 0x44, 0x8e, 0xda, 0x5f +}; +static const uint8_t aes_gcm_128_cipher_text[] = { + 0x93, 0xfe, 0x7d, 0x9e, 0x9b, 0xfd, 0x10, 0x34, + 0x8a, 0x56, 0x06, 0xe5, 0xca, 0xfa, 0x73, 0x54 +}; +static const uint8_t aes_gcm_128_tag[] = { + 0x00, 0x32, 0xa1, 0xdc, 0x85, 0xf1, 0xc9, 0x78, + 0x69, 0x25, 0xa2, 0xe7, 0x1d, 0x82, 0x72, 0xdd +}; + +/* + * https://tools.ietf.org/html/draft-mcgrew-gcm-test-01 case #7 + */ +static const uint8_t aes_gcm_192_key[] = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, +}; +static const uint8_t aes_gcm_192_plain_text[] = { + 0x45, 0x00, 0x00, 0x28, 0xa4, 0xad, 0x40, 0x00, + 0x40, 0x06, 0x78, 0x80, 0x0a, 0x01, 0x03, 0x8f, + 0x0a, 0x01, 0x06, 0x12, 0x80, 0x23, 0x06, 0xb8, + 0xcb, 0x71, 0x26, 0x02, 0xdd, 0x6b, 0xb0, 0x3e, + 0x50, 0x10, 0x16, 0xd0, 0x75, 0x68, 0x00, 0x01, +}; +static const uint8_t aes_gcm_192_aad[] = { + 0x00, 0x00, 0xa5, 0xf8, 0x00, 0x00, 0x00, 0x0a, +}; +static const uint8_t aes_gcm_192_iv[] = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88, +}; +static const uint8_t aes_gcm_192_cipher_text[] = { + 0xa5, 0xb1, 0xf8, 0x06, 0x60, 0x29, 0xae, 0xa4, + 0x0e, 0x59, 0x8b, 0x81, 0x22, 0xde, 0x02, 0x42, + 0x09, 0x38, 0xb3, 0xab, 0x33, 0xf8, 0x28, 0xe6, + 0x87, 0xb8, 0x85, 0x8b, 0x5b, 0xfb, 0xdb, 0xd0, + 0x31, 0x5b, 0x27, 0x45, 0x21, 0x44, 0xcc, 0x77, +}; +static const uint8_t aes_gcm_192_tag[] = { + 0x95, 0x45, 0x7b, 0x96, 0x52, 0x03, 0x7f, 0x53, + 0x18, 0x02, 0x7b, 0x5b, 0x4c, 0xd7, 0xa6, 0x36, +}; + +/* + * http://csrc.nist.gov/groups/ST/toolkit/BCM/ + * documents/proposedmodes/gcm/gcm-revised-spec.pdf + */ +static const uint8_t aes_gcm_256_key[] = { + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08, + 0xfe, 0xff, 0xe9, 0x92, 0x86, 0x65, 0x73, 0x1c, + 0x6d, 0x6a, 0x8f, 0x94, 0x67, 0x30, 0x83, 0x08 +}; +static const uint8_t aes_gcm_256_plain_text[] = { + 0xd9, 0x31, 0x32, 0x25, 0xf8, 0x84, 0x06, 0xe5, + 0xa5, 0x59, 0x09, 0xc5, 0xaf, 0xf5, 0x26, 0x9a, + 0x86, 0xa7, 0xa9, 0x53, 0x15, 0x34, 0xf7, 0xda, + 0x2e, 0x4c, 0x30, 0x3d, 0x8a, 0x31, 0x8a, 0x72, + 0x1c, 0x3c, 0x0c, 0x95, 0x95, 0x68, 0x09, 0x53, + 0x2f, 0xcf, 0x0e, 0x24, 0x49, 0xa6, 0xb5, 0x25, + 0xb1, 0x6a, 0xed, 0xf5, 0xaa, 0x0d, 0xe6, 0x57, + 0xba, 0x63, 0x7b, 0x39 +}; +static const uint8_t aes_gcm_256_aad[] = { + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef, + 0xab, 0xad, 0xda, 0xd2 +}; +static const uint8_t aes_gcm_256_iv[] = { + 0xca, 0xfe, 0xba, 0xbe, 0xfa, 0xce, 0xdb, 0xad, + 0xde, 0xca, 0xf8, 0x88 +}; +static const uint8_t aes_gcm_256_cipher_text[] = { + 0x52, 0x2d, 0xc1, 0xf0, 0x99, 0x56, 0x7d, 0x07, + 0xf4, 0x7f, 0x37, 0xa3, 0x2a, 0x84, 0x42, 0x7d, + 0x64, 0x3a, 0x8c, 0xdc, 0xbf, 0xe5, 0xc0, 0xc9, + 0x75, 0x98, 0xa2, 0xbd, 0x25, 0x55, 0xd1, 0xaa, + 0x8c, 0xb0, 0x8e, 0x48, 0x59, 0x0d, 0xbb, 0x3d, + 0xa7, 0xb0, 0x8b, 0x10, 0x56, 0x82, 0x88, 0x38, + 0xc5, 0xf6, 0x1e, 0x63, 0x93, 0xba, 0x7a, 0x0a, + 0xbc, 0xc9, 0xf6, 0x62 +}; +static const uint8_t aes_gcm_256_tag[] = { + 0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0x17, 0x68, + 0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b +}; + +#define ADD_GCM_VECTOR(_key,_iv,_aad,_plain,_cipher,_tag) \ + {IMB_AUTH_AES_GMAC, IMB_CIPHER_GCM, _key, sizeof(_key), \ + _iv, sizeof(_iv), _aad, sizeof(_aad), \ + _plain, sizeof(_plain), _cipher, \ + _tag, sizeof(_tag)} + +struct self_test_aead_vector aead_vectors[] = { + ADD_GCM_VECTOR(aes_gcm_128_key, aes_gcm_128_iv, aes_gcm_128_aad, + aes_gcm_128_plain_text, aes_gcm_128_cipher_text, + aes_gcm_128_tag), + ADD_GCM_VECTOR(aes_gcm_192_key, aes_gcm_192_iv, aes_gcm_192_aad, + aes_gcm_192_plain_text, aes_gcm_192_cipher_text, + aes_gcm_192_tag), + ADD_GCM_VECTOR(aes_gcm_256_key, aes_gcm_256_iv, aes_gcm_256_aad, + aes_gcm_256_plain_text, aes_gcm_256_cipher_text, + aes_gcm_256_tag), +}; + +static int self_test_aead(IMB_MGR *p_mgr) +{ + struct gcm_key_data gcm_key; + struct gcm_context_data ctx; + uint8_t text[128], tag[16]; + unsigned i; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (i = 0; i < IMB_DIM(aead_vectors); i++) { + struct self_test_aead_vector *v = &aead_vectors[i]; + + IMB_ASSERT(v->tag_size <= sizeof(tag)); + IMB_ASSERT(v->plain_text_size <= sizeof(text)); + + /* tag too long */ + if (v->tag_size > sizeof(tag)) + return 0; + + /* message too long */ + if (v->plain_text_size > sizeof(text)) + return 0; + + if (v->cipher_mode == IMB_CIPHER_GCM) { + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + default: + return 0; + } + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + /* encrypt test */ + job->cipher_mode = v->cipher_mode; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->key_len_in_bytes = v->cipher_key_size; + job->src = v->plain_text; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tag_size; + job->hash_alg = v->hash_mode; + if (v->cipher_mode == IMB_CIPHER_GCM) { + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aad_size; + } + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and get it processed */ + if (!process_job(p_mgr, &job)) + return 0; + + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->cipher_text, v->plain_text_size)) + return 0; + + /* decrypt test */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = v->cipher_mode; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->cipher_key_size; + job->src = v->cipher_text; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = v->cipher_iv; + job->iv_len_in_bytes = v->cipher_iv_size; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tag_size; + job->hash_alg = v->hash_mode; + if (v->cipher_mode == IMB_CIPHER_GCM) { + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aad_size; + } + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and get it processed */ + if (!process_job(p_mgr, &job)) + return 0; + + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->plain_text, v->plain_text_size)) + return 0; + + /* test direct API */ + if (v->cipher_mode != IMB_CIPHER_GCM) + continue; + + /* encrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES128_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->plain_text, + v->plain_text_size); + IMB_AES128_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES192_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->plain_text, + v->plain_text_size); + IMB_AES192_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES256_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->plain_text, + v->plain_text_size); + IMB_AES256_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + default: + return 0; + } + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->cipher_text, v->plain_text_size)) + return 0; + + /* decrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES128_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->cipher_text, + v->plain_text_size); + IMB_AES128_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES192_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->cipher_text, + v->plain_text_size); + IMB_AES192_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + v->cipher_iv, + v->cipher_iv_size, + v->aad, v->aad_size); + IMB_AES256_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + v->cipher_text, + v->plain_text_size); + IMB_AES256_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tag_size); + break; + default: + return 0; + } + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->plain_text, v->plain_text_size)) + return 0; + + } /* for(aead_vectors) */ + + return 1; +} /* * ============================================================================= @@ -1007,5 +1370,8 @@ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) if (!self_test_hash(p_mgr)) return 0; + if (!self_test_aead(p_mgr)) + return 0; + return 1; } -- GitLab From 4762414034003cb87b3bb8b7a988fa9e0fa75d36 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 8 Aug 2022 14:40:37 +0000 Subject: [PATCH 310/369] lib: [self-test] add job API tests for AES-CCM-128 and AES-CCM-256 --- lib/include/arch_x86_64.h | 2 +- lib/x86_64/self_test.c | 305 ++++++++++++++++++++++++++++++++------ 2 files changed, 257 insertions(+), 50 deletions(-) diff --git a/lib/include/arch_x86_64.h b/lib/include/arch_x86_64.h index 30abfc14..5fecfdb6 100644 --- a/lib/include/arch_x86_64.h +++ b/lib/include/arch_x86_64.h @@ -128,7 +128,7 @@ void docsis_des_dec_basic(const void *input, void *output, const int size, * * @return Self test status * @retval 0 self test failed - * @retval 1 seld test passed + * @retval 1 self test passed */ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr); diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 8c1cbc15..a126f1bf 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2022, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -32,19 +32,20 @@ #include "intel-ipsec-mb.h" #include "arch_x86_64.h" -static int process_job(IMB_MGR *p_mgr, IMB_JOB **p_job) +static int process_job(IMB_MGR *p_mgr) { IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); if (!job) { const int err = imb_get_errno(p_mgr); - + /* check for error */ if (err != 0) return 0; - - /* if no error then flush to get one */ + + /* flush to get the job processed */ job = IMB_FLUSH_JOB(p_mgr); + /* if flush returns nothing then it's an error */ if (!job) return 0; @@ -54,7 +55,6 @@ static int process_job(IMB_MGR *p_mgr, IMB_JOB **p_job) if (job->status != IMB_STATUS_COMPLETED) return 0; - *p_job = job; return 1; } @@ -308,7 +308,7 @@ static int self_test_ciphers(IMB_MGR *p_mgr) memset(scratch, 0, sizeof(scratch)); /* submit job and get it processed */ - if (!process_job(p_mgr, &job)) + if (!process_job(p_mgr)) return 0; /* check for cipher text mismatch */ @@ -336,7 +336,7 @@ static int self_test_ciphers(IMB_MGR *p_mgr) memset(scratch, 0, sizeof(scratch)); /* submit job and get it processed */ - if (!process_job(p_mgr, &job)) + if (!process_job(p_mgr)) return 0; /* check for plain text mismatch */ @@ -897,7 +897,7 @@ static int self_test_hash(IMB_MGR *p_mgr) memset(scratch, 0, sizeof(scratch)); /* submit job and get it processed */ - if (!process_job(p_mgr, &job)) + if (!process_job(p_mgr)) return 0; /* check for TAG mismatch */ @@ -991,7 +991,7 @@ static int self_test_hash(IMB_MGR *p_mgr) * ============================================================================= */ -struct self_test_aead_vector { +struct self_test_gcm_vector { IMB_HASH_ALG hash_mode; IMB_CIPHER_MODE cipher_mode; const uint8_t *cipher_key; @@ -1120,7 +1120,7 @@ static const uint8_t aes_gcm_256_tag[] = { _plain, sizeof(_plain), _cipher, \ _tag, sizeof(_tag)} -struct self_test_aead_vector aead_vectors[] = { +struct self_test_gcm_vector aead_gcm_vectors[] = { ADD_GCM_VECTOR(aes_gcm_128_key, aes_gcm_128_iv, aes_gcm_128_aad, aes_gcm_128_plain_text, aes_gcm_128_cipher_text, aes_gcm_128_tag), @@ -1129,10 +1129,11 @@ struct self_test_aead_vector aead_vectors[] = { aes_gcm_192_tag), ADD_GCM_VECTOR(aes_gcm_256_key, aes_gcm_256_iv, aes_gcm_256_aad, aes_gcm_256_plain_text, aes_gcm_256_cipher_text, - aes_gcm_256_tag), + aes_gcm_256_tag) }; -static int self_test_aead(IMB_MGR *p_mgr) + +static int self_test_aead_gcm(IMB_MGR *p_mgr) { struct gcm_key_data gcm_key; struct gcm_context_data ctx; @@ -1142,8 +1143,8 @@ static int self_test_aead(IMB_MGR *p_mgr) while (IMB_FLUSH_JOB(p_mgr) != NULL) ; - for (i = 0; i < IMB_DIM(aead_vectors); i++) { - struct self_test_aead_vector *v = &aead_vectors[i]; + for (i = 0; i < IMB_DIM(aead_gcm_vectors); i++) { + struct self_test_gcm_vector *v = &aead_gcm_vectors[i]; IMB_ASSERT(v->tag_size <= sizeof(tag)); IMB_ASSERT(v->plain_text_size <= sizeof(text)); @@ -1156,23 +1157,21 @@ static int self_test_aead(IMB_MGR *p_mgr) if (v->plain_text_size > sizeof(text)) return 0; - if (v->cipher_mode == IMB_CIPHER_GCM) { - switch (v->cipher_key_size) { - case IMB_KEY_128_BYTES: - IMB_AES128_GCM_PRE(p_mgr, v->cipher_key, - &gcm_key); - break; - case IMB_KEY_192_BYTES: - IMB_AES192_GCM_PRE(p_mgr, v->cipher_key, - &gcm_key); - break; - case IMB_KEY_256_BYTES: - IMB_AES256_GCM_PRE(p_mgr, v->cipher_key, - &gcm_key); - break; - default: - return 0; - } + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_PRE(p_mgr, v->cipher_key, + &gcm_key); + break; + default: + return 0; } /* test JOB API */ @@ -1192,17 +1191,16 @@ static int self_test_aead(IMB_MGR *p_mgr) job->auth_tag_output = tag; job->auth_tag_output_len_in_bytes = v->tag_size; job->hash_alg = v->hash_mode; - if (v->cipher_mode == IMB_CIPHER_GCM) { - job->enc_keys = &gcm_key; - job->dec_keys = &gcm_key; - job->u.GCM.aad = v->aad; - job->u.GCM.aad_len_in_bytes = v->aad_size; - } + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aad_size; + memset(text, 0, sizeof(text)); memset(tag, 0, sizeof(tag)); /* submit job and get it processed */ - if (!process_job(p_mgr, &job)) + if (!process_job(p_mgr)) return 0; /* check for TAG mismatch */ @@ -1229,17 +1227,16 @@ static int self_test_aead(IMB_MGR *p_mgr) job->auth_tag_output = tag; job->auth_tag_output_len_in_bytes = v->tag_size; job->hash_alg = v->hash_mode; - if (v->cipher_mode == IMB_CIPHER_GCM) { - job->enc_keys = &gcm_key; - job->dec_keys = &gcm_key; - job->u.GCM.aad = v->aad; - job->u.GCM.aad_len_in_bytes = v->aad_size; - } + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aad_size; + memset(text, 0, sizeof(text)); memset(tag, 0, sizeof(tag)); /* submit job and get it processed */ - if (!process_job(p_mgr, &job)) + if (!process_job(p_mgr)) return 0; /* check for TAG mismatch */ @@ -1251,8 +1248,6 @@ static int self_test_aead(IMB_MGR *p_mgr) return 0; /* test direct API */ - if (v->cipher_mode != IMB_CIPHER_GCM) - continue; /* encrypt direction */ memset(text, 0, sizeof(text)); @@ -1351,8 +1346,220 @@ static int self_test_aead(IMB_MGR *p_mgr) if (memcmp(text, v->plain_text, v->plain_text_size)) return 0; - } /* for(aead_vectors) */ + } /* for(gcm_vectors) */ + + return 1; +} + +struct self_test_aead_ccm_vector { + IMB_HASH_ALG hash_mode; + IMB_CIPHER_MODE cipher_mode; + const uint8_t *cipher_key; + size_t cipher_key_size; + const uint8_t *cipher_nonce; + size_t cipher_nonce_size; + const uint8_t *aad; + size_t aad_size; + const uint8_t *plain_text; + size_t plain_text_size; + const uint8_t *cipher_text; + const uint8_t *tag; + size_t tag_size; +}; + +/* + * Test vectors from https://tools.ietf.org/html/rfc3610 + */ +static const uint8_t aes_ccm_128_key[] = { + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF +}; +static const uint8_t aes_ccm_128_nonce[] = { + 0x00, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00, 0xA0, + 0xA1, 0xA2, 0xA3, 0xA4, 0xA5 +}; +static const uint8_t aes_ccm_128_plain_text[] = { + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E +}; +static const uint8_t aes_ccm_128_aad[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 +}; +static const uint8_t aes_ccm_128_cipher_text[] = { + 0x58, 0x8C, 0x97, 0x9A, 0x61, 0xC6, 0x63, 0xD2, + 0xF0, 0x66, 0xD0, 0xC2, 0xC0, 0xF9, 0x89, 0x80, + 0x6D, 0x5F, 0x6B, 0x61, 0xDA, 0xC3, 0x84, +}; +static const uint8_t aes_ccm_128_tag[] = { + 0x17, 0xE8, 0xD1, 0x2C, 0xFD, 0xF9, 0x26, 0xE0 +}; + +static const uint8_t aes_ccm_256_key[] = { + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, + 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, + 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF +}; +static const uint8_t aes_ccm_256_nonce[] = { + 0x00, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00, 0xA0, + 0xA1, 0xA2, 0xA3, 0xA4, 0xA5 +}; +static const uint8_t aes_ccm_256_plain_text[] = { + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E +}; +static const uint8_t aes_ccm_256_aad[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, +}; +static const uint8_t aes_ccm_256_cipher_text[] = { + 0x21, 0x61, 0x63, 0xDE, 0xCF, 0x74, 0xE0, 0x0C, + 0xAB, 0x04, 0x56, 0xFF, 0x45, 0xCD, 0xA7, 0x17, + 0x1F, 0xA5, 0x96, 0xD7, 0x0F, 0x76, 0x91 +}; +static const uint8_t aes_ccm_256_tag[] = { + 0xCA, 0x8A, 0xFA, 0xA2, 0x3F, 0x22, 0x3E, 0x64 +}; + +#define ADD_CCM_VECTOR(_key,_nonce,_aad,_plain,_cipher,_tag) \ + {IMB_AUTH_AES_CCM, IMB_CIPHER_CCM, _key, sizeof(_key), \ + _nonce, sizeof(_nonce), _aad, sizeof(_aad), \ + _plain, sizeof(_plain), _cipher, \ + _tag, sizeof(_tag)} +struct self_test_aead_ccm_vector aead_ccm_vectors[] = { + ADD_CCM_VECTOR(aes_ccm_128_key, aes_ccm_128_nonce, aes_ccm_128_aad, + aes_ccm_128_plain_text, aes_ccm_128_cipher_text, + aes_ccm_128_tag), + ADD_CCM_VECTOR(aes_ccm_256_key, aes_ccm_256_nonce, aes_ccm_256_aad, + aes_ccm_256_plain_text, aes_ccm_256_cipher_text, + aes_ccm_256_tag) +}; + +static int self_test_aead_ccm(IMB_MGR *p_mgr) +{ + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint8_t text[128], tag[16]; + unsigned i; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (i = 0; i < IMB_DIM(aead_ccm_vectors); i++) { + struct self_test_aead_ccm_vector *v = &aead_ccm_vectors[i]; + + IMB_ASSERT(v->tag_size <= sizeof(tag)); + IMB_ASSERT(v->plain_text_size <= sizeof(text)); + + /* tag too long */ + if (v->tag_size > sizeof(tag)) + return 0; + + /* message too long */ + if (v->plain_text_size > sizeof(text)) + return 0; + + switch (v->cipher_key_size) { + case IMB_KEY_128_BYTES: + IMB_AES_KEYEXP_128(p_mgr, v->cipher_key, expkey, + dust); + break; + case IMB_KEY_256_BYTES: + IMB_AES_KEYEXP_256(p_mgr, v->cipher_key, expkey, + dust); + break; + default: + return 0; + } + + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + /* encrypt test */ + job->cipher_mode = v->cipher_mode; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->cipher_key_size; + job->src = v->plain_text; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->msg_len_to_hash_in_bytes = v->plain_text_size; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->iv = v->cipher_nonce; + job->iv_len_in_bytes = v->cipher_nonce_size; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tag_size; + job->hash_alg = v->hash_mode; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->u.CCM.aad_len_in_bytes = v->aad_size; + job->u.CCM.aad = v->aad; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and get it processed */ + if (!process_job(p_mgr)) + return 0; + + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->cipher_text, v->plain_text_size)) + return 0; + + /* decrypt test */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = v->cipher_mode; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->key_len_in_bytes = v->cipher_key_size; + job->src = v->cipher_text; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->plain_text_size; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->msg_len_to_hash_in_bytes = v->plain_text_size; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->iv = v->cipher_nonce; + job->iv_len_in_bytes = v->cipher_nonce_size; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tag_size; + job->hash_alg = v->hash_mode; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->u.CCM.aad_len_in_bytes = v->aad_size; + job->u.CCM.aad = v->aad; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and get it processed */ + if (!process_job(p_mgr)) + return 0; + + /* check for TAG mismatch */ + if (memcmp(tag, v->tag, v->tag_size)) + return 0; + + /* check for text mismatch */ + if (memcmp(text, v->plain_text, v->plain_text_size)) + return 0; + } /* for(ccm_vectors) */ + + return 1; +} + +static int self_test_aead(IMB_MGR *p_mgr) +{ + if (!self_test_aead_gcm(p_mgr)) + return 0; + if (!self_test_aead_ccm(p_mgr)) + return 0; return 1; } -- GitLab From ec0e9fb4ca45620069d124c1c5c6becdd3c4f763 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 19 Aug 2022 15:28:30 +0000 Subject: [PATCH 311/369] test: [self-test] add check for library initialization self-test status --- test/acvp_app_main.c | 7 +++++++ test/ipsec_xvalid.c | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index b88930e9..a808fa6e 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1407,6 +1407,13 @@ int main(int argc, char **argv) goto exit; } + if (imb_get_errno(mb_mgr) != 0) { + fprintf(stderr, "Error initializing MB_MGR structure! %s\n", + imb_get_strerror(imb_get_errno(mb_mgr))); + free_mb_mgr(mb_mgr); + goto exit; + } + /* Parse request file, run crypto tests and write out response file */ acvp_run_vectors_from_file(ctx, req_filename, resp_filename); diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index 331a9dcb..c05d07fa 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -2479,6 +2479,12 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } + if (imb_get_errno(enc_mgr) != 0) { + fprintf(stderr, "Error initializing enc MB_MGR structure! %s\n", + imb_get_strerror(imb_get_errno(enc_mgr))); + exit(EXIT_FAILURE); + } + printf("Encrypting "); print_tested_arch(enc_mgr->features, enc_arch); @@ -2516,6 +2522,12 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } + if (imb_get_errno(dec_mgr) != 0) { + fprintf(stderr, "Error initializing dec MB_MGR structure! %s\n", + imb_get_strerror(imb_get_errno(dec_mgr))); + exit(EXIT_FAILURE); + } + printf("Decrypting "); print_tested_arch(dec_mgr->features, dec_arch); -- GitLab From 79846d4b22285a248ef04065f9d1ee2ea6c5aebd Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 19 Aug 2022 15:30:18 +0000 Subject: [PATCH 312/369] doc: FIPS compliance section added to README --- README | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/README b/README index af997511..5dd448b1 100644 --- a/README +++ b/README @@ -44,6 +44,7 @@ CONTENTS 9. Backwards compatibility 10. Disclaimer (ZUC, KASUMI, SNOW3G) 11. Legal Disclaimer +12. FIPS Compliance @@ -681,3 +682,72 @@ DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT. + + + +12. FIPS COMPLIANCE + + +The library does not fulfill technical requirements to achieve +Cryptographic Module (CMVP) certification as a standalone component. It +is fit for Cryptographic Algorithm validation and certification (CAVP) +and it can be part of CMVP as one of the components. + +CAVP + +ACVP test application located in test directory is to support CAVP +process. It implements validation of the following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-CTR +- AES-CMAC +- SHA1 +- SHA224 +- SHA256 +- SHA384 +- SHA512 +- HMAC-SHA1 +- HMAC-SHA224 +- HMAC-SHA256 +- HMAC-SHA384 +- HMAC-SHA512 + +Self-Test + +In order to support CMVP, the library implements Self-Test functionality +that is available with all compilation options. The test is always +performed as part of library initialization (power-up). There is no +conditional self-test functionality as none of such conditions occur +(i.e. pair-wise consistency test, software/firmware load test, manual +key entry test, continuous random number generator test, and bypass +test). The self-test consists of Cryptographic algorithm test (known +answer test) on following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-CTR +- AES-CMAC +- SHA1 +- SHA224 +- SHA256 +- SHA384 +- SHA512 +- HMAC-SHA1 +- HMAC-SHA224 +- HMAC-SHA256 +- HMAC-SHA384 +- HMAC-SHA512 + +Detecting library self-test failure in the application: + + IMB_ARCH arch; + IMB_MGR *p_mgr = alloc_mb_mgr(0); + + init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ + if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { + /* self-test error */ + exit(EXIT_FAILURE); + } diff --git a/README.md b/README.md index 32a1ec0d..9c7c3e62 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Contents 9. Backwards compatibility 10. Disclaimer (ZUC, KASUMI, SNOW3G) 11. Legal Disclaimer +12. FIPS Compliance 1\. Overview ============ @@ -626,3 +627,63 @@ DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT. + +12\. FIPS Compliance +=================== + +The library does not fulfill technical requirements to achieve Cryptographic Module (CMVP) certification as a standalone component. It is fit for Cryptographic Algorithm validation and certification (CAVP) and it can be part of CMVP as one of the components. + +### CAVP +ACVP test application located in `test` directory is to support CAVP process. It implements validation of the following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-CTR +- AES-CMAC +- SHA1 +- SHA224 +- SHA256 +- SHA384 +- SHA512 +- HMAC-SHA1 +- HMAC-SHA224 +- HMAC-SHA256 +- HMAC-SHA384 +- HMAC-SHA512 + +### Self-Test + +In order to support CMVP, the library implements Self-Test functionality that is available with all compilation options. +The test is always performed as part of library initialization (power-up). There is no conditional self-test functionality as none of such conditions occur (i.e. pair-wise consistency test, +software/firmware load test, manual key entry test, continuous random number generator test, and +bypass test). +The self-test consists of Cryptographic algorithm test (known answer test) on following algorithms: +- AES-GCM +- AES-GMAC +- AES-CCM +- AES-CBC +- AES-CTR +- AES-CMAC +- SHA1 +- SHA224 +- SHA256 +- SHA384 +- SHA512 +- HMAC-SHA1 +- HMAC-SHA224 +- HMAC-SHA256 +- HMAC-SHA384 +- HMAC-SHA512 + +Detecting library self-test failure in the application: +``` +IMB_ARCH arch; +IMB_MGR *p_mgr = alloc_mb_mgr(0); + +init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ +if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { + /* self-test error */ + exit(EXIT_FAILURE); +} +``` \ No newline at end of file -- GitLab From 0bed246d6d2b5f9a2d11866af824dbf5891bf1c7 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 26 Aug 2022 11:05:45 +0000 Subject: [PATCH 313/369] perf: add check for library initialization status --- perf/ipsec_perf.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index eac860cb..7c27f24f 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -3188,6 +3188,12 @@ run_tests(void *arg) break; } + if (imb_get_errno(p_mgr) != 0) { + printf("Error initializing MB_MGR! %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + goto exit_failure; + } + process_variant(p_mgr, arch, ¶ms, variant_ptr, run, buf, keys); -- GitLab From 1ff8ebb94641050b7c87792bc352a49adf8a51d2 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 24 Aug 2022 15:48:21 +0000 Subject: [PATCH 314/369] avx512: [ZUC-EIA3-256] fix stack space ZUC-256 remainder function requires 6 parameters when called, not the 5 that ZUC-128 remainder needs. --- lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm index 0c2df325..98316192 100644 --- a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm @@ -764,7 +764,11 @@ FLUSH_JOB_ZUC256_EEA3: call ZUC_KEYGEN_16 %%_exit: +%if %%KEY_SIZE == 128 RESERVE_STACK_SPACE 5 +%else + RESERVE_STACK_SPACE 6 +%endif ; Digest final bytes of data and generate tag for finished buffers lea arg1, [%%OOO + _zuc_args_digest] @@ -779,13 +783,13 @@ FLUSH_JOB_ZUC256_EEA3: %if %%KEY_SIZE == 128 call ZUC_REMAINDER_16 + RESTORE_STACK_SPACE 5 %else mov arg6, %%TAG_SIZE call ZUC256_REMAINDER_16 + RESTORE_STACK_SPACE 6 %endif - RESTORE_STACK_SPACE 5 - mov word [%%OOO + _zuc_init_not_done], 0 %endmacro -- GitLab From 5a8c7821b89dc6b00239d6cbb5a9943cbedc0b37 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Mon, 22 Aug 2022 16:02:36 +0000 Subject: [PATCH 315/369] lib:[SHA256-NI] single buffer implementation for sse --- lib/Makefile | 1 + lib/avx/sha256_one_block_avx.asm | 7 + lib/avx2_t1/sha256_oct_avx2.asm | 21 +- lib/include/sha_mb_mgr.h | 3 + .../mb_mgr_hmac_sha256_flush_ni_sse.asm | 21 +- lib/sse_t2/sha256_ni_x1_sse.asm | 427 ++++++++++++++++++ lib/sse_t2/sha256_ni_x2_sse.asm | 21 +- lib/win_x64.mak | 1 + 8 files changed, 452 insertions(+), 50 deletions(-) create mode 100644 lib/sse_t2/sha256_ni_x1_sse.asm diff --git a/lib/Makefile b/lib/Makefile index 149b1e26..5f5d299b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -410,6 +410,7 @@ asm_sse_lib_objs := \ sha1_ni_x2_sse.o \ sha1_ni_x1_sse.o \ sha256_ni_x2_sse.o \ + sha256_ni_x1_sse.o \ zuc_x4_sse.o \ zuc_x4_gfni_sse.o \ mb_mgr_aes128_cbc_enc_flush_x4_sse.o \ diff --git a/lib/avx/sha256_one_block_avx.asm b/lib/avx/sha256_one_block_avx.asm index a1ad4500..c50cd0a4 100644 --- a/lib/avx/sha256_one_block_avx.asm +++ b/lib/avx/sha256_one_block_avx.asm @@ -32,6 +32,9 @@ mksection .rodata default rel + +%ifndef FUNC +MKGLOBAL(K256,data,internal) align 64 K256: dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 @@ -50,7 +53,11 @@ K256: dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +%else +extern K256 +%endif +align 16 PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x0c0d0e0f08090a0b0405060700010203 dq 0x0405060700010203, 0x0c0d0e0f08090a0b diff --git a/lib/avx2_t1/sha256_oct_avx2.asm b/lib/avx2_t1/sha256_oct_avx2.asm index b7e53781..61f440fd 100644 --- a/lib/avx2_t1/sha256_oct_avx2.asm +++ b/lib/avx2_t1/sha256_oct_avx2.asm @@ -186,26 +186,7 @@ PSHUFFLE_BYTE_FLIP_MASK: ;ddq 0x0c0d0e0f08090a0b0405060700010203 dq 0x0405060700010203, 0x0c0d0e0f08090a0b dq 0x0405060700010203, 0x0c0d0e0f08090a0b -align 64 -MKGLOBAL(K256,data,internal) -K256: - dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 - dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 - dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 - dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 - dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc - dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da - dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 - dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 - dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 - dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 - dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 - dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 - dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 - dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 - dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 - dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 - +extern K256 mksection .text %define XMM_STORAGE 10*16 diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index fea65d9c..aaebfb7c 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -62,6 +62,9 @@ extern void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +extern void call_sha256_ni_x1_sse_from_c(SHA256_ARGS *args, + uint32_t size_in_blocks); + extern void call_sha512_x2_sse_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); diff --git a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm index a137d9e2..a566b12c 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm @@ -41,14 +41,16 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" -extern sha256_ni +extern sha256_ni_x1 %ifdef LINUX %define arg1 rdi %define arg2 rsi +%define arg3 rdx %else %define arg1 rcx %define arg2 rdx +%define arg3 r8 %endif %define state arg1 @@ -73,7 +75,7 @@ extern sha256_ni %define extra_blocks arg2 %define p arg2 -%define tmp4 r8 +%define tmp4 r11 %define tmp5 r9 @@ -136,26 +138,21 @@ flush_job_hmac_sha_256_ni_sse: DBGPRINTL64 "idx:", idx copy_lane_data: - ; copy idx to empty lanes - mov tmp, [state + _args_data_ptr_sha256 + PTR_SZ*idx] xor len2, len2 mov WORD(len2), word [state + _lens_sha256 + idx*2] - ; there are only two lanes so if one is empty it is easy to determine which one - xor idx, 1 - mov [state + _args_data_ptr_sha256 + PTR_SZ*idx], tmp - xor idx, 1 + mov arg3, idx ; No need to find min length - only two lanes available cmp len2, 0 je len_is_0 - ; set length on both lanes to 0 - mov dword [state + _lens_sha256], 0 + ; set length lane to 0 + mov dword [state + _lens_sha256 + idx*2], 0 - ; "state" and "args" are the same address, arg1 + ; "state" and "args" are the same address, arg1 ; len is arg2 - call sha256_ni + call sha256_ni_x1 ; state and idx are intact len_is_0: diff --git a/lib/sse_t2/sha256_ni_x1_sse.asm b/lib/sse_t2/sha256_ni_x1_sse.asm new file mode 100644 index 00000000..e1bef2a7 --- /dev/null +++ b/lib/sse_t2/sha256_ni_x1_sse.asm @@ -0,0 +1,427 @@ +; +;; Copyright (c) 2022, Intel Corporation +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are met: +;; +;; * Redistributions of source code must retain the above copyright notice, +;; this list of conditions and the following disclaimer. +;; * Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; * Neither the name of Intel Corporation nor the names of its contributors +;; may be used to endorse or promote products derived from this software +;; without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;; + +;; Stack must be aligned to 32 bytes before call +;; +;; Registers: RAX RBX RCX RDX RBP RSI RDI R8 R9 R10 R11 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Windows clobbers: RCX RDX RSI RDI R11 +;; Windows preserves: RAX RBX RBP R8 R9 R10 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; Linux clobbers: RCX RDX RSI RDI R11 +;; Linux preserves: RAX RBX RBP R8 R9 R10 R12 R13 R14 R15 +;; ----------------------------------------------------------- +;; +;; Linux/Windows clobbers: xmm0 - xmm15 + +%include "include/os.asm" +%include "include/cet.inc" +%include "include/mb_mgr_datastruct.asm" +%include "include/clear_regs.asm" + +; resdq = res0 => 16 bytes +struc frame +.ABEF_SAVE reso 1 +.CDGH_SAVE reso 1 +.align resq 1 +endstruc + +%ifdef LINUX +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%else +%define arg1 rcx +%define arg2 rdx +%define arg3 r8 +%define arg4 r9 +%endif + +%define args arg1 +%define NUM_BLKS arg2 +%define lane arg3 +%define tmp arg4 + +%define INP r10 + +%define SHA256CONSTANTS r11 + +;; MSG MUST be xmm0 (implicit argument) +%define MSG xmm0 +%define STATE0 xmm1 +%define STATE1 xmm2 +%define MSGTMP0 xmm3 +%define MSGTMP1 xmm4 +%define MSGTMP2 xmm5 +%define MSGTMP3 xmm6 +%define MSGTMP4 xmm7 +%define MSGTMP xmm14 +%define SHUF_MASK xmm15 + +mksection .rodata +default rel + +extern K256 + +align 64 +PSHUFFLE_BYTE_FLIP_MASK: + dq 0x0405060700010203, 0x0c0d0e0f08090a0b + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha256_ni_x1(SHA256_ARGS *args, UINT32 size_in_blocks) +;; arg1 : pointer to args +;; arg2 : size (in blocks) ;; assumed to be >= 1 +mksection .text + +%define XMM_STORAGE 10*16 +%define GP_STORAGE 6*8 + +%define VARIABLE_OFFSET XMM_STORAGE + GP_STORAGE +%define GP_OFFSET XMM_STORAGE + +%macro FUNC_SAVE 0 + mov r11, rsp + sub rsp, VARIABLE_OFFSET + and rsp, ~15 ; align rsp to 16 bytes + + mov [rsp + 0*8], rbx + mov [rsp + 1*8], rbp + mov [rsp + 2*8], r12 +%ifndef LINUX + mov [rsp + 3*8], rsi + mov [rsp + 4*8], rdi + movdqa [rsp + 3*16], xmm6 + movdqa [rsp + 4*16], xmm7 + movdqa [rsp + 5*16], xmm8 + movdqa [rsp + 6*16], xmm9 + movdqa [rsp + 7*16], xmm10 + movdqa [rsp + 8*16], xmm11 + movdqa [rsp + 9*16], xmm12 + movdqa [rsp + 10*16], xmm13 + movdqa [rsp + 11*16], xmm14 + movdqa [rsp + 12*16], xmm15 +%endif ; LINUX + mov [rsp + 5*8], r11 ;; rsp pointer +%endmacro + +%macro FUNC_RESTORE 0 + mov rbx, [rsp + 0*8] + mov rbp, [rsp + 1*8] + mov r12, [rsp + 2*8] +%ifndef LINUX + mov rsi, [rsp + 3*8] + mov rdi, [rsp + 4*8] + movdqa xmm6, [rsp + 3*16] + movdqa xmm7, [rsp + 4*16] + movdqa xmm8, [rsp + 5*16] + movdqa xmm9, [rsp + 6*16] + movdqa xmm10, [rsp + 7*16] + movdqa xmm11, [rsp + 8*16] + movdqa xmm12, [rsp + 9*16] + movdqa xmm13, [rsp + 10*16] + movdqa xmm14, [rsp + 11*16] + movdqa xmm15, [rsp + 12*16] + +%ifdef SAFE_DATA + pxor xmm5, xmm5 + movdqa [rsp + 3*16], xmm5 + movdqa [rsp + 4*16], xmm5 + movdqa [rsp + 5*16], xmm5 + movdqa [rsp + 6*16], xmm5 + movdqa [rsp + 7*16], xmm5 + movdqa [rsp + 8*16], xmm5 + movdqa [rsp + 9*16], xmm5 + movdqa [rsp + 10*16], xmm5 + movdqa [rsp + 11*16], xmm5 + movdqa [rsp + 12*16], xmm5 + +%endif +%endif ; LINUX + mov rsp, [rsp + 5*8] ;; rsp pointer +%endmacro + +MKGLOBAL(sha256_ni_x1,function,internal) +align 32 +sha256_ni_x1: + sub rsp, frame_size + + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + + ;; load input pointers + mov INP, [args + _data_ptr_sha256 + lane*PTR_SZ] + + add NUM_BLKS, INP ; pointer to end of data + + ;; load initial digest + ;; Probably need to reorder these appropriately + ;; DCBA, HGFE -> ABEF, CDGH + shl lane, 5 + movdqu STATE0, [args + lane] + movdqu STATE1, [args + lane + 16] + + pshufd STATE0, STATE0, 0xB1 ; CDAB + pshufd STATE1, STATE1, 0x1B ; EFGH + movdqa MSGTMP4, STATE0 + palignr STATE0, STATE1, 8 ; ABEF + pblendw STATE1, MSGTMP4, 0xF0 ; CDGH + + lea SHA256CONSTANTS,[rel K256] + movdqa SHUF_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK] + +.loop0: + ;; Save digests + movdqa [rsp + frame.ABEF_SAVE], STATE0 + movdqa [rsp + frame.CDGH_SAVE], STATE1 + + ;; Rounds 0-3 + movdqu MSG, [INP + 0*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP0, MSG + paddd MSG, [SHA256CONSTANTS + 0*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 4-7 + movdqu MSG, [INP + 1*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP1, MSG + paddd MSG, [SHA256CONSTANTS + 1*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 8-11 + movdqu MSG, [INP + 2*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP2, MSG + paddd MSG, [SHA256CONSTANTS + 2*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 12-15 + movdqu MSG, [INP + 3*16] + pshufb MSG, SHUF_MASK + movdqa MSGTMP3, MSG + paddd MSG, [SHA256CONSTANTS + 3*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 16-19 + movdqa MSG, MSGTMP0 + paddd MSG, [SHA256CONSTANTS + 4*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 20-23 + movdqa MSG, MSGTMP1 + paddd MSG, [SHA256CONSTANTS + 5*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 24-27 + movdqa MSG, MSGTMP2 + paddd MSG, [SHA256CONSTANTS + 6*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 28-31 + movdqa MSG, MSGTMP3 + paddd MSG, [SHA256CONSTANTS + 7*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 32-35 + movdqa MSG, MSGTMP0 + paddd MSG, [SHA256CONSTANTS + 8*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 36-39 + movdqa MSG, MSGTMP1 + paddd MSG, [SHA256CONSTANTS + 9*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP0, MSGTMP1 + + ;; Rounds 40-43 + movdqa MSG, MSGTMP2 + paddd MSG, [SHA256CONSTANTS + 10*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP1, MSGTMP2 + + ;; Rounds 44-47 + movdqa MSG, MSGTMP3 + paddd MSG, [SHA256CONSTANTS + 11*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP3 + palignr MSGTMP, MSGTMP2, 4 + paddd MSGTMP0, MSGTMP + sha256msg2 MSGTMP0, MSGTMP3 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP2, MSGTMP3 + + ;; Rounds 48-51 + movdqa MSG, MSGTMP0 + paddd MSG, [SHA256CONSTANTS + 12*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP0 + palignr MSGTMP, MSGTMP3, 4 + paddd MSGTMP1, MSGTMP + sha256msg2 MSGTMP1, MSGTMP0 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + sha256msg1 MSGTMP3, MSGTMP0 + + ;; Rounds 52-55 + movdqa MSG, MSGTMP1 + paddd MSG, [SHA256CONSTANTS + 13*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP1 + palignr MSGTMP, MSGTMP0, 4 + paddd MSGTMP2, MSGTMP + sha256msg2 MSGTMP2, MSGTMP1 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 56-59 + movdqa MSG, MSGTMP2 + paddd MSG, [SHA256CONSTANTS + 14*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + movdqa MSGTMP, MSGTMP2 + palignr MSGTMP, MSGTMP1, 4 + paddd MSGTMP3, MSGTMP + sha256msg2 MSGTMP3, MSGTMP2 + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + ;; Rounds 60-63 + movdqa MSG, MSGTMP3 + paddd MSG, [SHA256CONSTANTS + 15*16] + sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument + pshufd MSG, MSG, 0x0E + sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument + + paddd STATE0, [rsp + frame.ABEF_SAVE] + paddd STATE1, [rsp + frame.CDGH_SAVE] + + add INP, 64 + cmp INP, NUM_BLKS + jne .loop0 + + ;; update data pointers + mov [args + _data_ptr_sha256 + lane*PTR_SZ], INP + + ; Reorder for writeback + pshufd STATE0, STATE0, 0x1B ; FEBA + pshufd STATE1, STATE1, 0xB1 ; DCHG + movdqa MSGTMP4, STATE0 + pblendw STATE0, STATE1, 0xF0 ; DCBA + palignr STATE1, MSGTMP4, 8 ; HGFE + + ;; update digests + movdqu [args + lane + 0*16], STATE0 + movdqu [args + lane + 1*16], STATE1 + shr lane, 5 + +done_hash: + + ;; Clear stack frame (4*16 bytes) +%ifdef SAFE_DATA + clear_all_xmms_sse_asm + movdqa [rsp + frame.ABEF_SAVE], xmm0 + movdqa [rsp + frame.CDGH_SAVE], xmm0 +%endif + + add rsp, frame_size + ret + +; void call_sha256_ni_x1_sse_from_c(SHA256_ARGS *args, UINT32 size_in_blocks); +MKGLOBAL(call_sha256_ni_x1_sse_from_c,function,internal) +call_sha256_ni_x1_sse_from_c: + FUNC_SAVE + call sha256_ni_x1 + FUNC_RESTORE + ret + +mksection stack-noexec diff --git a/lib/sse_t2/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm index 91564561..e80ae407 100644 --- a/lib/sse_t2/sha256_ni_x2_sse.asm +++ b/lib/sse_t2/sha256_ni_x2_sse.asm @@ -96,25 +96,10 @@ endstruc mksection .rodata default rel -align 64 -K256: - dd 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 - dd 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 - dd 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 - dd 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 - dd 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc - dd 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da - dd 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 - dd 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 - dd 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 - dd 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 - dd 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 - dd 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 - dd 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 - dd 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 - dd 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 - dd 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +extern K256 + +align 16 PSHUFFLE_BYTE_FLIP_MASK: dq 0x0405060700010203, 0x0c0d0e0f08090a0b diff --git a/lib/win_x64.mak b/lib/win_x64.mak index f755978b..2febad47 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -232,6 +232,7 @@ lib_objs1 = \ $(OBJ_DIR)\sha256_one_block_avx.obj \ $(OBJ_DIR)\sha256_one_block_sse.obj \ $(OBJ_DIR)\sha256_ni_x2_sse.obj \ + $(OBJ_DIR)\sha256_ni_x1_sse.obj \ $(OBJ_DIR)\sha256_x16_avx512.obj \ $(OBJ_DIR)\sha384_one_block_avx.obj \ $(OBJ_DIR)\sha384_one_block_sse.obj \ -- GitLab From 499f98fe0b93150eb6df9846661220fc66dc88d3 Mon Sep 17 00:00:00 2001 From: De Lara Guarch Date: Mon, 29 Aug 2022 14:51:29 +0100 Subject: [PATCH 316/369] lib: [CHACHA20-POLY1305] preserve XMM registers on Windows Windows requires XMM6-XMM15 to be preserved, but none of the Chacha20/Poly1305 were doing it. --- lib/avx/chacha20_avx.asm | 48 ++++++++++ lib/avx2_t1/chacha20_avx2.asm | 39 ++++++++- lib/avx512_t1/chacha20_avx512.asm | 140 ++++++++++++++++++++++++------ lib/avx512_t1/poly_avx512.asm | 29 +++++-- lib/avx512_t2/poly_fma_avx512.asm | 29 +++++-- lib/sse_t1/chacha20_sse.asm | 97 +++++++++++++++++++-- 6 files changed, 332 insertions(+), 50 deletions(-) diff --git a/lib/avx/chacha20_avx.asm b/lib/avx/chacha20_avx.asm index 73c4b51a..ef0bacff 100644 --- a/lib/avx/chacha20_avx.asm +++ b/lib/avx/chacha20_avx.asm @@ -85,6 +85,7 @@ dq 0x0ffffffc0fffffff, 0x0ffffffc0ffffffc struc STACK _STATE: reso 16 ; Space to store first 4 states _XMM_SAVE: reso 2 ; Space to store up to 2 temporary XMM registers +_XMM_WIN_SAVE: reso 10 ; Space to store up to 10 XMM registers _GP_SAVE: resq 7 ; Space to store up to 7 GP registers _RSP_SAVE: resq 1 ; Space to store rsp pointer endstruc @@ -619,6 +620,15 @@ submit_job_chacha20_enc_dec_avx: mov rax, rsp sub rsp, STACK_SIZE and rsp, -16 +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov [rsp + _RSP_SAVE], rax ; save RSP xor off, off @@ -1140,6 +1150,15 @@ no_partial_block: vmovdqa [rsp + _XMM_SAVE + 16], xmm0 %endif +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov rsp, [rsp + _RSP_SAVE] exit: @@ -1188,6 +1207,13 @@ chacha20_enc_dec_ks_avx: mov [rsp + _GP_SAVE + 40], rbp %ifndef LINUX mov [rsp + _GP_SAVE + 48], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov [rsp + _RSP_SAVE], rax ; save RSP @@ -1703,6 +1729,13 @@ exit_ks: mov rbp, [rsp + _GP_SAVE + 40] %ifndef LINUX mov rdi, [rsp + _GP_SAVE + 48] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _RSP_SAVE]; restore RSP @@ -1712,6 +1745,15 @@ exit_ks: align 32 MKGLOBAL(poly1305_key_gen_avx,function,internal) poly1305_key_gen_avx: +%ifndef LINUX + mov rax, rsp + sub rsp, 3*16 + 8 + and rsp, -16 + vmovdqa [rsp], xmm6 + vmovdqa [rsp + 16], xmm7 + vmovdqa [rsp + 16*2], xmm8 + mov [rsp + 16*3], rax +%endif ;; prepare chacha state from IV, key vmovdqa xmm0, [rel constants] vmovdqu xmm1, [arg1] ; Load key bytes 0-15 @@ -1732,6 +1774,12 @@ poly1305_key_gen_avx: %ifdef SAFE_DATA clear_all_xmms_avx_asm +%endif +%ifndef LINUX + vmovdqa xmm6, [rsp] + vmovdqa xmm7, [rsp + 16] + vmovdqa xmm8, [rsp + 16*2] + mov rsp, [rsp + 16*3] %endif ret diff --git a/lib/avx2_t1/chacha20_avx2.asm b/lib/avx2_t1/chacha20_avx2.asm index 2b3b4680..9e11bf81 100644 --- a/lib/avx2_t1/chacha20_avx2.asm +++ b/lib/avx2_t1/chacha20_avx2.asm @@ -81,6 +81,7 @@ db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 struc STACK _STATE: reso 32 ; Space to store first 8 states _YMM_SAVE: resy 2 ; Space to store up to 2 temporary YMM registers +_XMM_WIN_SAVE: reso 10 ; Space to store up to 10 XMM registers _GP_SAVE: resq 7 ; Space to store up to 7 GP registers _RSP_SAVE: resq 1 ; Space to store rsp pointer endstruc @@ -582,6 +583,15 @@ submit_job_chacha20_enc_dec_avx2: mov rax, rsp sub rsp, STACK_SIZE and rsp, -32 +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov [rsp + _RSP_SAVE], rax ; save RSP xor off, off @@ -917,7 +927,7 @@ partial_block: no_partial_block: endbranch64 %ifdef SAFE_DATA - vpxor ymm0, ymm0 + clear_all_ymms_asm ; Clear stack frame %assign i 0 %rep 16 @@ -928,12 +938,20 @@ no_partial_block: vmovdqa [rsp + _YMM_SAVE + 32], ymm0 %endif +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov rsp, [rsp + _RSP_SAVE] exit: mov rax, job or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER - clear_all_ymms_asm ret @@ -981,6 +999,13 @@ chacha20_enc_dec_ks_avx2: mov [rsp + _GP_SAVE + 40], rbp %ifndef LINUX mov [rsp + _GP_SAVE + 48], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov [rsp + _RSP_SAVE], rax ; save RSP @@ -1353,7 +1378,7 @@ no_partial_block_ks: mov [ctx + LastBlkCount], blk_cnt %ifdef SAFE_DATA - vpxor ymm0, ymm0 + clear_all_ymms_asm ; Clear stack frame %assign i 0 %rep 16 @@ -1372,11 +1397,17 @@ no_partial_block_ks: mov rbp, [rsp + _GP_SAVE + 40] %ifndef LINUX mov rdi, [rsp + _GP_SAVE + 48] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _RSP_SAVE] exit_ks: - clear_all_ymms_asm ret diff --git a/lib/avx512_t1/chacha20_avx512.asm b/lib/avx512_t1/chacha20_avx512.asm index d2274187..845e531d 100644 --- a/lib/avx512_t1/chacha20_avx512.asm +++ b/lib/avx512_t1/chacha20_avx512.asm @@ -130,6 +130,13 @@ poly_clamp_r: dq 0x0ffffffc0fffffff, 0x0ffffffc0ffffffc dq 0xffffffffffffffff, 0xffffffffffffffff +struc STACK +_XMM_WIN_SAVE: reso 10 ; Space to store up to 10 XMM registers +_GP_SAVE: resq 7 ; Space to store up to 7 GP registers +_RSP_SAVE: resq 1 ; Space to store rsp pointer +endstruc +%define STACK_SIZE STACK_size + %define APPEND(a,b) a %+ b %define APPEND3(a,b,c) a %+ b %+ c @@ -1302,10 +1309,22 @@ submit_job_chacha20_poly_enc_avx512: %define tmp r13 %define off rax - sub rsp, 16 - mov [rsp], r12 - mov [rsp + 8], r13 + mov rax, rsp + sub rsp, STACK_SIZE + and rsp, -16 + mov [rsp + _GP_SAVE], r12 + mov [rsp + _GP_SAVE + 8], r13 +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif + mov [rsp + _RSP_SAVE], rax ; save RSP mov added_len, 64 xor off, off @@ -1554,9 +1573,18 @@ no_partial_block_poly: mov rax, job or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER - mov r12, [rsp] - mov r13, [rsp + 8] - add rsp, 16 + mov r12, [rsp + _GP_SAVE] + mov r13, [rsp + _GP_SAVE + 8] +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif + mov rsp, [rsp + _RSP_SAVE] ret @@ -1571,6 +1599,19 @@ gen_keystr_poly_key_avx512: %define off rax +%ifndef LINUX + mov rax, rsp + sub rsp, STACK_SIZE + and rsp, -16 +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep + mov [rsp + _RSP_SAVE], rax ; save RSP +%endif ; Generate up to 1KB of keystream ; If less than or equal to 64*8 bytes, prepare directly states for up to 8 blocks @@ -1624,7 +1665,7 @@ gen_keystr_poly_key_avx512: vmovdqa64 [ks + 64*14], zmm30 vmovdqa64 [ks + 64*15], zmm18 - ret + jmp exit_gen_keystr less_than_512_ks: @@ -1648,7 +1689,7 @@ less_than_512_ks: vmovdqa64 [ks + 64*2], zmm17 vmovdqa64 [ks + 64*3], zmm29 - ret + jmp exit_gen_keystr more_than_256_ks: xor off, off @@ -1672,6 +1713,17 @@ more_than_256_ks: vmovdqa64 [ks + 64*6], zmm26 vmovdqa64 [ks + 64*7], zmm23 +exit_gen_keystr: +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep + mov rsp, [rsp + _RSP_SAVE] +%endif ret align 32 @@ -1694,7 +1746,18 @@ submit_job_chacha20_poly_dec_avx512: %define len_xor iv %ifndef LINUX - push rsi + mov rax, rsp + sub rsp, STACK_SIZE + and rsp, -16 + mov [rsp + _GP_SAVE], rsi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep + mov [rsp + _RSP_SAVE], rax ; save RSP %endif mov len_xor, arg3 @@ -1979,7 +2042,15 @@ no_partial_block_dec: or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER %ifndef LINUX - pop rsi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep + mov rsi, [rsp + _GP_SAVE] + mov rsp, [rsp + _RSP_SAVE] %endif ret @@ -2016,16 +2087,26 @@ chacha20_enc_dec_ks_avx512: mov ctx, arg5 - sub rsp, 8*7 - mov [rsp], r12 - mov [rsp + 8], r13 - mov [rsp + 16], r14 - mov [rsp + 24], r15 - mov [rsp + 32], rbx - mov [rsp + 40], rbp + mov rax, rsp + sub rsp, STACK_SIZE + and rsp, -16 + mov [rsp + _GP_SAVE], r12 + mov [rsp + _GP_SAVE + 8], r13 + mov [rsp + _GP_SAVE + 16], r14 + mov [rsp + _GP_SAVE + 24], r15 + mov [rsp + _GP_SAVE + 32], rbx + mov [rsp + _GP_SAVE + 40], rbp %ifndef LINUX - mov [rsp + 48], rdi + mov [rsp + _GP_SAVE + 48], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif + mov [rsp + _RSP_SAVE], rax ; save RSP xor off, off mov blk_cnt, [ctx + LastBlkCount] @@ -2255,16 +2336,23 @@ no_partial_block_ks: mov [ctx + LastBlkCount], blk_cnt - mov r12, [rsp] - mov r13, [rsp + 8] - mov r14, [rsp + 16] - mov r15, [rsp + 24] - mov rbx, [rsp + 32] - mov rbp, [rsp + 40] + mov r12, [rsp + _GP_SAVE] + mov r13, [rsp + _GP_SAVE + 8] + mov r14, [rsp + _GP_SAVE + 16] + mov r15, [rsp + _GP_SAVE + 24] + mov rbx, [rsp + _GP_SAVE + 32] + mov rbp, [rsp + _GP_SAVE + 40] %ifndef LINUX - mov rdi, [rsp + 48] + mov rdi, [rsp + _GP_SAVE + 48] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif - add rsp, 8*7 + mov rsp, [rsp + _RSP_SAVE]; restore RSP %ifdef SAFE_DATA clear_all_zmms_asm %else diff --git a/lib/avx512_t1/poly_avx512.asm b/lib/avx512_t1/poly_avx512.asm index f5c5f3ff..400dade9 100644 --- a/lib/avx512_t1/poly_avx512.asm +++ b/lib/avx512_t1/poly_avx512.asm @@ -217,6 +217,7 @@ dw 0, 0x1, 0x5, 0x15, 0x55, 0x57, 0x5f, 0x7f, 0xff struc STACKFRAME _r_save: resq 16 ; Memory to save limbs of powers of R _rp_save: resq 8 ; Memory to save limbs of powers of R' +_xmm_save: reso 10 ; Memory to save XMM registers _gpr_save: resq 8 ; Memory to save GP registers _rsp_save: resq 1 ; Memory to save RSP pointer endstruc @@ -1298,6 +1299,13 @@ APPEND(%%_shuffle_blocks_, i): %ifndef LINUX mov [rsp + _gpr_save + 8*6], rsi mov [rsp + _gpr_save + 8*7], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _xmm_save + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov [rsp + _rsp_save], rax @@ -1308,6 +1316,13 @@ APPEND(%%_shuffle_blocks_, i): ;; Restores registers and removes the stack frame ;; ============================================================================= %macro FUNC_EXIT 0 +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -1317,16 +1332,16 @@ APPEND(%%_shuffle_blocks_, i): %ifndef LINUX mov rsi, [rsp + _gpr_save + 8*6] mov rdi, [rsp + _gpr_save + 8*7] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _xmm_save + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _rsp_save] -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - %endmacro ;; ============================================================================= diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index 6da2e441..f0df908e 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -166,6 +166,7 @@ dw 0, 0x1, 0x5, 0x15, 0x55, 0x57, 0x5f, 0x7f, 0xff struc STACKFRAME _r_save: resz 6 ; Memory to save limbs of powers of R _gpr_save: resq 8 ; Memory to save GP registers +_xmm_save: reso 10 ; Memory to save XMM registers _rsp_save: resq 1 ; Memory to save RSP endstruc @@ -1643,6 +1644,13 @@ APPEND(%%_shuffle_blocks_, i): %ifndef LINUX mov [rsp + _gpr_save + 8*6], rsi mov [rsp + _gpr_save + 8*7], rdi +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa [rsp + _xmm_save + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov [rsp + _rsp_save], rax @@ -1653,6 +1661,13 @@ APPEND(%%_shuffle_blocks_, i): ;; Restores registers and removes the stack frame ;; ============================================================================= %macro FUNC_EXIT 0 +%ifdef SAFE_DATA + clear_scratch_gps_asm + clear_all_zmms_asm +%else + vzeroupper +%endif ;; SAFE_DATA + mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] @@ -1662,16 +1677,16 @@ APPEND(%%_shuffle_blocks_, i): %ifndef LINUX mov rsi, [rsp + _gpr_save + 8*6] mov rdi, [rsp + _gpr_save + 8*7] +%assign i 0 +%assign j 6 +%rep 10 + vmovdqa APPEND(xmm, j), [rsp + _xmm_save + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _rsp_save] -%ifdef SAFE_DATA - clear_scratch_gps_asm - clear_all_zmms_asm -%else - vzeroupper -%endif ;; SAFE_DATA - %endmacro ;; ============================================================================= diff --git a/lib/sse_t1/chacha20_sse.asm b/lib/sse_t1/chacha20_sse.asm index c2229755..253b17dc 100644 --- a/lib/sse_t1/chacha20_sse.asm +++ b/lib/sse_t1/chacha20_sse.asm @@ -93,6 +93,7 @@ dq 0x0ffffffc0fffffff, 0x0ffffffc0ffffffc struc STACK _STATE: reso 16 ; Space to store first 4 states _XMM_SAVE: reso 2 ; Space to store up to 2 temporary XMM registers +_XMM_WIN_SAVE: reso 10 ; Space to store up to 10 XMM registers _GP_SAVE: resq 7 ; Space to store up to 7 GP registers _RSP_SAVE: resq 1 ; Space to store rsp pointer endstruc @@ -1109,7 +1110,15 @@ chacha20_enc_dec_ks_sse: mov [rsp + _GP_SAVE + 40], rbp %ifndef LINUX mov [rsp + _GP_SAVE + 48], rdi +%assign i 0 +%assign j 6 +%rep 10 + movdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif + mov [rsp + _RSP_SAVE], rax ; save RSP ; Check if there is nothing to encrypt @@ -1548,6 +1557,13 @@ exit_ks: mov rbp, [rsp + _GP_SAVE + 40] %ifndef LINUX mov rdi, [rsp + _GP_SAVE + 48] +%assign i 0 +%assign j 6 +%rep 10 + movdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _RSP_SAVE]; restore RSP @@ -1558,6 +1574,16 @@ exit_ks: align 32 MKGLOBAL(poly1305_key_gen_sse,function,internal) poly1305_key_gen_sse: + +%ifndef LINUX + mov rax, rsp + sub rsp, 3*16 + 8 + and rsp, -16 + movdqa [rsp], xmm6 + movdqa [rsp + 16], xmm7 + movdqa [rsp + 16*2], xmm8 + mov [rsp + 16*3], rax +%endif ;; prepare chacha state from IV, key movdqa xmm0, [rel constants] movdqu xmm1, [arg1] ; Load key bytes 0-15 @@ -1580,6 +1606,12 @@ poly1305_key_gen_sse: clear_all_xmms_sse_asm %endif +%ifndef LINUX + movdqa xmm6, [rsp] + movdqa xmm7, [rsp + 16] + movdqa xmm8, [rsp + 16*2] + mov rsp, [rsp + 16*3] +%endif ret align 32 @@ -1601,6 +1633,15 @@ submit_job_chacha20_poly_enc_sse: mov [rsp + _GP_SAVE], r12 mov [rsp + _GP_SAVE + 8], r13 mov [rsp + _GP_SAVE + 16], r14 +;%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + movdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +;%endif mov [rsp + _RSP_SAVE], rax ; save RSP mov added_len, 64 @@ -2160,6 +2201,15 @@ no_partial_block_poly: mov r12, [rsp + _GP_SAVE] mov r13, [rsp + _GP_SAVE + 8] mov r14, [rsp + _GP_SAVE + 16] +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + movdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov rsp, [rsp + _RSP_SAVE] mov rax, job @@ -2193,6 +2243,13 @@ submit_job_chacha20_poly_dec_sse: mov [rsp + _GP_SAVE + 8], r13 %ifndef LINUX mov [rsp + _GP_SAVE + 16], rsi +%assign i 0 +%assign j 6 +%rep 10 + movdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov [rsp + _RSP_SAVE], rax ; save RSP @@ -2631,6 +2688,15 @@ no_partial_block_dec: mov r13, [rsp + _GP_SAVE + 8] %ifndef LINUX mov rsi, [rsp + _GP_SAVE + 16] +%endif +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + movdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep %endif mov rsp, [rsp + _RSP_SAVE] ret @@ -2643,16 +2709,25 @@ gen_keystr_poly_key_sse: %define len arg3 %define ks arg4 - ; If less than or equal to 64*2 bytes, prepare directly states for - ; up to 2 blocks - cmp len, 64*2 - jbe check_1_or_2_blocks_left_gen - mov rax, rsp sub rsp, STACK_SIZE and rsp, -16 +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + movdqa [rsp + _XMM_WIN_SAVE + i*16], APPEND(xmm, j) +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov [rsp + _RSP_SAVE], rax ; save RSP + ; If less than or equal to 64*2 bytes, prepare directly states for + ; up to 2 blocks + cmp len, 64*2 + jbe check_1_or_2_blocks_left_gen + ; Prepare first 4 chacha states movdqa xmm0, [rel constants0] movdqa xmm1, [rel constants1] @@ -2758,6 +2833,16 @@ gen_keystr_poly_key_sse: movdqa [rsp + _XMM_SAVE + 16], xmm0 %endif +restore_gen_keystr: +%ifndef LINUX +%assign i 0 +%assign j 6 +%rep 10 + movdqa APPEND(xmm, j), [rsp + _XMM_WIN_SAVE + i*16] +%assign i (i + 1) +%assign j (j + 1) +%endrep +%endif mov rsp, [rsp + _RSP_SAVE] ret @@ -2833,6 +2918,6 @@ exit_gen: %ifdef SAFE_DATA clear_all_xmms_sse_asm %endif - ret + jmp restore_gen_keystr mksection stack-noexec -- GitLab From 556e1a54dd2c39f282a105f87f097f0d4f5eca67 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 30 Aug 2022 13:30:13 +0000 Subject: [PATCH 317/369] test: [ACVP] rename SHAx handler functions --- test/acvp_app_main.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index a808fa6e..628fdd57 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -587,7 +587,7 @@ static int aes_cmac_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_hmac_sha1_handler(ACVP_TEST_CASE *test_case) +static int hmac_sha1_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; @@ -653,7 +653,7 @@ static int aes_hmac_sha1_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_hmac_sha256_handler(ACVP_TEST_CASE *test_case) +static int hmac_sha256_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; @@ -719,7 +719,7 @@ static int aes_hmac_sha256_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_hmac_sha224_handler(ACVP_TEST_CASE *test_case) +static int hmac_sha224_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; @@ -785,7 +785,7 @@ static int aes_hmac_sha224_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_hmac_sha384_handler(ACVP_TEST_CASE *test_case) +static int hmac_sha384_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; @@ -851,7 +851,7 @@ static int aes_hmac_sha384_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_hmac_sha512_handler(ACVP_TEST_CASE *test_case) +static int hmac_sha512_handler(ACVP_TEST_CASE *test_case) { ACVP_HMAC_TC *tc; IMB_JOB *job = NULL; @@ -917,7 +917,7 @@ static int aes_hmac_sha512_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_sha1_handler(ACVP_TEST_CASE *test_case) +static int sha1_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; IMB_JOB *job = NULL; @@ -975,7 +975,7 @@ static int aes_sha1_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_sha2_224_handler(ACVP_TEST_CASE *test_case) +static int sha2_224_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; IMB_JOB *job = NULL; @@ -1033,7 +1033,7 @@ static int aes_sha2_224_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_sha2_256_handler(ACVP_TEST_CASE *test_case) +static int sha2_256_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; IMB_JOB *job = NULL; @@ -1091,7 +1091,7 @@ static int aes_sha2_256_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_sha2_384_handler(ACVP_TEST_CASE *test_case) +static int sha2_384_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; IMB_JOB *job = NULL; @@ -1149,7 +1149,7 @@ static int aes_sha2_384_handler(ACVP_TEST_CASE *test_case) return EXIT_SUCCESS; } -static int aes_sha2_512_handler(ACVP_TEST_CASE *test_case) +static int sha2_512_handler(ACVP_TEST_CASE *test_case) { ACVP_HASH_TC *tc; IMB_JOB *job = NULL; @@ -1331,43 +1331,43 @@ int main(int argc, char **argv) goto exit; if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA1, - &aes_hmac_sha1_handler) != ACVP_SUCCESS) + &hmac_sha1_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_256, - &aes_hmac_sha256_handler) != ACVP_SUCCESS) + &hmac_sha256_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_224, - &aes_hmac_sha224_handler) != ACVP_SUCCESS) + &hmac_sha224_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_384, - &aes_hmac_sha384_handler) != ACVP_SUCCESS) + &hmac_sha384_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hmac_enable(ctx, ACVP_HMAC_SHA2_512, - &aes_hmac_sha512_handler) != ACVP_SUCCESS) + &hmac_sha512_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA1, - &aes_sha1_handler) != ACVP_SUCCESS) + &sha1_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA224, - &aes_sha2_224_handler) != ACVP_SUCCESS) + &sha2_224_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA256, - &aes_sha2_256_handler) != ACVP_SUCCESS) + &sha2_256_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA384, - &aes_sha2_384_handler) != ACVP_SUCCESS) + &sha2_384_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_hash_enable(ctx, ACVP_HASH_SHA512, - &aes_sha2_512_handler) != ACVP_SUCCESS) + &sha2_512_handler) != ACVP_SUCCESS) goto exit; /* Allocate and initialize MB_MGR */ -- GitLab From 9ee9dbf8fbdc41cc0828d2347a0c9a739cae5bd2 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 30 Aug 2022 13:30:52 +0000 Subject: [PATCH 318/369] test: [ACVP] support for AES-GMAC --- test/acvp_app_main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 628fdd57..86c98c34 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1321,6 +1321,9 @@ int main(int argc, char **argv) if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CTR, &aes_ctr_handler) != ACVP_SUCCESS) goto exit; + if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_GMAC, + &aes_gcm_handler) != ACVP_SUCCESS) + goto exit; if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CCM, &aes_ccm_handler) != ACVP_SUCCESS) -- GitLab From b1314b4ea0f51351c133573e4674d0e1600b4222 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 2 Sep 2022 11:20:08 +0000 Subject: [PATCH 319/369] doc: release notes update --- ReleaseNotes.txt | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index b35dedd7..55e68064 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -7,30 +7,47 @@ Unreleased Library - ZUC-EIA3-256 8-byte and 16-byte tag support added for SSE, AVX, AVX2 and AVX512 - AES-ECB AVX512-VAES implementation added -- JOB API GHASH support added - AES-ECB optimizations for AVX and SSE - AES-ECB AVX2-VAES implementation added +- JOB API GHASH support added - SHA1/224/256/384/512 multi-buffer implementation added -- Burst API added -- Cipher-only burst API added (AES-CBC/CTR support only) -- Hash-only burst API added (HMAC-SHA1/224/256/384/512 support only) +- Multi-buffer SHA1, SHA224 and SHA256 use SHANI if available +- Synchronous cipher and hash burst API added + - cipher API only supports AES-CBC and AES-CTR + - hash API only supports HMAC-SHA1, HMAC-224, HMAC-256, HMAC-384 and HMAC-512 +- Asynchronous burst API added that supports all cipher and hash modes - SNOW3G-UEA2 SSE multi-buffer implementation added -- SNOW3G-UIA2 SSE multi-buffer initialization and keystream generation added - -Fixes -- Fixed 23-byte IV expansion for ZUC-256 (intel/intel-ipsec-mb#102) +- SNOW3G-UIA2 SSE multi-buffer initialization and key-stream generation added +- API documentation added (doxygen generated) +- New SGL job API (AES-GCM and CHACHA20-POLY1305 only) +- Enforced EVEX PMADD52 encoding in AVX512 code +- Restructured reset flow of architecture managers +- SSE, AVX, AVX2 and AVX512 managers were split to better cover different types +- Added library self-test functionality +- enbranch64 not emitted on Windows builds (CET related) Test Applications - GHASH JOB API support added in the test application, fuzzing and xvalid tools - Burst API support added for supported algorithms +- ACVP test application extended to support: AES-GCM, AES-GMAC, AES-CCM, + AES-CBC, AES-CTR, AES-CMAC, SHA1, SHA224, SHA256, SHA384, SHA512, HMAC-SHA1, + HMAC-SHA224, HMAC-SHA256, HMAC-SHA384, HMAC-SHA512 +- Cross validation (xvalid) tool improvements in pattern search functionality +- FreeBSD added to github CI +- Added AVX-SSE transition check to the cross validation tool (xvalid) Performance Application - GHASH support added (through JOB and direct API) +- CHACHA20-POLY1305 support through direct API - Support added for SHA1/224/256/384/512 - Burst API support added for supported algorithms +- SGL support added (AES-GCM and CHACHA20-POLY1305 only) Fixes -- Fixed incorrect 8-buffer SNOW3G keystream generation +- Fixed 23-byte IV expansion for ZUC-256 (issue #102) +- Fixed incorrect 8-buffer SNOW3G key-stream generation (issue #104) +- Numerous AVX-SSE transition fixes with SAFE_OPTIONS=n +- [ZUC-EIA3] allow unaligned digest load/stores v1.2 February 2022 ======================================================================== -- GitLab From 6fb46b76bcf4794cd3913c0dcb1a9554f4f9d115 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 31 Aug 2022 14:59:36 +0000 Subject: [PATCH 320/369] perf: replace --gcm-job-api option with --force-job-api in ipsec_perf_tool --- perf/ipsec_perf_tool.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/perf/ipsec_perf_tool.py b/perf/ipsec_perf_tool.py index 4ef9bc0b..34d4f60d 100755 --- a/perf/ipsec_perf_tool.py +++ b/perf/ipsec_perf_tool.py @@ -60,7 +60,7 @@ class Variant: """Class to setup and run test case variant""" def __init__(self, idx=None, arch=None, direction='encrypt', cipher_alg=None, hash_alg=None, aead_alg=None, sizes=None, offset=None, - cold_cache=False, shani_off=False, gcm_job_api=False, + cold_cache=False, shani_off=False, force_job_api=False, unhalted_cycles=False, quick_test=False, smoke_test=False, imix=None, aad_size=None, job_iter=None, no_time_box=False): """Build perf app command line""" @@ -80,7 +80,7 @@ class Variant: self.core = None self.cold_cache = cold_cache self.shani_off = shani_off - self.gcm_job_api = gcm_job_api + self.force_job_api = force_job_api self.unhalted_cycles = unhalted_cycles self.quick_test = quick_test self.smoke_test = smoke_test @@ -128,8 +128,8 @@ class Variant: if self.shani_off is True: self.cmd += ' --shani-off' - if self.gcm_job_api is True: - self.cmd += ' --gcm-job-api' + if self.force_job_api is True: + self.cmd += ' --force-job-api' if self.unhalted_cycles is True: self.cmd += ' --unhalted-cycles' @@ -393,8 +393,8 @@ def parse_args(): parser.add_argument("--arch-best", action='store_true', help="detect available architectures and run only on the best one") parser.add_argument("--shani-off", action='store_true', help="don't use SHA extensions") - parser.add_argument("--gcm-job-api", action='store_true', - help="use JOB API for GCM perf tests (raw GCM API is default)") + parser.add_argument("--force-job-api", action='store_true', + help="use JOB API for algorithms supported through direct API (i.e. AES-GCM, chacha20-poly1305)") parser.add_argument("--unhalted-cycles", action='store_true', help=textwrap.dedent('''\ measure using unhalted cycles (requires root). @@ -457,7 +457,7 @@ def parse_args(): return args.arch, cores, directions, args.offset, \ alg_types, args.job_size, args.cold_cache, args.arch_best, \ - args.shani_off, args.gcm_job_api, args.unhalted_cycles, \ + args.shani_off, args.force_job_api, args.unhalted_cycles, \ args.quick, args.smoke, args.imix, \ args.aad_size, args.job_iter, args.no_time_box @@ -528,7 +528,7 @@ def main(): # parse command line args archs, cores, directions, offset, alg_types, sizes, cold_cache, arch_best, \ - shani_off, gcm_job_api, unhalted_cycles, quick_test, smoke_test, \ + shani_off, force_job_api, unhalted_cycles, quick_test, smoke_test, \ imix, aad_size, job_iter, no_time_box = parse_args() # validate requested archs are supported @@ -560,7 +560,7 @@ def main(): print(' Cores: {}'.format(cores), file=sys.stderr) print(' Cache: {}'.format("cold" if cold_cache else "warm"), file=sys.stderr) print(' SHANI: {}'.format("off" if shani_off else "on"), file=sys.stderr) - print(' GCM API: {}'.format("job" if gcm_job_api else "direct"), file=sys.stderr) + print(' API: {}'.format("job" if force_job_api else "direct"), file=sys.stderr) print(' Measuring using {}'.format("unhalted cycles" if unhalted_cycles \ else "rdtsc"), file=sys.stderr) if quick_test is True or smoke_test is True: @@ -578,7 +578,7 @@ def main(): TODO_Q.put(Variant(idx=TOTAL_VARIANTS, arch=arch, direction=direction, offset=offset, sizes=sizes, cipher_alg=cipher_alg, cold_cache=cold_cache, shani_off=shani_off, - gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, + force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 @@ -589,7 +589,7 @@ def main(): TODO_Q.put(Variant(idx=TOTAL_VARIANTS, arch=arch, direction=None, offset=offset, sizes=sizes, hash_alg=hash_alg, cold_cache=cold_cache, shani_off=shani_off, - gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, + force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 @@ -600,7 +600,7 @@ def main(): TODO_Q.put(Variant(idx=TOTAL_VARIANTS, arch=arch, direction=direction, offset=offset, sizes=sizes, aead_alg=aead_alg, cold_cache=cold_cache, shani_off=shani_off, - gcm_job_api=gcm_job_api, unhalted_cycles=unhalted_cycles, + force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) TOTAL_VARIANTS += 1 @@ -613,7 +613,7 @@ def main(): TODO_Q.put(Variant(idx=TOTAL_VARIANTS, arch=arch, direction=direction, offset=offset, sizes=sizes, cipher_alg=cipher_alg, hash_alg=hash_alg, cold_cache=cold_cache, - shani_off=shani_off, gcm_job_api=gcm_job_api, + shani_off=shani_off, force_job_api=force_job_api, unhalted_cycles=unhalted_cycles, quick_test=quick_test, smoke_test=smoke_test, imix=imix, aad_size=aad_size, job_iter=job_iter, no_time_box=no_time_box)) -- GitLab From db12a5e58019be19b70d7216e9dfa42b45672e10 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 23 Aug 2022 16:05:01 +0000 Subject: [PATCH 321/369] sse: [MB_MGR] split into type-1, type-2 and type-3 manager implementations - get rid of the stubs and multiple checks for available CPU features - produce different code paths type-1, type-2 and type-3 - create SSE JOB API interface that hides differences behind the types - this is only needed for cases where legacy apps use sse JOB API directly --- lib/Makefile | 3 + lib/include/arch_sse_type1.h | 7 +- lib/include/ipsec_ooo_mgr.h | 117 +++++ lib/include/mb_mgr_code.h | 79 --- lib/intel-ipsec-mb.h | 2 + lib/sse_t1/mb_mgr_sse.c | 932 ++--------------------------------- lib/sse_t1/mb_mgr_sse_t1.c | 652 ++++++++++++++++++++++++ lib/sse_t2/mb_mgr_sse_t2.c | 649 ++++++++++++++++++++++++ lib/sse_t3/mb_mgr_sse_t3.c | 650 ++++++++++++++++++++++++ lib/win_x64.mak | 3 + 10 files changed, 2118 insertions(+), 976 deletions(-) create mode 100644 lib/sse_t1/mb_mgr_sse_t1.c create mode 100644 lib/sse_t2/mb_mgr_sse_t2.c create mode 100644 lib/sse_t3/mb_mgr_sse_t3.c diff --git a/lib/Makefile b/lib/Makefile index 5f5d299b..c3db3749 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -252,6 +252,9 @@ c_lib_objs := \ mb_mgr_avx512_t1.o \ mb_mgr_avx512_t2.o \ mb_mgr_sse.o \ + mb_mgr_sse_t1.o \ + mb_mgr_sse_t2.o \ + mb_mgr_sse_t3.o \ alloc.o \ aes_xcbc_expand_key.o \ md5_one_block.o \ diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index 941e9991..eeb0d895 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -111,7 +111,8 @@ uint32_t crc10_iuup_data_sse(const void *msg, const uint64_t len); uint32_t crc6_iuup_header_sse(const void *msg, const uint64_t len); uint32_t crc32_wimax_ofdma_data_sse(const void *msg, const uint64_t len); uint32_t crc8_wimax_ofdma_hcs_sse(const void *msg, const uint64_t len); - +uint32_t ethernet_fcs_sse_local(const void *msg, const uint64_t len, + const void *tag_ouput); /* moved from MB MGR */ IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, @@ -175,10 +176,6 @@ IMB_JOB *flush_job_aes128_ccm_auth_sse(MB_MGR_CCM_OOO *state); IMB_JOB *flush_job_aes256_ccm_auth_sse(MB_MGR_CCM_OOO *state); -IMB_JOB *submit_job_aes_cntr_sse(IMB_JOB *job); - -IMB_JOB *submit_job_aes_cntr_bit_sse(IMB_JOB *job); - IMB_JOB *submit_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state, IMB_JOB *job); IMB_JOB *flush_job_zuc_eea3_no_gfni_sse(MB_MGR_ZUC_OOO *state); diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index ee832af6..30df925c 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -416,6 +416,12 @@ init_mb_mgr_sse_no_aesni_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void +init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void +init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void +init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); @@ -429,6 +435,12 @@ init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_EXPORT uint32_t get_next_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +get_next_burst_sse_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +get_next_burst_sse_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +get_next_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t get_next_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -440,6 +452,12 @@ get_next_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_sse_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_sse_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -452,6 +470,15 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse_t1(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse_t2(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_sse_t3(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -467,6 +494,12 @@ submit_burst_nocheck_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_DLL_EXPORT uint32_t flush_burst_sse(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +flush_burst_sse_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_sse_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_sse_t3(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t flush_burst_avx(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); @@ -482,6 +515,24 @@ submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_sse_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -513,6 +564,24 @@ submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_sse_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -541,6 +610,18 @@ submit_hash_burst_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_sse_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_sse_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_sse_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -561,6 +642,18 @@ submit_hash_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_sse_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_sse_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_sse_t3(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -577,6 +670,30 @@ submit_hash_burst_nocheck_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); +/* SSE TYPE1 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_sse_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_sse_t1(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_sse_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_t1(IMB_MGR *state); + +/* SSE TYPE2 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_sse_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_sse_t2(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_sse_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_t2(IMB_MGR *state); + +/* SSE TYPE3 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_sse_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_sse_t3(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_sse_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_t3(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_t3(IMB_MGR *state); + /* AVX512 TYPE1 manager functions */ IMB_DLL_EXPORT IMB_JOB *submit_job_avx512_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *flush_job_avx512_t1(IMB_MGR *state); diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 3973e8bc..c786d520 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -656,24 +656,10 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) switch (job->hash_alg) { case IMB_AUTH_HMAC_SHA_1: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_HMAC_NI(hmac_sha_1_ooo, job); -#endif return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); case IMB_AUTH_HMAC_SHA_224: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_HMAC_SHA_224_NI - (hmac_sha_224_ooo, job); -#endif return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); case IMB_AUTH_HMAC_SHA_256: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_HMAC_SHA_256_NI - (hmac_sha_256_ooo, job); -#endif return SUBMIT_JOB_HMAC_SHA_256(hmac_sha_256_ooo, job); case IMB_AUTH_HMAC_SHA_384: return SUBMIT_JOB_HMAC_SHA_384(hmac_sha_384_ooo, job); @@ -706,22 +692,10 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) job->msg_len_to_hash_in_bytes * 8; return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); case IMB_AUTH_SHA_1: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_SHA1_NI(sha_1_ooo, job); -#endif return SUBMIT_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_SHA224_NI(sha_224_ooo, job); -#endif return SUBMIT_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return SUBMIT_JOB_SHA256_NI(sha_256_ooo, job); -#endif return SUBMIT_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: return SUBMIT_JOB_SHA384(sha_384_ooo, job); @@ -857,46 +831,20 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) switch (job->hash_alg) { case IMB_AUTH_HMAC_SHA_1: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_HMAC_NI(hmac_sha_1_ooo); -#endif return FLUSH_JOB_HMAC(hmac_sha_1_ooo); case IMB_AUTH_HMAC_SHA_224: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_HMAC_SHA_224_NI - (hmac_sha_224_ooo); -#endif return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); case IMB_AUTH_HMAC_SHA_256: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_HMAC_SHA_256_NI - (hmac_sha_256_ooo); -#endif return FLUSH_JOB_HMAC_SHA_256(hmac_sha_256_ooo); case IMB_AUTH_HMAC_SHA_384: return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); case IMB_AUTH_HMAC_SHA_512: return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); case IMB_AUTH_SHA_1: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_SHA1_NI(sha_1_ooo, job); -#endif return FLUSH_JOB_SHA1(sha_1_ooo, job); case IMB_AUTH_SHA_224: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_SHA224_NI(sha_224_ooo, job); -#endif return FLUSH_JOB_SHA224(sha_224_ooo, job); case IMB_AUTH_SHA_256: -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - return FLUSH_JOB_SHA256_NI(sha_256_ooo, job); -#endif return FLUSH_JOB_SHA256(sha_256_ooo, job); case IMB_AUTH_SHA_384: return FLUSH_JOB_SHA384(sha_384_ooo, job); @@ -3242,15 +3190,6 @@ uint32_t submit_burst_hmac_sha_1(IMB_MGR *state, const uint32_t n_jobs, const int run_check) { -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_1, - (void *)state->hmac_sha_1_ooo, - (void *)SUBMIT_JOB_HMAC_NI, - (void *)FLUSH_JOB_HMAC_NI); - } -#endif return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_1, (void *)state->hmac_sha_1_ooo, @@ -3264,15 +3203,6 @@ uint32_t submit_burst_hmac_sha_224(IMB_MGR *state, const uint32_t n_jobs, const int run_check) { -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_224, - (void *)state->hmac_sha_224_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_224_NI, - (void *)FLUSH_JOB_HMAC_SHA_224_NI); - } -#endif return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_224, (void *)state->hmac_sha_224_ooo, @@ -3287,15 +3217,6 @@ uint32_t submit_burst_hmac_sha_256(IMB_MGR *state, const uint32_t n_jobs, const int run_check) { -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_256, - (void *)state->hmac_sha_256_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_256_NI, - (void *)FLUSH_JOB_HMAC_SHA_256_NI); - } -#endif return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_256, (void *)state->hmac_sha_256_ooo, diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 5192f8ea..2860171b 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1037,6 +1037,8 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_CPUFLAGS_NO_AESNI (IMB_FEATURE_SSE4_2 | IMB_FEATURE_CMOV) #define IMB_CPUFLAGS_SSE (IMB_CPUFLAGS_NO_AESNI | IMB_FEATURE_AESNI | \ IMB_FEATURE_PCLMULQDQ) +#define IMB_CPUFLAGS_SSE_T2 (IMB_CPUFLAGS_SSE | IMB_FEATURE_SHANI) +#define IMB_CPUFLAGS_SSE_T3 (IMB_CPUFLAGS_SSE_T2 | IMB_FEATURE_GFNI) #define IMB_CPUFLAGS_AVX (IMB_CPUFLAGS_SSE | IMB_FEATURE_AVX) #define IMB_CPUFLAGS_AVX2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_AVX2 | \ IMB_FEATURE_BMI2) diff --git a/lib/sse_t1/mb_mgr_sse.c b/lib/sse_t1/mb_mgr_sse.c index d33a6f37..a116969e 100644 --- a/lib/sse_t1/mb_mgr_sse.c +++ b/lib/sse_t1/mb_mgr_sse.c @@ -25,930 +25,78 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ -#include -#include -#include - -#define SSE - #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_interface.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" -#include "include/snow3g_submit.h" - -#include "include/save_xmms.h" -#include "include/des.h" +#include "include/error.h" #include "include/cpu_feature.h" -#include "include/noaesni.h" #include "include/aesni_emu.h" -#include "include/error.h" - -#include "include/arch_x86_64.h" /* self-test */ -#include "include/arch_sse_type1.h" -#include "include/arch_sse_type2.h" -#include "include/arch_sse_type3.h" - -#include "include/ooo_mgr_reset.h" - -#define SAVE_XMMS save_xmms -#define RESTORE_XMMS restore_xmms - -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_ptr -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_ptr -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_ptr -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_ptr -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_ptr -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_ptr -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse -#define SUBMIT_JOB_HMAC submit_job_hmac_sse -#define FLUSH_JOB_HMAC flush_job_hmac_sse -#define SUBMIT_JOB_HMAC_NI submit_job_hmac_ni_sse -#define FLUSH_JOB_HMAC_NI flush_job_hmac_ni_sse -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse -#define SUBMIT_JOB_HMAC_SHA_224_NI submit_job_hmac_sha_224_ni_sse -#define FLUSH_JOB_HMAC_SHA_224_NI flush_job_hmac_sha_224_ni_sse -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse -#define SUBMIT_JOB_HMAC_SHA_256_NI submit_job_hmac_sha_256_ni_sse -#define FLUSH_JOB_HMAC_SHA_256_NI flush_job_hmac_sha_256_ni_sse -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse -#define SUBMIT_JOB_SHA1 submit_job_sha1_sse -#define FLUSH_JOB_SHA1 flush_job_sha1_sse -#define SUBMIT_JOB_SHA224 submit_job_sha224_sse -#define FLUSH_JOB_SHA224 flush_job_sha224_sse -#define SUBMIT_JOB_SHA256 submit_job_sha256_sse -#define FLUSH_JOB_SHA256 flush_job_sha256_sse -#define SUBMIT_JOB_SHA1_NI submit_job_sha1_ni_sse -#define FLUSH_JOB_SHA1_NI flush_job_sha1_ni_sse -#define SUBMIT_JOB_SHA224_NI submit_job_sha224_ni_sse -#define FLUSH_JOB_SHA224_NI flush_job_sha224_ni_sse -#define SUBMIT_JOB_SHA256_NI submit_job_sha256_ni_sse -#define FLUSH_JOB_SHA256_NI flush_job_sha256_ni_sse -#define SUBMIT_JOB_SHA384 submit_job_sha384_sse -#define FLUSH_JOB_SHA384 flush_job_sha384_sse -#define SUBMIT_JOB_SHA512 submit_job_sha512_sse -#define FLUSH_JOB_SHA512 flush_job_sha512_sse - -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_sse -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_sse -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_sse -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_sse -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_sse -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_sse -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_sse -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_sse - -#define AES_CBC_DEC_128 aes128_cbc_dec_ptr -#define AES_CBC_DEC_192 aes192_cbc_dec_ptr -#define AES_CBC_DEC_256 aes256_cbc_dec_ptr - -#define AES_CNTR_128 aes_cntr_128_sse -#define AES_CNTR_192 aes_cntr_192_sse -#define AES_CNTR_256 aes_cntr_256_sse - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse - -#define AES_ECB_ENC_128 aes_ecb_enc_128_sse -#define AES_ECB_ENC_192 aes_ecb_enc_192_sse -#define AES_ECB_ENC_256 aes_ecb_enc_256_sse -#define AES_ECB_DEC_128 aes_ecb_dec_128_sse -#define AES_ECB_DEC_192 aes_ecb_dec_192_sse -#define AES_ECB_DEC_256 aes_ecb_dec_256_sse - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse - -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_sse -#define FLUSH_JOB flush_job_sse -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse -#define GET_NEXT_JOB get_next_job_sse -#define GET_COMPLETED_JOB get_completed_job_sse -#define GET_NEXT_BURST get_next_burst_sse -#define SUBMIT_BURST submit_burst_sse -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse -#define FLUSH_BURST flush_burst_sse -#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse -#define SUBMIT_HASH_BURST submit_hash_burst_sse -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse - -#define QUEUE_SIZE queue_size_sse - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_sse -#define AES_CFB_256_ONE aes_cfb_256_one_sse - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_ptr -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_ptr - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_ptr -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_ptr - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_ptr -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_ptr - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_ptr -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_ptr - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_sse -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_sse -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_sse -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_sse - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_sse -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_sse -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse - -#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse -#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse - -/* ====================================================================== */ - -/* - * Used to decide if SHA1/SHA256 SIMD or SHA1NI OOO scheduler should be - * called. - */ -#define HASH_USE_SHAEXT 1 - - -/* ====================================================================== */ - -uint32_t -ethernet_fcs_sse_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_sse_local - - -/* ====================================================================== */ - -/* - * CBC decrypt function pointers - */ - -typedef void (* cbc_dec_fn_t)(const void *, const uint8_t *, const void *, - void *, uint64_t); - -static cbc_dec_fn_t aes128_cbc_dec_ptr = aes_cbc_dec_128_sse; -static cbc_dec_fn_t aes192_cbc_dec_ptr = aes_cbc_dec_192_sse; -static cbc_dec_fn_t aes256_cbc_dec_ptr = aes_cbc_dec_256_sse; - -/* - * CBC encrypt function pointers - */ - -typedef IMB_JOB *(*aes_submit_job_t)(MB_MGR_AES_OOO *, IMB_JOB *); - -static aes_submit_job_t submit_job_aes128_enc_ptr = - submit_job_aes128_enc_sse; -static aes_submit_job_t submit_job_aes192_enc_ptr = - submit_job_aes192_enc_sse; -static aes_submit_job_t submit_job_aes256_enc_ptr = - submit_job_aes256_enc_sse; +#include "include/arch_x86_64.h" -typedef IMB_JOB *(*aes_flush_job_t)(MB_MGR_AES_OOO *); - -static aes_flush_job_t flush_job_aes128_enc_ptr = flush_job_aes128_enc_sse; -static aes_flush_job_t flush_job_aes192_enc_ptr = flush_job_aes192_enc_sse; -static aes_flush_job_t flush_job_aes256_enc_ptr = flush_job_aes256_enc_sse; - -/* ====================================================================== */ - -/* - * CMAC function pointers - */ - -typedef IMB_JOB *(*cmac_submit_job_t)(MB_MGR_CMAC_OOO *, IMB_JOB *); -typedef IMB_JOB *(*cmac_flush_job_t)(MB_MGR_CMAC_OOO *); - -static cmac_submit_job_t submit_job_aes128_cmac_auth_ptr = - submit_job_aes128_cmac_auth_sse; -static cmac_flush_job_t flush_job_aes128_cmac_auth_ptr = - flush_job_aes128_cmac_auth_sse; -static cmac_submit_job_t submit_job_aes256_cmac_auth_ptr = - submit_job_aes256_cmac_auth_sse; -static cmac_flush_job_t flush_job_aes256_cmac_auth_ptr = - flush_job_aes256_cmac_auth_sse; - -/* ====================================================================== */ - -/* - * CCM function pointers - */ - -typedef IMB_JOB *(*ccm_submit_job_t)(MB_MGR_CCM_OOO *, IMB_JOB *); -typedef IMB_JOB *(*ccm_flush_job_t)(MB_MGR_CCM_OOO *); - -static ccm_submit_job_t submit_job_aes128_ccm_auth_ptr = - submit_job_aes128_ccm_auth_sse; -static ccm_flush_job_t flush_job_aes128_ccm_auth_ptr = - flush_job_aes128_ccm_auth_sse; - -static ccm_submit_job_t submit_job_aes256_ccm_auth_ptr = - submit_job_aes256_ccm_auth_sse; -static ccm_flush_job_t flush_job_aes256_ccm_auth_ptr = - flush_job_aes256_ccm_auth_sse; - -/* ====================================================================== */ - -/* - * ECB function pointers - */ -static void -(*aes_ecb_enc_128_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_128_by4_sse; - -static void -(*aes_ecb_enc_192_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_192_by4_sse; - -static void -(*aes_ecb_enc_256_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_256_by4_sse; - -static void -(*aes_ecb_dec_128_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_128_by4_sse; - -static void -(*aes_ecb_dec_192_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_192_by4_sse; - -static void -(*aes_ecb_dec_256_sse) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_256_by4_sse; - -/* ====================================================================== */ - -/* - * GCM submit / flush API for SSE arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) +IMB_DLL_LOCAL void +init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) { - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; } +#endif - job->status = IMB_STATUS_COMPLETED; - return job; -} + if (!(state->features & IMB_FEATURE_AESNI)) { + fallback_no_aesni(state, 1); + return; + } -static IMB_JOB * -submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } + /* reset error status */ + imb_set_errno(state, 0); - job->status = IMB_STATUS_COMPLETED; - return job; + if ((state->features & IMB_CPUFLAGS_SSE_T3) == IMB_CPUFLAGS_SSE_T3) + init_mb_mgr_sse_t3_internal(state, reset_mgrs); + else if ((state->features & IMB_CPUFLAGS_SSE_T2) == IMB_CPUFLAGS_SSE_T2) + init_mb_mgr_sse_t2_internal(state, reset_mgrs); + else + init_mb_mgr_sse_t1_internal(state, reset_mgrs); } -static IMB_JOB * -submit_snow3g_uea2_job_sse(IMB_MGR *state, IMB_JOB *job) +void +init_mb_mgr_sse(IMB_MGR *state) { - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - if ((job->msg_len_to_cipher_in_bits & 7) || - (job->cipher_start_offset_in_bits & 7)) - return def_submit_snow3g_uea2_job(state, job); + init_mb_mgr_sse_internal(state, 1); - return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); } -static IMB_JOB * -flush_snow3g_uea2_job_sse(IMB_MGR *state) +IMB_JOB *submit_job_sse(IMB_MGR *state) { - MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; - - return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); + return IMB_SUBMIT_JOB(state); } -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_sse(IMB_JOB *job) +IMB_JOB *flush_job_sse(IMB_MGR *state) { - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; + return IMB_FLUSH_JOB(state); } -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_bit_sse(IMB_JOB *job) +uint32_t queue_size_sse(IMB_MGR *state) { - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; + return IMB_QUEUE_SIZE(state); } -/* ====================================================================== */ - -static IMB_JOB * -(*submit_job_zuc_eea3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eea3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc_eea3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eea3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc_eia3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc_eia3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc_eia3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc_eia3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc256_eea3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job) = - submit_job_zuc256_eea3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc256_eea3_sse) - (MB_MGR_ZUC_OOO *state) = flush_job_zuc256_eea3_no_gfni_sse; - -static IMB_JOB * -(*submit_job_zuc256_eia3_sse) - (MB_MGR_ZUC_OOO *state, IMB_JOB *job, const uint64_t tag_sz) = - submit_job_zuc256_eia3_no_gfni_sse; - -static IMB_JOB * -(*flush_job_zuc256_eia3_sse) - (MB_MGR_ZUC_OOO *state, const uint64_t tag_sz) = - flush_job_zuc256_eia3_no_gfni_sse; - -static void -reset_ooo_mgrs(IMB_MGR *state) +IMB_JOB *submit_job_nocheck_sse(IMB_MGR *state) { - /* Init AES out-of-order fields */ - if (state->features & IMB_FEATURE_GFNI) { - ooo_mgr_aes_reset(state->aes128_ooo, 8); - submit_job_aes128_enc_ptr = submit_job_aes128_enc_x8_sse; - flush_job_aes128_enc_ptr = flush_job_aes128_enc_x8_sse; - } else { - ooo_mgr_aes_reset(state->aes128_ooo, 4); - } - - if (state->features & IMB_FEATURE_GFNI) { - ooo_mgr_aes_reset(state->aes192_ooo, 8); - submit_job_aes192_enc_ptr = submit_job_aes192_enc_x8_sse; - flush_job_aes192_enc_ptr = flush_job_aes192_enc_x8_sse; - } else { - ooo_mgr_aes_reset(state->aes192_ooo, 4); - } - - if (state->features & IMB_FEATURE_GFNI) { - ooo_mgr_aes_reset(state->aes256_ooo, 8); - submit_job_aes256_enc_ptr = submit_job_aes256_enc_x8_sse; - flush_job_aes256_enc_ptr = flush_job_aes256_enc_x8_sse; - } else { - ooo_mgr_aes_reset(state->aes256_ooo, 4); - } - - if (state->features & IMB_FEATURE_GFNI) { - /* change AES-CBC decrypt implementation */ - aes128_cbc_dec_ptr = aes_cbc_dec_128_by8_sse; - aes192_cbc_dec_ptr = aes_cbc_dec_192_by8_sse; - aes256_cbc_dec_ptr = aes_cbc_dec_256_by8_sse; - } - - if (state->features & IMB_FEATURE_GFNI) { - /* change AES-ECB implementation */ - aes_ecb_enc_128_sse = aes_ecb_enc_128_by8_sse; - aes_ecb_enc_192_sse = aes_ecb_enc_192_by8_sse; - aes_ecb_enc_256_sse = aes_ecb_enc_256_by8_sse; - - aes_ecb_dec_128_sse = aes_ecb_dec_128_by8_sse; - aes_ecb_dec_192_sse = aes_ecb_dec_192_by8_sse; - aes_ecb_dec_256_sse = aes_ecb_dec_256_by8_sse; - } - - /* DOCSIS SEC BPI uses same settings as AES CBC */ - if (state->features & IMB_FEATURE_GFNI) { - ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); - } else { - ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 4); - ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 4); - ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 4); - ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 4); - } - - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); - - /* Init HMAC/SHA1 out-of-order fields */ - ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, SSE_NUM_SHA1_LANES); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA1 NI out-of-order fields */ - ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, 2); - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA224 out-of-order fields */ - ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, - SSE_NUM_SHA256_LANES); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA224 NI out-of-order fields */ - ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA_256 out-of-order fields */ - ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, - SSE_NUM_SHA256_LANES); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) { - /* Init HMAC/SHA256 NI out-of-order fields */ - ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); - } -#endif /* HASH_USE_SHAEXT */ - - /* Init HMAC/SHA384 out-of-order fields */ - ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, - SSE_NUM_SHA512_LANES); - - /* Init HMAC/SHA512 out-of-order fields */ - ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, - SSE_NUM_SHA512_LANES); - - /* Init HMAC/MD5 out-of-order fields */ - ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); - - /* Init AES/XCBC OOO fields */ - ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); - - /* Init AES-CCM auth out-of-order fields */ - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes128_ccm_auth_ptr = - submit_job_aes128_ccm_auth_x8_sse; - flush_job_aes128_ccm_auth_ptr = - flush_job_aes128_ccm_auth_x8_sse; - ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); - } else { - ooo_mgr_ccm_reset(state->aes_ccm_ooo, 4); - } - - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes256_ccm_auth_ptr = - submit_job_aes256_ccm_auth_x8_sse; - flush_job_aes256_ccm_auth_ptr = - flush_job_aes256_ccm_auth_x8_sse; - ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); - } else { - ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 4); - } - - /* Init AES-CMAC auth out-of-order fields */ - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes128_cmac_auth_ptr = - submit_job_aes128_cmac_auth_x8_sse; - flush_job_aes128_cmac_auth_ptr = - flush_job_aes128_cmac_auth_x8_sse; - ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); - } else { - ooo_mgr_cmac_reset(state->aes_cmac_ooo, 4); - } - - if (state->features & IMB_FEATURE_GFNI) { - submit_job_aes256_cmac_auth_ptr = - submit_job_aes256_cmac_auth_x8_sse; - flush_job_aes256_cmac_auth_ptr = - flush_job_aes256_cmac_auth_x8_sse; - ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); - } else { - ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 4); - } - - /* Init AES-CBCS out-of-order fields */ - ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - /* Init SHA1 NI out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, 2); - else -#endif /* HASH_USE_SHAEXT */ - /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - /* Init SHA224 NI out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, 2); - else -#endif /* HASH_USE_SHAEXT */ - /* Init SHA224 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); - -#ifdef HASH_USE_SHAEXT - if (state->features & IMB_FEATURE_SHANI) - /* Init SHA256 NI out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, 2); - else -#endif /* HASH_USE_SHAEXT */ - /* Init SHA256 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); - - /* Init SHA384 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); - - /* Init SHA512 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); - - /* Init SNOW3G-UEA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); - - /* Init SNOW3G-UIA out-of-order fields */ - ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); + return IMB_SUBMIT_JOB_NOCHECK(state); } -IMB_DLL_LOCAL void -init_mb_mgr_sse_internal(IMB_MGR *state, const int reset_mgrs) +IMB_JOB *get_next_job_sse(IMB_MGR *state) { -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - /* Check if CPU flags needed for SSE interface are present */ - if ((state->features & IMB_CPUFLAGS_SSE) != IMB_CPUFLAGS_SSE) { - imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); - return; - } - - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_SSE; - - if (state->features & IMB_FEATURE_GFNI) { - submit_job_zuc_eea3_sse = submit_job_zuc_eea3_gfni_sse; - flush_job_zuc_eea3_sse = flush_job_zuc_eea3_gfni_sse; - submit_job_zuc_eia3_sse = submit_job_zuc_eia3_gfni_sse; - flush_job_zuc_eia3_sse = flush_job_zuc_eia3_gfni_sse; - submit_job_zuc256_eea3_sse = submit_job_zuc256_eea3_gfni_sse; - flush_job_zuc256_eea3_sse = flush_job_zuc256_eea3_gfni_sse; - submit_job_zuc256_eia3_sse = submit_job_zuc256_eia3_gfni_sse; - flush_job_zuc256_eia3_sse = flush_job_zuc256_eia3_gfni_sse; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set SSE handlers */ - state->get_next_job = get_next_job_sse; - state->submit_job = submit_job_sse; - state->get_next_burst = get_next_burst_sse; - state->submit_burst = submit_burst_sse; - state->submit_burst_nocheck= submit_burst_nocheck_sse; - state->flush_burst = flush_burst_sse; - state->submit_cipher_burst = submit_cipher_burst_sse; - state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_sse; - state->submit_hash_burst = submit_hash_burst_sse; - state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_sse; - state->submit_job_nocheck = submit_job_nocheck_sse; - state->get_completed_job = get_completed_job_sse; - state->flush_job = flush_job_sse; - state->queue_size = queue_size_sse; - state->keyexp_128 = aes_keyexp_128_sse; - state->keyexp_192 = aes_keyexp_192_sse; - state->keyexp_256 = aes_keyexp_256_sse; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse; - state->xcbc_keyexp = aes_xcbc_expand_key_sse; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_sse; - state->sha1 = sha1_sse; - state->sha224_one_block = sha224_one_block_sse; - state->sha224 = sha224_sse; - state->sha256_one_block = sha256_one_block_sse; - state->sha256 = sha256_sse; - state->sha384_one_block = sha384_one_block_sse; - state->sha384 = sha384_sse; - state->sha512_one_block = sha512_one_block_sse; - state->sha512 = sha512_sse; - state->md5_one_block = md5_one_block_sse; - state->aes128_cfb_one = aes_cfb_128_one_sse; - state->crc32_ethernet_fcs = ethernet_fcs_sse; - state->crc16_x25 = crc16_x25_sse; - state->crc32_sctp = crc32_sctp_sse; - state->crc24_lte_a = crc24_lte_a_sse; - state->crc24_lte_b = crc24_lte_b_sse; - state->crc16_fp_data = crc16_fp_data_sse; - state->crc11_fp_header = crc11_fp_header_sse; - state->crc7_fp_header = crc7_fp_header_sse; - state->crc10_iuup_data = crc10_iuup_data_sse; - state->crc6_iuup_header = crc6_iuup_header_sse; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse; - - state->eea3_1_buffer = zuc_eea3_1_buffer_sse; - if (state->features & IMB_FEATURE_GFNI) { - state->eea3_4_buffer = zuc_eea3_4_buffer_gfni_sse; - state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_sse; - state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_sse; - } else { - state->eea3_4_buffer = zuc_eea3_4_buffer_sse; - state->eea3_n_buffer = zuc_eea3_n_buffer_sse; - state->eia3_n_buffer = zuc_eia3_n_buffer_sse; - } - - state->eia3_1_buffer = zuc_eia3_1_buffer_sse; - - state->f8_1_buffer = kasumi_f8_1_buffer_sse; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse; - state->f8_2_buffer = kasumi_f8_2_buffer_sse; - state->f8_3_buffer = kasumi_f8_3_buffer_sse; - state->f8_4_buffer = kasumi_f8_4_buffer_sse; - state->f8_n_buffer = kasumi_f8_n_buffer_sse; - state->f9_1_buffer = kasumi_f9_1_buffer_sse; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse; - state->kasumi_key_sched_size = kasumi_key_sched_size_sse; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse; - state->snow3g_init_key_sched = snow3g_init_key_sched_sse; - state->snow3g_key_sched_size = snow3g_key_sched_size_sse; - - state->gcm128_enc = aes_gcm_enc_128_sse; - state->gcm192_enc = aes_gcm_enc_192_sse; - state->gcm256_enc = aes_gcm_enc_256_sse; - state->gcm128_dec = aes_gcm_dec_128_sse; - state->gcm192_dec = aes_gcm_dec_192_sse; - state->gcm256_dec = aes_gcm_dec_256_sse; - state->gcm128_init = aes_gcm_init_128_sse; - state->gcm192_init = aes_gcm_init_192_sse; - state->gcm256_init = aes_gcm_init_256_sse; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; - state->gcm128_enc_update = aes_gcm_enc_128_update_sse; - state->gcm192_enc_update = aes_gcm_enc_192_update_sse; - state->gcm256_enc_update = aes_gcm_enc_256_update_sse; - state->gcm128_dec_update = aes_gcm_dec_128_update_sse; - state->gcm192_dec_update = aes_gcm_dec_192_update_sse; - state->gcm256_dec_update = aes_gcm_dec_256_update_sse; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; - state->gcm128_precomp = aes_gcm_precomp_128_sse; - state->gcm192_precomp = aes_gcm_precomp_192_sse; - state->gcm256_precomp = aes_gcm_precomp_256_sse; - state->gcm128_pre = aes_gcm_pre_128_sse; - state->gcm192_pre = aes_gcm_pre_192_sse; - state->gcm256_pre = aes_gcm_pre_256_sse; - state->ghash = ghash_sse; - state->ghash_pre = ghash_pre_sse; - - state->gmac128_init = imb_aes_gmac_init_128_sse; - state->gmac192_init = imb_aes_gmac_init_192_sse; - state->gmac256_init = imb_aes_gmac_init_256_sse; - state->gmac128_update = imb_aes_gmac_update_128_sse; - state->gmac192_update = imb_aes_gmac_update_192_sse; - state->gmac256_update = imb_aes_gmac_update_256_sse; - state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; - state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; - state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; - - state->hec_32 = hec_32_sse; - state->hec_64 = hec_64_sse; - - state->chacha20_poly1305_init = init_chacha20_poly1305_sse; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; + return IMB_GET_NEXT_JOB(state); } -void -init_mb_mgr_sse(IMB_MGR *state) +IMB_JOB *get_completed_job_sse(IMB_MGR *state) { - init_mb_mgr_sse_internal(state, 1); - - if (!self_test(state)) - imb_set_errno(state, IMB_ERR_SELFTEST); + return IMB_GET_COMPLETED_JOB(state); } -#include "mb_mgr_code.h" diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c new file mode 100644 index 00000000..55d2994d --- /dev/null +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -0,0 +1,652 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define SSE + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms +#define RESTORE_XMMS restore_xmms + +/* JOB API */ +#define SUBMIT_JOB submit_job_sse_t1 +#define FLUSH_JOB flush_job_sse_t1 +#define QUEUE_SIZE queue_size_sse_t1 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_t1 +#define GET_NEXT_JOB get_next_job_sse_t1 +#define GET_COMPLETED_JOB get_completed_job_sse_t1 +#define GET_NEXT_BURST get_next_burst_sse_t1 +#define SUBMIT_BURST submit_burst_sse_t1 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_t1 +#define FLUSH_BURST flush_burst_sse_t1 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_t1 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_t1 +#define SUBMIT_HASH_BURST submit_hash_burst_sse_t1 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t1 + + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE + +/* AES-GCM */ +#define AES_GCM_DEC_128 aes_gcm_dec_128_sse +#define AES_GCM_ENC_128 aes_gcm_enc_128_sse +#define AES_GCM_DEC_192 aes_gcm_dec_192_sse +#define AES_GCM_ENC_192 aes_gcm_enc_192_sse +#define AES_GCM_DEC_256 aes_gcm_dec_256_sse +#define AES_GCM_ENC_256 aes_gcm_enc_256_sse + +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse + +static IMB_JOB * +submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_128(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_192(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_DEC_IV_256(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +static IMB_JOB * +submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_128(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_192(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_ENC_IV_256(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse + +/* AES-CBC */ +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse + +#define AES_CBC_DEC_128 aes_cbc_dec_128_sse +#define AES_CBC_DEC_192 aes_cbc_dec_192_sse +#define AES_CBC_DEC_256 aes_cbc_dec_256_sse + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_sse +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_sse +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_sse +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_sse + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse + +#define AES_ECB_ENC_128 aes_ecb_enc_128_by4_sse +#define AES_ECB_ENC_192 aes_ecb_enc_192_by4_sse +#define AES_ECB_ENC_256 aes_ecb_enc_256_by4_sse +#define AES_ECB_DEC_128 aes_ecb_dec_128_by4_sse +#define AES_ECB_DEC_192 aes_ecb_dec_192_by4_sse +#define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse + +/* AES-CTR */ +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse + +#define AES_CNTR_128 aes_cntr_128_sse +#define AES_CNTR_192 aes_cntr_192_sse +#define AES_CNTR_256 aes_cntr_256_sse + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_sse +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_sse + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_sse +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_sse + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_sse +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_sse + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_sse +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_sse + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_sse +#define AES_CFB_256_ONE aes_cfb_256_one_sse + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_sse +#define FLUSH_JOB_SHA384 flush_job_sha384_sse +#define SUBMIT_JOB_SHA512 submit_job_sha512_sse +#define FLUSH_JOB_SHA512 flush_job_sha512_sse + +/* HMAC-SHA1/224/256/384/512/MD5 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_sse +#define FLUSH_JOB_HMAC flush_job_hmac_sse +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_sse +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse + +/* DES & 3DES */ + +/* - default x86-64 implementation */ + +/* DES-DOCSIS */ + +/* - default x86-64 implementation */ + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_sse +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_no_gfni_sse +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_no_gfni_sse +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_no_gfni_sse +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_no_gfni_sse +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_sse +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_sse(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_sse(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_sse_local + +/* ====================================================================== */ + +static IMB_JOB * +submit_job_aes_cntr_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +static IMB_JOB * +submit_job_aes_cntr_bit_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + aes_cntr_bit_128_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + aes_cntr_bit_192_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + aes_cntr_bit_256_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +/* ====================================================================== */ + + +static void +reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 4); + ooo_mgr_aes_reset(state->aes192_ooo, 4); + ooo_mgr_aes_reset(state->aes256_ooo, 4); + + /* DOCSIS SEC BPI uses same settings as AES CBC */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 4); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, SSE_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + SSE_NUM_SHA256_LANES); + + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + SSE_NUM_SHA256_LANES); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 4); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 4); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 4); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 4); + + /* Init AES-CBCS out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, SSE_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, SSE_NUM_SHA256_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, SSE_NUM_SHA256_LANES); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} + +IMB_DLL_LOCAL void +init_mb_mgr_sse_t1_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for SSE interface are present */ + if ((state->features & IMB_CPUFLAGS_SSE) != IMB_CPUFLAGS_SSE) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_SSE; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_sse; + state->keyexp_192 = aes_keyexp_192_sse; + state->keyexp_256 = aes_keyexp_256_sse; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse; + + state->xcbc_keyexp = aes_xcbc_expand_key_sse; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_sse; + state->sha1 = sha1_sse; + state->sha224_one_block = sha224_one_block_sse; + state->sha224 = sha224_sse; + state->sha256_one_block = sha256_one_block_sse; + state->sha256 = sha256_sse; + state->sha384_one_block = sha384_one_block_sse; + state->sha384 = sha384_sse; + state->sha512_one_block = sha512_one_block_sse; + state->sha512 = sha512_sse; + state->md5_one_block = md5_one_block_sse; + + state->aes128_cfb_one = aes_cfb_128_one_sse; + + state->eea3_1_buffer = zuc_eea3_1_buffer_sse; + state->eea3_4_buffer = zuc_eea3_4_buffer_sse; + state->eea3_n_buffer = zuc_eea3_n_buffer_sse; + state->eia3_n_buffer = zuc_eia3_n_buffer_sse; + state->eia3_1_buffer = zuc_eia3_1_buffer_sse; + + state->f8_1_buffer = kasumi_f8_1_buffer_sse; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse; + state->f8_2_buffer = kasumi_f8_2_buffer_sse; + state->f8_3_buffer = kasumi_f8_3_buffer_sse; + state->f8_4_buffer = kasumi_f8_4_buffer_sse; + state->f8_n_buffer = kasumi_f8_n_buffer_sse; + state->f9_1_buffer = kasumi_f9_1_buffer_sse; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse; + state->kasumi_key_sched_size = kasumi_key_sched_size_sse; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse; + state->snow3g_init_key_sched = snow3g_init_key_sched_sse; + state->snow3g_key_sched_size = snow3g_key_sched_size_sse; + + state->hec_32 = hec_32_sse; + state->hec_64 = hec_64_sse; + + state->crc32_ethernet_fcs = ethernet_fcs_sse; + state->crc16_x25 = crc16_x25_sse; + state->crc32_sctp = crc32_sctp_sse; + state->crc24_lte_a = crc24_lte_a_sse; + state->crc24_lte_b = crc24_lte_b_sse; + state->crc16_fp_data = crc16_fp_data_sse; + state->crc11_fp_header = crc11_fp_header_sse; + state->crc7_fp_header = crc7_fp_header_sse; + state->crc10_iuup_data = crc10_iuup_data_sse; + state->crc6_iuup_header = crc6_iuup_header_sse; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse; + + state->chacha20_poly1305_init = init_chacha20_poly1305_sse; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; + + state->gcm128_enc = aes_gcm_enc_128_sse; + state->gcm192_enc = aes_gcm_enc_192_sse; + state->gcm256_enc = aes_gcm_enc_256_sse; + state->gcm128_dec = aes_gcm_dec_128_sse; + state->gcm192_dec = aes_gcm_dec_192_sse; + state->gcm256_dec = aes_gcm_dec_256_sse; + state->gcm128_init = aes_gcm_init_128_sse; + state->gcm192_init = aes_gcm_init_192_sse; + state->gcm256_init = aes_gcm_init_256_sse; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; + state->gcm128_enc_update = aes_gcm_enc_128_update_sse; + state->gcm192_enc_update = aes_gcm_enc_192_update_sse; + state->gcm256_enc_update = aes_gcm_enc_256_update_sse; + state->gcm128_dec_update = aes_gcm_dec_128_update_sse; + state->gcm192_dec_update = aes_gcm_dec_192_update_sse; + state->gcm256_dec_update = aes_gcm_dec_256_update_sse; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; + state->gcm128_precomp = aes_gcm_precomp_128_sse; + state->gcm192_precomp = aes_gcm_precomp_192_sse; + state->gcm256_precomp = aes_gcm_precomp_256_sse; + state->gcm128_pre = aes_gcm_pre_128_sse; + state->gcm192_pre = aes_gcm_pre_192_sse; + state->gcm256_pre = aes_gcm_pre_256_sse; + + state->ghash = ghash_sse; + state->ghash_pre = ghash_pre_sse; + + state->gmac128_init = imb_aes_gmac_init_128_sse; + state->gmac192_init = imb_aes_gmac_init_192_sse; + state->gmac256_init = imb_aes_gmac_init_256_sse; + state->gmac128_update = imb_aes_gmac_update_128_sse; + state->gmac192_update = imb_aes_gmac_update_192_sse; + state->gmac256_update = imb_aes_gmac_update_256_sse; + state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; + state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; + state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; +} + +#include "mb_mgr_code.h" diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c new file mode 100644 index 00000000..ecd4ff5f --- /dev/null +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -0,0 +1,649 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define SSE + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" +#include "include/arch_sse_type2.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms +#define RESTORE_XMMS restore_xmms + +/* JOB API */ +#define SUBMIT_JOB submit_job_sse_t2 +#define FLUSH_JOB flush_job_sse_t2 +#define QUEUE_SIZE queue_size_sse_t2 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_t2 +#define GET_NEXT_JOB get_next_job_sse_t2 +#define GET_COMPLETED_JOB get_completed_job_sse_t2 +#define GET_NEXT_BURST get_next_burst_sse_t2 +#define SUBMIT_BURST submit_burst_sse_t2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_t2 +#define FLUSH_BURST flush_burst_sse_t2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_t2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_t2 +#define SUBMIT_HASH_BURST submit_hash_burst_sse_t2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t2 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE_T2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE_T2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE_T2 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE_T2 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE_T2 + +/* AES-GCM */ +#define AES_GCM_DEC_128 aes_gcm_dec_128_sse +#define AES_GCM_ENC_128 aes_gcm_enc_128_sse +#define AES_GCM_DEC_192 aes_gcm_dec_192_sse +#define AES_GCM_ENC_192 aes_gcm_enc_192_sse +#define AES_GCM_DEC_256 aes_gcm_dec_256_sse +#define AES_GCM_ENC_256 aes_gcm_enc_256_sse + +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse + +static IMB_JOB * +submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_128(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_192(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_DEC_IV_256(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +static IMB_JOB * +submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_128(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_192(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_ENC_IV_256(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse + +/* AES-CBC */ +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse + +#define AES_CBC_DEC_128 aes_cbc_dec_128_sse +#define AES_CBC_DEC_192 aes_cbc_dec_192_sse +#define AES_CBC_DEC_256 aes_cbc_dec_256_sse + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_sse +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_sse +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_sse +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_sse + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse + +#define AES_ECB_ENC_128 aes_ecb_enc_128_by4_sse +#define AES_ECB_ENC_192 aes_ecb_enc_192_by4_sse +#define AES_ECB_ENC_256 aes_ecb_enc_256_by4_sse +#define AES_ECB_DEC_128 aes_ecb_dec_128_by4_sse +#define AES_ECB_DEC_192 aes_ecb_dec_192_by4_sse +#define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse + +/* AES-CTR */ +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse + +#define AES_CNTR_128 aes_cntr_128_sse +#define AES_CNTR_192 aes_cntr_192_sse +#define AES_CNTR_256 aes_cntr_256_sse + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_sse +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_sse + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_sse +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_sse + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_sse +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_sse + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_sse +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_sse + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_sse +#define AES_CFB_256_ONE aes_cfb_256_one_sse + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_ni_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_ni_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_sse +#define FLUSH_JOB_SHA384 flush_job_sha384_sse +#define SUBMIT_JOB_SHA512 submit_job_sha512_sse +#define FLUSH_JOB_SHA512 flush_job_sha512_sse + +/* HMAC-SHA1/224/256/384/512/MD5 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_ni_sse +#define FLUSH_JOB_HMAC flush_job_hmac_ni_sse +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_ni_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_ni_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_ni_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_ni_sse +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse + +/* DES & 3DES */ + +/* - default x86-64 implementation */ + +/* DES-DOCSIS */ + +/* - default x86-64 implementation */ + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_sse +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_no_gfni_sse +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_no_gfni_sse +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_no_gfni_sse +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_no_gfni_sse +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_no_gfni_sse +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_no_gfni_sse +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_no_gfni_sse + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_sse +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_sse(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_sse(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_sse_local + +/* ====================================================================== */ + +static IMB_JOB * +submit_job_aes_cntr_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +static IMB_JOB * +submit_job_aes_cntr_bit_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + aes_cntr_bit_128_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + aes_cntr_bit_192_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + aes_cntr_bit_256_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +/* ====================================================================== */ + +static void +reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 4); + ooo_mgr_aes_reset(state->aes192_ooo, 4); + ooo_mgr_aes_reset(state->aes256_ooo, 4); + + /* DOCSIS SEC BPI uses same settings as AES CBC */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 4); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 4); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, 2); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); + + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 4); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 4); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 4); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 4); + + /* Init AES-CBCS out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, 2); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} + +IMB_DLL_LOCAL void +init_mb_mgr_sse_t2_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for SSE interface are present */ + if ((state->features & IMB_CPUFLAGS_SSE_T2) != IMB_CPUFLAGS_SSE_T2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_SSE; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_sse; + state->keyexp_192 = aes_keyexp_192_sse; + state->keyexp_256 = aes_keyexp_256_sse; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse; + + state->xcbc_keyexp = aes_xcbc_expand_key_sse; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_sse; + state->sha1 = sha1_sse; + state->sha224_one_block = sha224_one_block_sse; + state->sha224 = sha224_sse; + state->sha256_one_block = sha256_one_block_sse; + state->sha256 = sha256_sse; + state->sha384_one_block = sha384_one_block_sse; + state->sha384 = sha384_sse; + state->sha512_one_block = sha512_one_block_sse; + state->sha512 = sha512_sse; + state->md5_one_block = md5_one_block_sse; + + state->aes128_cfb_one = aes_cfb_128_one_sse; + + state->eea3_1_buffer = zuc_eea3_1_buffer_sse; + state->eea3_4_buffer = zuc_eea3_4_buffer_sse; + state->eea3_n_buffer = zuc_eea3_n_buffer_sse; + state->eia3_n_buffer = zuc_eia3_n_buffer_sse; + state->eia3_1_buffer = zuc_eia3_1_buffer_sse; + + state->f8_1_buffer = kasumi_f8_1_buffer_sse; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse; + state->f8_2_buffer = kasumi_f8_2_buffer_sse; + state->f8_3_buffer = kasumi_f8_3_buffer_sse; + state->f8_4_buffer = kasumi_f8_4_buffer_sse; + state->f8_n_buffer = kasumi_f8_n_buffer_sse; + state->f9_1_buffer = kasumi_f9_1_buffer_sse; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse; + state->kasumi_key_sched_size = kasumi_key_sched_size_sse; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse; + state->snow3g_init_key_sched = snow3g_init_key_sched_sse; + state->snow3g_key_sched_size = snow3g_key_sched_size_sse; + + state->hec_32 = hec_32_sse; + state->hec_64 = hec_64_sse; + + state->crc32_ethernet_fcs = ethernet_fcs_sse; + state->crc16_x25 = crc16_x25_sse; + state->crc32_sctp = crc32_sctp_sse; + state->crc24_lte_a = crc24_lte_a_sse; + state->crc24_lte_b = crc24_lte_b_sse; + state->crc16_fp_data = crc16_fp_data_sse; + state->crc11_fp_header = crc11_fp_header_sse; + state->crc7_fp_header = crc7_fp_header_sse; + state->crc10_iuup_data = crc10_iuup_data_sse; + state->crc6_iuup_header = crc6_iuup_header_sse; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse; + + state->chacha20_poly1305_init = init_chacha20_poly1305_sse; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; + + state->gcm128_enc = aes_gcm_enc_128_sse; + state->gcm192_enc = aes_gcm_enc_192_sse; + state->gcm256_enc = aes_gcm_enc_256_sse; + state->gcm128_dec = aes_gcm_dec_128_sse; + state->gcm192_dec = aes_gcm_dec_192_sse; + state->gcm256_dec = aes_gcm_dec_256_sse; + state->gcm128_init = aes_gcm_init_128_sse; + state->gcm192_init = aes_gcm_init_192_sse; + state->gcm256_init = aes_gcm_init_256_sse; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; + state->gcm128_enc_update = aes_gcm_enc_128_update_sse; + state->gcm192_enc_update = aes_gcm_enc_192_update_sse; + state->gcm256_enc_update = aes_gcm_enc_256_update_sse; + state->gcm128_dec_update = aes_gcm_dec_128_update_sse; + state->gcm192_dec_update = aes_gcm_dec_192_update_sse; + state->gcm256_dec_update = aes_gcm_dec_256_update_sse; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; + state->gcm128_precomp = aes_gcm_precomp_128_sse; + state->gcm192_precomp = aes_gcm_precomp_192_sse; + state->gcm256_precomp = aes_gcm_precomp_256_sse; + state->gcm128_pre = aes_gcm_pre_128_sse; + state->gcm192_pre = aes_gcm_pre_192_sse; + state->gcm256_pre = aes_gcm_pre_256_sse; + + state->ghash = ghash_sse; + state->ghash_pre = ghash_pre_sse; + + state->gmac128_init = imb_aes_gmac_init_128_sse; + state->gmac192_init = imb_aes_gmac_init_192_sse; + state->gmac256_init = imb_aes_gmac_init_256_sse; + state->gmac128_update = imb_aes_gmac_update_128_sse; + state->gmac192_update = imb_aes_gmac_update_192_sse; + state->gmac256_update = imb_aes_gmac_update_256_sse; + state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; + state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; + state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; +} + +#include "mb_mgr_code.h" diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c new file mode 100644 index 00000000..9a6cb1a0 --- /dev/null +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -0,0 +1,650 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define SSE + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/snow3g_submit.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" +#include "include/arch_sse_type2.h" +#include "include/arch_sse_type3.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms +#define RESTORE_XMMS restore_xmms + +/* JOB API */ +#define SUBMIT_JOB submit_job_sse_t3 +#define FLUSH_JOB flush_job_sse_t3 +#define QUEUE_SIZE queue_size_sse_t3 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_sse_t3 +#define GET_NEXT_JOB get_next_job_sse_t3 +#define GET_COMPLETED_JOB get_completed_job_sse_t3 +#define GET_NEXT_BURST get_next_burst_sse_t3 +#define SUBMIT_BURST submit_burst_sse_t3 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_sse_t3 +#define FLUSH_BURST flush_burst_sse_t3 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_sse_t3 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_sse_t3 +#define SUBMIT_HASH_BURST submit_hash_burst_sse_t3 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_sse_t3 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE_T3 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE_T3 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE_T3 +#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE_T3 +#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE_T3 + +/* AES-GCM */ +#define AES_GCM_DEC_128 aes_gcm_dec_128_sse +#define AES_GCM_ENC_128 aes_gcm_enc_128_sse +#define AES_GCM_DEC_192 aes_gcm_dec_192_sse +#define AES_GCM_ENC_192 aes_gcm_enc_192_sse +#define AES_GCM_DEC_256 aes_gcm_dec_256_sse +#define AES_GCM_ENC_256 aes_gcm_enc_256_sse + +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_sse +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse + +static IMB_JOB * +submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_128(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_192(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_DEC_IV_256(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +static IMB_JOB * +submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_128(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_192(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_ENC_IV_256(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse + +/* AES-CBC */ +#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_x8_sse +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_x8_sse + +#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_x8_sse +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_x8_sse + +#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_x8_sse +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_x8_sse + +#define AES_CBC_DEC_128 aes_cbc_dec_128_by8_sse +#define AES_CBC_DEC_192 aes_cbc_dec_192_by8_sse +#define AES_CBC_DEC_256 aes_cbc_dec_256_by8_sse + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_sse +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_sse +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_sse +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_sse + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_sse +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_sse +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_sse + +#define AES_ECB_ENC_128 aes_ecb_enc_128_by8_sse +#define AES_ECB_ENC_192 aes_ecb_enc_192_by8_sse +#define AES_ECB_ENC_256 aes_ecb_enc_256_by8_sse +#define AES_ECB_DEC_128 aes_ecb_dec_128_by8_sse +#define AES_ECB_DEC_192 aes_ecb_dec_192_by8_sse +#define AES_ECB_DEC_256 aes_ecb_dec_256_by8_sse + +/* AES-CTR */ +#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse +#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse + +#define AES_CNTR_128 aes_cntr_128_sse +#define AES_CNTR_192 aes_cntr_192_sse +#define AES_CNTR_256 aes_cntr_256_sse + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_x8_sse +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_x8_sse + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_x8_sse +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_x8_sse + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_x8_sse +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_x8_sse + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_x8_sse +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_x8_sse + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_sse +#define AES_CFB_256_ONE aes_cfb_256_one_sse + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_sse +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_sse + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_sse +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_sse +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_ni_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_ni_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_sse +#define FLUSH_JOB_SHA384 flush_job_sha384_sse +#define SUBMIT_JOB_SHA512 submit_job_sha512_sse +#define FLUSH_JOB_SHA512 flush_job_sha512_sse + +/* HMAC-SHA1/224/256/384/512/MD5 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_ni_sse +#define FLUSH_JOB_HMAC flush_job_hmac_ni_sse +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_ni_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_ni_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_ni_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_ni_sse +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_sse +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_sse +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_sse +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_sse +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_sse +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_sse + +/* DES & 3DES */ + +/* - default x86-64 implementation */ + +/* DES-DOCSIS */ + +/* - default x86-64 implementation */ + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_sse +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_sse +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_sse +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_gfni_sse +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_gfni_sse +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_gfni_sse +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_gfni_sse +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_gfni_sse +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_gfni_sse +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_gfni_sse +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_gfni_sse + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_sse +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_sse + +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_sse(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_sse(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_sse +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_sse + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_sse_local + +/* ====================================================================== */ + +static IMB_JOB * +submit_job_aes_cntr_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +static IMB_JOB * +submit_job_aes_cntr_bit_sse(IMB_JOB *job) +{ + if (16 == job->key_len_in_bytes) + aes_cntr_bit_128_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else if (24 == job->key_len_in_bytes) + aes_cntr_bit_192_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + else /* assume 32 bytes */ + aes_cntr_bit_256_sse(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +/* ====================================================================== */ + +static void +reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI uses same settings as AES CBC */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, 2); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); + + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + SSE_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, SSE_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 4); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES-CBCS out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 4); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, 2); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, SSE_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, SSE_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); +} + +IMB_DLL_LOCAL void +init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for SSE interface are present */ + if ((state->features & IMB_CPUFLAGS_SSE_T3) != IMB_CPUFLAGS_SSE_T3) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_SSE; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_sse; + state->keyexp_192 = aes_keyexp_192_sse; + state->keyexp_256 = aes_keyexp_256_sse; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_sse; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_sse; + + state->xcbc_keyexp = aes_xcbc_expand_key_sse; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_sse; + state->sha1 = sha1_sse; + state->sha224_one_block = sha224_one_block_sse; + state->sha224 = sha224_sse; + state->sha256_one_block = sha256_one_block_sse; + state->sha256 = sha256_sse; + state->sha384_one_block = sha384_one_block_sse; + state->sha384 = sha384_sse; + state->sha512_one_block = sha512_one_block_sse; + state->sha512 = sha512_sse; + state->md5_one_block = md5_one_block_sse; + + state->aes128_cfb_one = aes_cfb_128_one_sse; + + state->eea3_1_buffer = zuc_eea3_1_buffer_sse; + state->eea3_4_buffer = zuc_eea3_4_buffer_gfni_sse; + state->eea3_n_buffer = zuc_eea3_n_buffer_gfni_sse; + state->eia3_n_buffer = zuc_eia3_n_buffer_gfni_sse; + state->eia3_1_buffer = zuc_eia3_1_buffer_sse; + + state->f8_1_buffer = kasumi_f8_1_buffer_sse; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_sse; + state->f8_2_buffer = kasumi_f8_2_buffer_sse; + state->f8_3_buffer = kasumi_f8_3_buffer_sse; + state->f8_4_buffer = kasumi_f8_4_buffer_sse; + state->f8_n_buffer = kasumi_f8_n_buffer_sse; + state->f9_1_buffer = kasumi_f9_1_buffer_sse; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_sse; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_sse; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_sse; + state->kasumi_key_sched_size = kasumi_key_sched_size_sse; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_sse; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_sse; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_sse; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_sse; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_sse; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_sse; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_sse; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_sse; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_sse; + state->snow3g_init_key_sched = snow3g_init_key_sched_sse; + state->snow3g_key_sched_size = snow3g_key_sched_size_sse; + + state->hec_32 = hec_32_sse; + state->hec_64 = hec_64_sse; + + state->crc32_ethernet_fcs = ethernet_fcs_sse; + state->crc16_x25 = crc16_x25_sse; + state->crc32_sctp = crc32_sctp_sse; + state->crc24_lte_a = crc24_lte_a_sse; + state->crc24_lte_b = crc24_lte_b_sse; + state->crc16_fp_data = crc16_fp_data_sse; + state->crc11_fp_header = crc11_fp_header_sse; + state->crc7_fp_header = crc7_fp_header_sse; + state->crc10_iuup_data = crc10_iuup_data_sse; + state->crc6_iuup_header = crc6_iuup_header_sse; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_sse; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_sse; + + state->chacha20_poly1305_init = init_chacha20_poly1305_sse; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_sse; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_sse; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_sse; + + state->gcm128_enc = aes_gcm_enc_128_sse; + state->gcm192_enc = aes_gcm_enc_192_sse; + state->gcm256_enc = aes_gcm_enc_256_sse; + state->gcm128_dec = aes_gcm_dec_128_sse; + state->gcm192_dec = aes_gcm_dec_192_sse; + state->gcm256_dec = aes_gcm_dec_256_sse; + state->gcm128_init = aes_gcm_init_128_sse; + state->gcm192_init = aes_gcm_init_192_sse; + state->gcm256_init = aes_gcm_init_256_sse; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_sse; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_sse; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_sse; + state->gcm128_enc_update = aes_gcm_enc_128_update_sse; + state->gcm192_enc_update = aes_gcm_enc_192_update_sse; + state->gcm256_enc_update = aes_gcm_enc_256_update_sse; + state->gcm128_dec_update = aes_gcm_dec_128_update_sse; + state->gcm192_dec_update = aes_gcm_dec_192_update_sse; + state->gcm256_dec_update = aes_gcm_dec_256_update_sse; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_sse; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_sse; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_sse; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_sse; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_sse; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_sse; + state->gcm128_precomp = aes_gcm_precomp_128_sse; + state->gcm192_precomp = aes_gcm_precomp_192_sse; + state->gcm256_precomp = aes_gcm_precomp_256_sse; + state->gcm128_pre = aes_gcm_pre_128_sse; + state->gcm192_pre = aes_gcm_pre_192_sse; + state->gcm256_pre = aes_gcm_pre_256_sse; + + state->ghash = ghash_sse; + state->ghash_pre = ghash_pre_sse; + + state->gmac128_init = imb_aes_gmac_init_128_sse; + state->gmac192_init = imb_aes_gmac_init_192_sse; + state->gmac256_init = imb_aes_gmac_init_256_sse; + state->gmac128_update = imb_aes_gmac_update_128_sse; + state->gmac192_update = imb_aes_gmac_update_192_sse; + state->gmac256_update = imb_aes_gmac_update_256_sse; + state->gmac128_finalize = imb_aes_gmac_finalize_128_sse; + state->gmac192_finalize = imb_aes_gmac_finalize_192_sse; + state->gmac256_finalize = imb_aes_gmac_finalize_256_sse; +} + +#include "mb_mgr_code.h" diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 2febad47..4ba3e524 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -454,6 +454,9 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_avx512_t2.obj \ $(OBJ_DIR)\mb_mgr_des_avx512.obj \ $(OBJ_DIR)\mb_mgr_sse.obj \ + $(OBJ_DIR)\mb_mgr_sse_t1.obj \ + $(OBJ_DIR)\mb_mgr_sse_t2.obj \ + $(OBJ_DIR)\mb_mgr_sse_t3.obj \ $(OBJ_DIR)\alloc.obj \ $(OBJ_DIR)\version.obj \ $(OBJ_DIR)\cpu_feature.obj \ -- GitLab From 3075d95e1f60726314be84930c13c0cd1331e096 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 09:41:13 +0000 Subject: [PATCH 322/369] lib: [MB_MGR] is_job_invalid() function moved to a separate file --- lib/include/mb_mgr_code.h | 1484 +------------------------------ lib/include/mb_mgr_job_check.h | 1517 ++++++++++++++++++++++++++++++++ 2 files changed, 1518 insertions(+), 1483 deletions(-) create mode 100644 lib/include/mb_mgr_job_check.h diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index c786d520..45ae310e 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -50,6 +50,7 @@ #include "include/job_api_gcm.h" #include "include/job_api_snowv.h" #include "include/job_api_kasumi.h" +#include "include/mb_mgr_job_check.h" /* is_job_invalid() */ #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -890,1489 +891,6 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) /* Job submit & flush functions */ /* ========================================================================= */ -/* GCM NIST standard: len(M) < 2^39 - 256 */ -#define GCM_MAX_LEN UINT64_C(((1ULL << 39) - 256) - 1) -#define SNOW3G_MAX_BITLEN (UINT32_MAX) -#define MB_MAX_LEN16 ((1 << 16) - 2) - -__forceinline int -is_job_invalid(IMB_MGR *state, const IMB_JOB *job, - const IMB_CIPHER_MODE cipher_mode, const IMB_HASH_ALG hash_alg, - const IMB_CIPHER_DIRECTION cipher_direction, - const IMB_KEY_SIZE_BYTES key_len_in_bytes) -{ - const uint64_t auth_tag_len_fips[] = { - 0, /* INVALID selection */ - 20, /* IMB_AUTH_HMAC_SHA_1 */ - 28, /* IMB_AUTH_HMAC_SHA_224 */ - 32, /* IMB_AUTH_HMAC_SHA_256 */ - 48, /* IMB_AUTH_HMAC_SHA_384 */ - 64, /* IMB_AUTH_HMAC_SHA_512 */ - 12, /* IMB_AUTH_AES_XCBC */ - 16, /* IMB_AUTH_MD5 */ - 0, /* IMB_AUTH_NULL */ - 16, /* IMB_AUTH_AES_GMAC */ - 0, /* IMB_AUTH_CUSTOM */ - 0, /* IMB_AUTH_AES_CCM */ - 16, /* IMB_AUTH_AES_CMAC */ - 20, /* IMB_AUTH_SHA_1 */ - 28, /* IMB_AUTH_SHA_224 */ - 32, /* IMB_AUTH_SHA_256 */ - 48, /* IMB_AUTH_SHA_384 */ - 64, /* IMB_AUTH_SHA_512 */ - 4, /* IMB_AUTH_AES_CMAC 3GPP */ - 8, /* IMB_AUTH_PON_CRC_BIP */ - 4, /* IMB_AUTH_ZUC_EIA3_BITLEN */ - 4, /* IMB_AUTH_DOCSIS_CRC32 */ - 4, /* IMB_AUTH_SNOW3G_UIA2_BITLEN */ - 4, /* IMB_AUTH_KASUMI_UIA1 */ - 16, /* IMB_AUTH_AES_GMAC_128 */ - 16, /* IMB_AUTH_AES_GMAC_192 */ - 16, /* IMB_AUTH_AES_GMAC_256 */ - 16, /* IMB_AUTH_AES_CMAC_256 */ - 16, /* IMB_AUTH_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305_SGL */ - 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ - 16, /* IMB_AUTH_SNOW_V_AEAD */ - 16, /* IMB_AUTH_AES_GCM_SGL */ - 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ - 4, /* IMB_AUTH_CRC32_SCTP */ - 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ - 4, /* IMB_AUTH_CRC24_LTE_A */ - 4, /* IMB_AUTH_CRC24_LTE_B */ - 4, /* IMB_AUTH_CRC16_X25 */ - 4, /* IMB_AUTH_CRC16_FP_DATA */ - 4, /* IMB_AUTH_CRC11_FP_HEADER */ - 4, /* IMB_AUTH_CRC10_IUUP_DATA */ - 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ - 4, /* IMB_AUTH_CRC7_FP_HEADER */ - 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ - 16, /* IMB_AUTH_GHASH */ - }; - const uint64_t auth_tag_len_ipsec[] = { - 0, /* INVALID selection */ - 12, /* IMB_AUTH_HMAC_SHA_1 */ - 14, /* IMB_AUTH_HMAC_SHA_224 */ - 16, /* IMB_AUTH_HMAC_SHA_256 */ - 24, /* IMB_AUTH_HMAC_SHA_384 */ - 32, /* IMB_AUTH_HMAC_SHA_512 */ - 12, /* IMB_AUTH_AES_XCBC */ - 12, /* IMB_AUTH_MD5 */ - 0, /* IMB_AUTH_NULL */ - 16, /* IMB_AUTH_AES_GMAC */ - 0, /* IMB_AUTH_CUSTOM */ - 0, /* IMB_AUTH_AES_CCM */ - 16, /* IMB_AUTH_AES_CMAC */ - 20, /* IMB_AUTH_SHA_1 */ - 28, /* IMB_AUTH_SHA_224 */ - 32, /* IMB_AUTH_SHA_256 */ - 48, /* IMB_AUTH_SHA_384 */ - 64, /* IMB_AUTH_SHA_512 */ - 4, /* IMB_AUTH_AES_CMAC 3GPP */ - 8, /* IMB_AUTH_PON_CRC_BIP */ - 4, /* IMB_AUTH_ZUC_EIA3_BITLEN */ - 4, /* IMB_AUTH_DOCSIS_CRC32 */ - 4, /* IMB_AUTH_SNOW3G_UIA2_BITLEN */ - 4, /* IMB_AUTH_KASUMI_UIA1 */ - 16, /* IMB_AUTH_AES_GMAC_128 */ - 16, /* IMB_AUTH_AES_GMAC_192 */ - 16, /* IMB_AUTH_AES_GMAC_256 */ - 16, /* IMB_AUTH_AES_CMAC_256 */ - 16, /* IMB_AUTH_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305 */ - 16, /* IMB_AUTH_CHACHA_POLY1305_SGL */ - 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ - 16, /* IMB_AUTH_SNOW_V_AEAD */ - 16, /* IMB_AUTH_AES_GCM_SGL */ - 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ - 4, /* IMB_AUTH_CRC32_SCTP */ - 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ - 4, /* IMB_AUTH_CRC24_LTE_A */ - 4, /* IMB_AUTH_CRC24_LTE_B */ - 4, /* IMB_AUTH_CRC16_X25 */ - 4, /* IMB_AUTH_CRC16_FP_DATA */ - 4, /* IMB_AUTH_CRC11_FP_HEADER */ - 4, /* IMB_AUTH_CRC10_IUUP_DATA */ - 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ - 4, /* IMB_AUTH_CRC7_FP_HEADER */ - 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ - 16, /* IMB_AUTH_GHASH */ - }; - - /* Maximum length of buffer in PON is 2^14 + 8, since maximum - * PLI value is 2^14 - 1 + 1 extra byte of padding + 8 bytes - * of XGEM header */ - const uint64_t max_pon_len = (1 << 14) + 8; - - if (cipher_direction != IMB_DIR_DECRYPT && - cipher_direction != IMB_DIR_ENCRYPT && - cipher_mode != IMB_CIPHER_NULL) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); - return 1; - } - switch (cipher_mode) { - case IMB_CIPHER_CBC: - case IMB_CIPHER_CBCS_1_9: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (cipher_direction == IMB_DIR_ENCRYPT && - job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(24) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (cipher_mode == IMB_CIPHER_CBCS_1_9) { - if (job->msg_len_to_cipher_in_bytes > - ((1ULL << (60)) - 1)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->cipher_fields.CBCS.next_iv == NULL) { - imb_set_errno(state, - IMB_ERR_JOB_NULL_NEXT_IV); - return 1; - } - } else if (cipher_direction == IMB_DIR_ENCRYPT && - job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_ECB: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (cipher_direction == IMB_DIR_ENCRYPT && - job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(24) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(0)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_CNTR: - case IMB_CIPHER_CNTR_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(24) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if ((cipher_mode == IMB_CIPHER_CNTR && - job->iv_len_in_bytes != UINT64_C(16) && - job->iv_len_in_bytes != UINT64_C(12)) || - (cipher_mode == IMB_CIPHER_CNTR_BITLEN && - job->iv_len_in_bytes != UINT64_C(16))) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - /* - * msg_len_to_cipher_in_bits is used with CNTR_BITLEN, but it is - * effectively the same field as msg_len_to_cipher_in_bytes, - * since it is part of the same union - */ - if (job->msg_len_to_cipher_in_bytes == 0) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - break; - case IMB_CIPHER_NULL: - /* - * No checks required for this mode - * @note NULL cipher doesn't perform memory copy operation - * from source to destination - */ - break; - case IMB_CIPHER_DOCSIS_SEC_BPI: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - /* it has to be set regardless of direction (AES-CFB) */ - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if ((key_len_in_bytes != UINT64_C(16)) && - (key_len_in_bytes != UINT64_C(32))) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - break; - case IMB_CIPHER_GCM: - case IMB_CIPHER_GCM_SGL: - if (job->msg_len_to_cipher_in_bytes > GCM_MAX_LEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - /* Same key structure used for encrypt and decrypt */ - if (cipher_direction == IMB_DIR_ENCRYPT && - job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(24) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->iv_len_in_bytes == 0) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (cipher_mode == IMB_CIPHER_GCM && - hash_alg != IMB_AUTH_AES_GMAC) { - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - if (cipher_mode == IMB_CIPHER_GCM_SGL && - hash_alg != IMB_AUTH_GCM_SGL) { - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - break; - case IMB_CIPHER_CUSTOM: - /* no checks here */ - if (job->cipher_func == NULL) { - imb_set_errno(state, EFAULT); - return 1; - } - break; - case IMB_CIPHER_DES: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (cipher_direction == IMB_DIR_ENCRYPT && - job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_DOCSIS_DES: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (cipher_direction == IMB_DIR_ENCRYPT && - job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (cipher_direction == IMB_DIR_DECRYPT && - job->dec_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_CCM: - if (job->msg_len_to_cipher_in_bytes != 0) { - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - } - if (job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - /* AES-CTR and CBC-MAC use only encryption keys */ - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - /* currently only AES-CCM-128 and AES-CCM-256 supported */ - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - /* - * From RFC3610: - * Nonce length = 15 - L - * Valid L values are: 2 to 8 - * Then valid nonce lengths 13 to 7 (inclusive). - */ - if (job->iv_len_in_bytes > UINT64_C(13) || - job->iv_len_in_bytes < UINT64_C(7)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (hash_alg != IMB_AUTH_AES_CCM) { - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - break; - case IMB_CIPHER_DES3: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (key_len_in_bytes != UINT64_C(24)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (cipher_direction == IMB_DIR_ENCRYPT) { - const void * const *ks_ptr = - (const void * const *)job->enc_keys; - - if (ks_ptr == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (ks_ptr[0] == NULL || ks_ptr[1] == NULL || - ks_ptr[2] == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - } else { - const void * const *ks_ptr = - (const void * const *)job->dec_keys; - - if (ks_ptr == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (ks_ptr[0] == NULL || ks_ptr[1] == NULL || - ks_ptr[2] == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - } - break; - case IMB_CIPHER_PON_AES_CNTR: - /* - * CRC and cipher are done together. A few assumptions: - * - CRC and cipher start offsets are the same - * - last 4 bytes (32 bits) of the buffer is CRC - * - updated CRC value is put into the source buffer - * (encryption only) - * - CRC length is msg_len_to_cipher_in_bytes - 4 bytes - * - msg_len_to_cipher_in_bytes is aligned to 4 bytes - * - If msg_len_to_cipher_in_bytes is 0, IV and key pointers - * are not required, as encryption is not done - */ - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - - /* source and destination buffer pointers cannot be the same, - * as there are always 8 bytes that are not ciphered */ - if ((job->src + job->cipher_start_src_offset_in_bytes) - != job->dst) { - imb_set_errno(state, EINVAL); - return 1; - } - if (hash_alg != IMB_AUTH_PON_CRC_BIP) { - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - /* - * If message length to cipher != 0, AES-CTR is performed and - * key and IV require to be set properly - */ - if (job->msg_len_to_cipher_in_bytes != UINT64_C(0)) { - - /* message size needs to be aligned to 4 bytes */ - if ((job->msg_len_to_cipher_in_bytes & 3) != 0) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - - /* Subtract 8 bytes to maximum length since - * XGEM header is not ciphered */ - if ((job->msg_len_to_cipher_in_bytes > - (max_pon_len - 8))) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - - if (key_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - } - if (job->msg_len_to_cipher_in_bytes >= 4) { - const uint64_t xgem_hdr = *(const uint64_t *) - (job->src + - job->hash_start_src_offset_in_bytes); - - /* PLI is 14 MS bits of XGEM header */ - const uint16_t pli = BSWAP64(xgem_hdr) >> 50; - - /* CRC only if PLI is more than 4 bytes */ - if (pli > 4) { - const uint16_t crc_len = pli - 4; - - if (crc_len > - job->msg_len_to_cipher_in_bytes - 4) { - imb_set_errno(state, - IMB_ERR_JOB_PON_PLI); - return 1; - } - } - } - break; - case IMB_CIPHER_ZUC_EEA3: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16) && - key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > ZUC_MAX_BYTELEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (key_len_in_bytes == UINT64_C(16)) { - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - } else { - if (job->iv_len_in_bytes != UINT64_C(23) && - job->iv_len_in_bytes != UINT64_C(25)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - } - break; - case IMB_CIPHER_SNOW3G_UEA2_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bits == 0 || - job->msg_len_to_cipher_in_bits > SNOW3G_MAX_BITLEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_KASUMI_UEA1_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->msg_len_to_cipher_in_bits == 0 || - job->msg_len_to_cipher_in_bits > KASUMI_MAX_LEN) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(8)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_CHACHA20: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - /* Per RFC 7539, max cipher size is (2^32 - 1) x 64 */ - if (job->msg_len_to_cipher_in_bytes == 0 || - job->msg_len_to_cipher_in_bytes > ((1ULL << 38) - 64)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(12)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_CHACHA20_POLY1305: - case IMB_CIPHER_CHACHA20_POLY1305_SGL: - if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - /* Per RFC 7539, max cipher size is (2^32 - 1) x 64 */ - if (job->msg_len_to_cipher_in_bytes > ((1ULL << 38) - 64)) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(12)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - break; - case IMB_CIPHER_SNOW_V_AEAD: - case IMB_CIPHER_SNOW_V: - if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (job->iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->enc_keys == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (key_len_in_bytes != UINT64_C(32)) { - imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); - return 1; - } - if (job->iv_len_in_bytes != UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (cipher_mode == IMB_CIPHER_SNOW_V_AEAD && - hash_alg != IMB_AUTH_SNOW_V_AEAD) { - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - break; - default: - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - - switch (hash_alg) { - case IMB_AUTH_HMAC_SHA_1: - case IMB_AUTH_MD5: - case IMB_AUTH_HMAC_SHA_224: - case IMB_AUTH_HMAC_SHA_256: - case IMB_AUTH_HMAC_SHA_384: - case IMB_AUTH_HMAC_SHA_512: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg] && - job->auth_tag_output_len_in_bytes != - auth_tag_len_fips[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->msg_len_to_hash_in_bytes == 0 || - job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->u.HMAC._hashed_auth_key_xor_ipad == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_HMAC_IPAD); - return 1; - } - if (job->u.HMAC._hashed_auth_key_xor_opad == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_HMAC_OPAD); - return 1; - } - break; - case IMB_AUTH_AES_XCBC: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg] && - job->auth_tag_output_len_in_bytes != - auth_tag_len_fips[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.XCBC._k1_expanded == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K1_EXP); - return 1; - } - if (job->u.XCBC._k2 == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K2); - return 1; - } - if (job->u.XCBC._k3 == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K3); - return 1; - } - break; - case IMB_AUTH_NULL: - break; - case IMB_AUTH_CRC32_ETHERNET_FCS: - case IMB_AUTH_CRC32_SCTP: - case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: - case IMB_AUTH_CRC24_LTE_A: - case IMB_AUTH_CRC24_LTE_B: - case IMB_AUTH_CRC16_X25: - case IMB_AUTH_CRC16_FP_DATA: - case IMB_AUTH_CRC11_FP_HEADER: - case IMB_AUTH_CRC10_IUUP_DATA: - case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: - case IMB_AUTH_CRC7_FP_HEADER: - case IMB_AUTH_CRC6_IUUP_HEADER: - if (job->src == NULL && job->msg_len_to_hash_in_bytes != 0) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - break; - case IMB_AUTH_AES_GMAC: - if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || - job->auth_tag_output_len_in_bytes > UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if ((job->u.GCM.aad_len_in_bytes > 0) && - (job->u.GCM.aad == NULL)) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - if (cipher_mode != IMB_CIPHER_GCM) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - /* - * msg_len_to_hash_in_bytes not checked against zero. - * It is not used for AES-GCM & GMAC - see - * SUBMIT_JOB_AES_GCM_ENC and SUBMIT_JOB_AES_GCM_DEC functions. - */ - break; - case IMB_AUTH_GCM_SGL: - if (cipher_mode != IMB_CIPHER_GCM_SGL) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->u.GCM.ctx == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SGL_CTX); - return 1; - } - if (job->sgl_state == IMB_SGL_COMPLETE) { - if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || - job->auth_tag_output_len_in_bytes > UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - } - if (job->sgl_state == IMB_SGL_INIT) { - if ((job->u.GCM.aad_len_in_bytes > 0) && - (job->u.GCM.aad == NULL)) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - } - break; - case IMB_AUTH_AES_GMAC_128: - case IMB_AUTH_AES_GMAC_192: - case IMB_AUTH_AES_GMAC_256: - if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || - job->auth_tag_output_len_in_bytes > UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - /* This GMAC mode is to be used as stand-alone, - * not combined with GCM */ - if (cipher_mode == IMB_CIPHER_GCM) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->u.GMAC._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); - return 1; - } - if (job->u.GMAC._iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->u.GMAC.iv_len_in_bytes == 0) { - imb_set_errno(state, IMB_ERR_JOB_IV_LEN); - return 1; - } - if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - break; - case IMB_AUTH_GHASH: - if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || - job->auth_tag_output_len_in_bytes > UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->u.GHASH._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); - return 1; - } - if (job->u.GHASH._init_tag == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_GHASH_INIT_TAG); - return 1; - } - if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - break; - case IMB_AUTH_CUSTOM: - if (job->hash_func == NULL) { - imb_set_errno(state, EFAULT); - return 1; - } - break; - case IMB_AUTH_AES_CCM: - if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->u.CCM.aad_len_in_bytes > 46) { - /* 3 x AES_BLOCK - 2 bytes for AAD len */ - imb_set_errno(state, IMB_ERR_JOB_AAD_LEN); - return 1; - } - if ((job->u.CCM.aad_len_in_bytes > 0) && - (job->u.CCM.aad == NULL)) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - /* M can be any even number from 4 to 16 */ - if (job->auth_tag_output_len_in_bytes < UINT64_C(4) || - job->auth_tag_output_len_in_bytes > UINT64_C(16) || - ((job->auth_tag_output_len_in_bytes & 1) != 0)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (cipher_mode != IMB_CIPHER_CCM) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - /* - * AES-CCM allows for only one message for - * cipher and authentication. - * AAD can be used to extend authentication over - * clear text fields. - */ - if (job->msg_len_to_cipher_in_bytes != - job->msg_len_to_hash_in_bytes) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->cipher_start_src_offset_in_bytes != - job->hash_start_src_offset_in_bytes) { - imb_set_errno(state, IMB_ERR_JOB_SRC_OFFSET); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_AES_CMAC: - case IMB_AUTH_AES_CMAC_BITLEN: - case IMB_AUTH_AES_CMAC_256: - /* - * WARNING: When using IMB_AUTH_AES_CMAC_BITLEN, length of - * message is passed in bits, using job->msg_len_to_hash_in_bits - * (unlike "normal" IMB_AUTH_AES_CMAC, where is passed in bytes, - * using job->msg_len_to_hash_in_bytes). - */ - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->u.CMAC._key_expanded == NULL) || - (job->u.CMAC._skey1 == NULL) || - (job->u.CMAC._skey2 == NULL)) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - /* T is 128 bits but 96 bits is also allowed due to - * IPsec use case (RFC 4494) and 32 bits for CMAC 3GPP. - * ACVP validation requires tag size of 8 bits. - */ - if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || - job->auth_tag_output_len_in_bytes > UINT64_C(16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - break; - case IMB_AUTH_SHA_1: - case IMB_AUTH_SHA_224: - case IMB_AUTH_SHA_256: - case IMB_AUTH_SHA_384: - case IMB_AUTH_SHA_512: - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - break; - case IMB_AUTH_PON_CRC_BIP: - /* - * Authentication tag in PON is BIP 32-bit value only - * CRC is done together with cipher, - * its initial value is read from the source buffer and - * updated value put into the destination buffer. - * - msg_len_to_hash_in_bytes is aligned to 4 bytes - */ - if (((job->msg_len_to_hash_in_bytes & UINT64_C(3)) != 0) || - (job->msg_len_to_hash_in_bytes < UINT64_C(8)) || - (job->msg_len_to_hash_in_bytes > max_pon_len)) { - /* - * Length aligned to 4 bytes (and at least 8 bytes, - * including 8-byte XGEM header and no more - * than max length) - */ - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - /* 64-bits: - * - BIP 32-bits - * - CRC 32-bits - */ - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (cipher_mode != IMB_CIPHER_PON_AES_CNTR) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_ZUC_EIA3_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || - (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.ZUC_EIA3._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.ZUC_EIA3._iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_ZUC256_EIA3_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || - (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.ZUC_EIA3._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.ZUC_EIA3._iv == NULL) { - /* If 25-byte IV is NULL, check 23-byte IV */ - if (job->u.ZUC_EIA3._iv23 == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - } - if ((job->auth_tag_output_len_in_bytes != 4) && - (job->auth_tag_output_len_in_bytes != 8) && - (job->auth_tag_output_len_in_bytes != 16)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_DOCSIS_CRC32: - /** - * Use only in combination with DOCSIS_SEC_BPI. - * Assumptions about Ethernet PDU carried over DOCSIS: - * - cipher_start_src_offset_in_bytes <= - * (hash_start_src_offset_in_bytes + 12) - * - msg_len_to_cipher_in_bytes <= - * (msg_len_to_hash_in_bytes - 12 + 4) - * - @note: in-place operation allowed only - * - authentication tag size is 4 bytes - * - @note: in encrypt direction, computed CRC value is put into - * the source buffer - * - encrypt chain order: hash, cipher - * - decrypt chain order: cipher, hash - */ - if (cipher_mode != IMB_CIPHER_DOCSIS_SEC_BPI) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->msg_len_to_cipher_in_bytes && - job->msg_len_to_hash_in_bytes) { - const uint64_t ciph_adjust = - IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE - - 2 - /* ETH TYPE */ - IMB_DOCSIS_CRC32_TAG_SIZE; - - if ((job->msg_len_to_cipher_in_bytes + ciph_adjust) > - job->msg_len_to_hash_in_bytes) { - imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); - return 1; - } - if (job->cipher_start_src_offset_in_bytes < - (job->hash_start_src_offset_in_bytes + 12)) { - imb_set_errno(state, IMB_ERR_JOB_SRC_OFFSET); - return 1; - } - } - if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - /* Ethernet FCS CRC is 32-bits */ - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if ((cipher_direction == IMB_DIR_ENCRYPT && - job->chain_order != IMB_ORDER_HASH_CIPHER) || - (cipher_direction == IMB_DIR_DECRYPT && - job->chain_order != IMB_ORDER_CIPHER_HASH)) { - imb_set_errno(state, IMB_ERR_JOB_CHAIN_ORDER); - return 1; - } - break; - case IMB_AUTH_SNOW3G_UIA2_BITLEN: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if ((job->msg_len_to_hash_in_bits == 0) || - (job->msg_len_to_hash_in_bits > SNOW3G_MAX_BITLEN)) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.SNOW3G_UIA2._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->u.SNOW3G_UIA2._iv == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_IV); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_KASUMI_UIA1: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - /* - * KASUMI-UIA1 needs to be at least 8 bytes - * (IV + direction bit + '1' + 0s to align to byte boundary) - */ - if ((job->msg_len_to_hash_in_bytes < - (IMB_KASUMI_BLOCK_SIZE + 1)) || - (job->msg_len_to_hash_in_bytes > - (KASUMI_MAX_LEN / BYTESIZE))) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); - return 1; - } - if (job->u.KASUMI_UIA1._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - break; - case IMB_AUTH_POLY1305: - if (job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->u.POLY1305._key == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - break; - case IMB_AUTH_CHACHA20_POLY1305: - if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->msg_len_to_hash_in_bytes != 0 && job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->u.CHACHA20_POLY1305.aad == NULL && - job->u.CHACHA20_POLY1305.aad_len_in_bytes > 0) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - break; - case IMB_AUTH_CHACHA20_POLY1305_SGL: - if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); - return 1; - } - if (job->msg_len_to_hash_in_bytes != 0 && job->dst == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_DST); - return 1; - } - if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305_SGL) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - if (job->u.CHACHA20_POLY1305.aad == NULL && - job->u.CHACHA20_POLY1305.aad_len_in_bytes > 0) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (job->u.CHACHA20_POLY1305.ctx == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_SGL_CTX); - return 1; - } - break; - case IMB_AUTH_SNOW_V_AEAD: - if ((job->u.SNOW_V_AEAD.aad_len_in_bytes > 0) && - (job->u.SNOW_V_AEAD.aad == NULL)) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); - return 1; - } - if (job->auth_tag_output == NULL) { - imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); - return 1; - } - if (job->auth_tag_output_len_in_bytes != - auth_tag_len_ipsec[hash_alg]) { - imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); - return 1; - } - if (cipher_mode != IMB_CIPHER_SNOW_V_AEAD) { - imb_set_errno(state, IMB_ERR_CIPH_MODE); - return 1; - } - break; - default: - imb_set_errno(state, IMB_ERR_HASH_ALGO); - return 1; - } - return 0; -} - __forceinline IMB_JOB *SUBMIT_JOB_AES(IMB_MGR *state, IMB_JOB *job) { diff --git a/lib/include/mb_mgr_job_check.h b/lib/include/mb_mgr_job_check.h new file mode 100644 index 00000000..c9d49443 --- /dev/null +++ b/lib/include/mb_mgr_job_check.h @@ -0,0 +1,1517 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef MB_MGR_JOB_CHECK_H +#define MB_MGR_JOB_CHECK_H + +#include "intel-ipsec-mb.h" +#include "include/error.h" + +/* GCM NIST standard: len(M) < 2^39 - 256 */ +#define GCM_MAX_LEN UINT64_C(((1ULL << 39) - 256) - 1) +#define SNOW3G_MAX_BITLEN (UINT32_MAX) +#define MB_MAX_LEN16 ((1 << 16) - 2) + +__forceinline int +is_job_invalid(IMB_MGR *state, const IMB_JOB *job, + const IMB_CIPHER_MODE cipher_mode, const IMB_HASH_ALG hash_alg, + const IMB_CIPHER_DIRECTION cipher_direction, + const IMB_KEY_SIZE_BYTES key_len_in_bytes) +{ + const uint64_t auth_tag_len_fips[] = { + 0, /* INVALID selection */ + 20, /* IMB_AUTH_HMAC_SHA_1 */ + 28, /* IMB_AUTH_HMAC_SHA_224 */ + 32, /* IMB_AUTH_HMAC_SHA_256 */ + 48, /* IMB_AUTH_HMAC_SHA_384 */ + 64, /* IMB_AUTH_HMAC_SHA_512 */ + 12, /* IMB_AUTH_AES_XCBC */ + 16, /* IMB_AUTH_MD5 */ + 0, /* IMB_AUTH_NULL */ + 16, /* IMB_AUTH_AES_GMAC */ + 0, /* IMB_AUTH_CUSTOM */ + 0, /* IMB_AUTH_AES_CCM */ + 16, /* IMB_AUTH_AES_CMAC */ + 20, /* IMB_AUTH_SHA_1 */ + 28, /* IMB_AUTH_SHA_224 */ + 32, /* IMB_AUTH_SHA_256 */ + 48, /* IMB_AUTH_SHA_384 */ + 64, /* IMB_AUTH_SHA_512 */ + 4, /* IMB_AUTH_AES_CMAC 3GPP */ + 8, /* IMB_AUTH_PON_CRC_BIP */ + 4, /* IMB_AUTH_ZUC_EIA3_BITLEN */ + 4, /* IMB_AUTH_DOCSIS_CRC32 */ + 4, /* IMB_AUTH_SNOW3G_UIA2_BITLEN */ + 4, /* IMB_AUTH_KASUMI_UIA1 */ + 16, /* IMB_AUTH_AES_GMAC_128 */ + 16, /* IMB_AUTH_AES_GMAC_192 */ + 16, /* IMB_AUTH_AES_GMAC_256 */ + 16, /* IMB_AUTH_AES_CMAC_256 */ + 16, /* IMB_AUTH_POLY1305 */ + 16, /* IMB_AUTH_CHACHA_POLY1305 */ + 16, /* IMB_AUTH_CHACHA_POLY1305_SGL */ + 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ + 16, /* IMB_AUTH_SNOW_V_AEAD */ + 16, /* IMB_AUTH_AES_GCM_SGL */ + 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ + 4, /* IMB_AUTH_CRC32_SCTP */ + 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ + 4, /* IMB_AUTH_CRC24_LTE_A */ + 4, /* IMB_AUTH_CRC24_LTE_B */ + 4, /* IMB_AUTH_CRC16_X25 */ + 4, /* IMB_AUTH_CRC16_FP_DATA */ + 4, /* IMB_AUTH_CRC11_FP_HEADER */ + 4, /* IMB_AUTH_CRC10_IUUP_DATA */ + 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ + 4, /* IMB_AUTH_CRC7_FP_HEADER */ + 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ + }; + const uint64_t auth_tag_len_ipsec[] = { + 0, /* INVALID selection */ + 12, /* IMB_AUTH_HMAC_SHA_1 */ + 14, /* IMB_AUTH_HMAC_SHA_224 */ + 16, /* IMB_AUTH_HMAC_SHA_256 */ + 24, /* IMB_AUTH_HMAC_SHA_384 */ + 32, /* IMB_AUTH_HMAC_SHA_512 */ + 12, /* IMB_AUTH_AES_XCBC */ + 12, /* IMB_AUTH_MD5 */ + 0, /* IMB_AUTH_NULL */ + 16, /* IMB_AUTH_AES_GMAC */ + 0, /* IMB_AUTH_CUSTOM */ + 0, /* IMB_AUTH_AES_CCM */ + 16, /* IMB_AUTH_AES_CMAC */ + 20, /* IMB_AUTH_SHA_1 */ + 28, /* IMB_AUTH_SHA_224 */ + 32, /* IMB_AUTH_SHA_256 */ + 48, /* IMB_AUTH_SHA_384 */ + 64, /* IMB_AUTH_SHA_512 */ + 4, /* IMB_AUTH_AES_CMAC 3GPP */ + 8, /* IMB_AUTH_PON_CRC_BIP */ + 4, /* IMB_AUTH_ZUC_EIA3_BITLEN */ + 4, /* IMB_AUTH_DOCSIS_CRC32 */ + 4, /* IMB_AUTH_SNOW3G_UIA2_BITLEN */ + 4, /* IMB_AUTH_KASUMI_UIA1 */ + 16, /* IMB_AUTH_AES_GMAC_128 */ + 16, /* IMB_AUTH_AES_GMAC_192 */ + 16, /* IMB_AUTH_AES_GMAC_256 */ + 16, /* IMB_AUTH_AES_CMAC_256 */ + 16, /* IMB_AUTH_POLY1305 */ + 16, /* IMB_AUTH_CHACHA_POLY1305 */ + 16, /* IMB_AUTH_CHACHA_POLY1305_SGL */ + 4, /* IMB_AUTH_ZUC256_EIA3_BITLEN */ + 16, /* IMB_AUTH_SNOW_V_AEAD */ + 16, /* IMB_AUTH_AES_GCM_SGL */ + 4, /* IMB_AUTH_CRC32_ETHERNET_FCS */ + 4, /* IMB_AUTH_CRC32_SCTP */ + 4, /* IMB_AUTH_CRC32_WIMAX_OFDMA_DATA */ + 4, /* IMB_AUTH_CRC24_LTE_A */ + 4, /* IMB_AUTH_CRC24_LTE_B */ + 4, /* IMB_AUTH_CRC16_X25 */ + 4, /* IMB_AUTH_CRC16_FP_DATA */ + 4, /* IMB_AUTH_CRC11_FP_HEADER */ + 4, /* IMB_AUTH_CRC10_IUUP_DATA */ + 4, /* IMB_AUTH_CRC8_WIMAX_OFDMA_HCS */ + 4, /* IMB_AUTH_CRC7_FP_HEADER */ + 4, /* IMB_AUTH_CRC6_IUUP_HEADER */ + 16, /* IMB_AUTH_GHASH */ + }; + + /* Maximum length of buffer in PON is 2^14 + 8, since maximum + * PLI value is 2^14 - 1 + 1 extra byte of padding + 8 bytes + * of XGEM header */ + const uint64_t max_pon_len = (1 << 14) + 8; + + if (cipher_direction != IMB_DIR_DECRYPT && + cipher_direction != IMB_DIR_ENCRYPT && + cipher_mode != IMB_CIPHER_NULL) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_DIR); + return 1; + } + switch (cipher_mode) { + case IMB_CIPHER_CBC: + case IMB_CIPHER_CBCS_1_9: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (cipher_direction == IMB_DIR_ENCRYPT && + job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (cipher_mode == IMB_CIPHER_CBCS_1_9) { + if (job->msg_len_to_cipher_in_bytes > + ((1ULL << (60)) - 1)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->cipher_fields.CBCS.next_iv == NULL) { + imb_set_errno(state, + IMB_ERR_JOB_NULL_NEXT_IV); + return 1; + } + } else if (cipher_direction == IMB_DIR_ENCRYPT && + job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_ECB: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (cipher_direction == IMB_DIR_ENCRYPT && + job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes & UINT64_C(15)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(0)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_CNTR: + case IMB_CIPHER_CNTR_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if ((cipher_mode == IMB_CIPHER_CNTR && + job->iv_len_in_bytes != UINT64_C(16) && + job->iv_len_in_bytes != UINT64_C(12)) || + (cipher_mode == IMB_CIPHER_CNTR_BITLEN && + job->iv_len_in_bytes != UINT64_C(16))) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + /* + * msg_len_to_cipher_in_bits is used with CNTR_BITLEN, but it is + * effectively the same field as msg_len_to_cipher_in_bytes, + * since it is part of the same union + */ + if (job->msg_len_to_cipher_in_bytes == 0) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + break; + case IMB_CIPHER_NULL: + /* + * No checks required for this mode + * @note NULL cipher doesn't perform memory copy operation + * from source to destination + */ + break; + case IMB_CIPHER_DOCSIS_SEC_BPI: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + /* it has to be set regardless of direction (AES-CFB) */ + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if ((key_len_in_bytes != UINT64_C(16)) && + (key_len_in_bytes != UINT64_C(32))) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + break; + case IMB_CIPHER_GCM: + case IMB_CIPHER_GCM_SGL: + if (job->msg_len_to_cipher_in_bytes > GCM_MAX_LEN) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + /* Same key structure used for encrypt and decrypt */ + if (cipher_direction == IMB_DIR_ENCRYPT && + job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(24) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->iv_len_in_bytes == 0) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (cipher_mode == IMB_CIPHER_GCM && + hash_alg != IMB_AUTH_AES_GMAC) { + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + if (cipher_mode == IMB_CIPHER_GCM_SGL && + hash_alg != IMB_AUTH_GCM_SGL) { + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + break; + case IMB_CIPHER_CUSTOM: + /* no checks here */ + if (job->cipher_func == NULL) { + imb_set_errno(state, EFAULT); + return 1; + } + break; + case IMB_CIPHER_DES: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (cipher_direction == IMB_DIR_ENCRYPT && + job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_DOCSIS_DES: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (cipher_direction == IMB_DIR_ENCRYPT && + job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (cipher_direction == IMB_DIR_DECRYPT && + job->dec_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_CCM: + if (job->msg_len_to_cipher_in_bytes != 0) { + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + } + if (job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + /* AES-CTR and CBC-MAC use only encryption keys */ + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + /* currently only AES-CCM-128 and AES-CCM-256 supported */ + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + /* + * From RFC3610: + * Nonce length = 15 - L + * Valid L values are: 2 to 8 + * Then valid nonce lengths 13 to 7 (inclusive). + */ + if (job->iv_len_in_bytes > UINT64_C(13) || + job->iv_len_in_bytes < UINT64_C(7)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (hash_alg != IMB_AUTH_AES_CCM) { + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + break; + case IMB_CIPHER_DES3: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (key_len_in_bytes != UINT64_C(24)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes & UINT64_C(7)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (cipher_direction == IMB_DIR_ENCRYPT) { + const void * const *ks_ptr = + (const void * const *)job->enc_keys; + + if (ks_ptr == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (ks_ptr[0] == NULL || ks_ptr[1] == NULL || + ks_ptr[2] == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + } else { + const void * const *ks_ptr = + (const void * const *)job->dec_keys; + + if (ks_ptr == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (ks_ptr[0] == NULL || ks_ptr[1] == NULL || + ks_ptr[2] == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + } + break; + case IMB_CIPHER_PON_AES_CNTR: + /* + * CRC and cipher are done together. A few assumptions: + * - CRC and cipher start offsets are the same + * - last 4 bytes (32 bits) of the buffer is CRC + * - updated CRC value is put into the source buffer + * (encryption only) + * - CRC length is msg_len_to_cipher_in_bytes - 4 bytes + * - msg_len_to_cipher_in_bytes is aligned to 4 bytes + * - If msg_len_to_cipher_in_bytes is 0, IV and key pointers + * are not required, as encryption is not done + */ + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + + /* source and destination buffer pointers cannot be the same, + * as there are always 8 bytes that are not ciphered */ + if ((job->src + job->cipher_start_src_offset_in_bytes) + != job->dst) { + imb_set_errno(state, EINVAL); + return 1; + } + if (hash_alg != IMB_AUTH_PON_CRC_BIP) { + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + /* + * If message length to cipher != 0, AES-CTR is performed and + * key and IV require to be set properly + */ + if (job->msg_len_to_cipher_in_bytes != UINT64_C(0)) { + + /* message size needs to be aligned to 4 bytes */ + if ((job->msg_len_to_cipher_in_bytes & 3) != 0) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + + /* Subtract 8 bytes to maximum length since + * XGEM header is not ciphered */ + if ((job->msg_len_to_cipher_in_bytes > + (max_pon_len - 8))) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + + if (key_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + } + if (job->msg_len_to_cipher_in_bytes >= 4) { + const uint64_t xgem_hdr = *(const uint64_t *) + (job->src + + job->hash_start_src_offset_in_bytes); + + /* PLI is 14 MS bits of XGEM header */ + const uint16_t pli = BSWAP64(xgem_hdr) >> 50; + + /* CRC only if PLI is more than 4 bytes */ + if (pli > 4) { + const uint16_t crc_len = pli - 4; + + if (crc_len > + job->msg_len_to_cipher_in_bytes - 4) { + imb_set_errno(state, + IMB_ERR_JOB_PON_PLI); + return 1; + } + } + } + break; + case IMB_CIPHER_ZUC_EEA3: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16) && + key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > ZUC_MAX_BYTELEN) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (key_len_in_bytes == UINT64_C(16)) { + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + } else { + if (job->iv_len_in_bytes != UINT64_C(23) && + job->iv_len_in_bytes != UINT64_C(25)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + } + break; + case IMB_CIPHER_SNOW3G_UEA2_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bits == 0 || + job->msg_len_to_cipher_in_bits > SNOW3G_MAX_BITLEN) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_KASUMI_UEA1_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->msg_len_to_cipher_in_bits == 0 || + job->msg_len_to_cipher_in_bits > KASUMI_MAX_LEN) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(8)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_CHACHA20: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + /* Per RFC 7539, max cipher size is (2^32 - 1) x 64 */ + if (job->msg_len_to_cipher_in_bytes == 0 || + job->msg_len_to_cipher_in_bytes > ((1ULL << 38) - 64)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(12)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_CHACHA20_POLY1305: + case IMB_CIPHER_CHACHA20_POLY1305_SGL: + if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + /* Per RFC 7539, max cipher size is (2^32 - 1) x 64 */ + if (job->msg_len_to_cipher_in_bytes > ((1ULL << 38) - 64)) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(12)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + break; + case IMB_CIPHER_SNOW_V_AEAD: + case IMB_CIPHER_SNOW_V: + if (job->msg_len_to_cipher_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->msg_len_to_cipher_in_bytes != 0 && job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (job->iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->enc_keys == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (key_len_in_bytes != UINT64_C(32)) { + imb_set_errno(state, IMB_ERR_JOB_KEY_LEN); + return 1; + } + if (job->iv_len_in_bytes != UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (cipher_mode == IMB_CIPHER_SNOW_V_AEAD && + hash_alg != IMB_AUTH_SNOW_V_AEAD) { + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + break; + default: + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + + switch (hash_alg) { + case IMB_AUTH_HMAC_SHA_1: + case IMB_AUTH_MD5: + case IMB_AUTH_HMAC_SHA_224: + case IMB_AUTH_HMAC_SHA_256: + case IMB_AUTH_HMAC_SHA_384: + case IMB_AUTH_HMAC_SHA_512: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg] && + job->auth_tag_output_len_in_bytes != + auth_tag_len_fips[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->msg_len_to_hash_in_bytes == 0 || + job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->u.HMAC._hashed_auth_key_xor_ipad == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_HMAC_IPAD); + return 1; + } + if (job->u.HMAC._hashed_auth_key_xor_opad == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_HMAC_OPAD); + return 1; + } + break; + case IMB_AUTH_AES_XCBC: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg] && + job->auth_tag_output_len_in_bytes != + auth_tag_len_fips[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->u.XCBC._k1_expanded == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K1_EXP); + return 1; + } + if (job->u.XCBC._k2 == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K2); + return 1; + } + if (job->u.XCBC._k3 == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_XCBC_K3); + return 1; + } + break; + case IMB_AUTH_NULL: + break; + case IMB_AUTH_CRC32_ETHERNET_FCS: + case IMB_AUTH_CRC32_SCTP: + case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: + case IMB_AUTH_CRC24_LTE_A: + case IMB_AUTH_CRC24_LTE_B: + case IMB_AUTH_CRC16_X25: + case IMB_AUTH_CRC16_FP_DATA: + case IMB_AUTH_CRC11_FP_HEADER: + case IMB_AUTH_CRC10_IUUP_DATA: + case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: + case IMB_AUTH_CRC7_FP_HEADER: + case IMB_AUTH_CRC6_IUUP_HEADER: + if (job->src == NULL && job->msg_len_to_hash_in_bytes != 0) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + break; + case IMB_AUTH_AES_GMAC: + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if ((job->u.GCM.aad_len_in_bytes > 0) && + (job->u.GCM.aad == NULL)) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + if (cipher_mode != IMB_CIPHER_GCM) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + /* + * msg_len_to_hash_in_bytes not checked against zero. + * It is not used for AES-GCM & GMAC - see + * SUBMIT_JOB_AES_GCM_ENC and SUBMIT_JOB_AES_GCM_DEC functions. + */ + break; + case IMB_AUTH_GCM_SGL: + if (cipher_mode != IMB_CIPHER_GCM_SGL) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->u.GCM.ctx == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SGL_CTX); + return 1; + } + if (job->sgl_state == IMB_SGL_COMPLETE) { + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + } + if (job->sgl_state == IMB_SGL_INIT) { + if ((job->u.GCM.aad_len_in_bytes > 0) && + (job->u.GCM.aad == NULL)) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + } + break; + case IMB_AUTH_AES_GMAC_128: + case IMB_AUTH_AES_GMAC_192: + case IMB_AUTH_AES_GMAC_256: + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + /* This GMAC mode is to be used as stand-alone, + * not combined with GCM */ + if (cipher_mode == IMB_CIPHER_GCM) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->u.GMAC._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); + return 1; + } + if (job->u.GMAC._iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->u.GMAC.iv_len_in_bytes == 0) { + imb_set_errno(state, IMB_ERR_JOB_IV_LEN); + return 1; + } + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + break; + case IMB_AUTH_GHASH: + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->u.GHASH._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); + return 1; + } + if (job->u.GHASH._init_tag == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_GHASH_INIT_TAG); + return 1; + } + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + break; + case IMB_AUTH_CUSTOM: + if (job->hash_func == NULL) { + imb_set_errno(state, EFAULT); + return 1; + } + break; + case IMB_AUTH_AES_CCM: + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->u.CCM.aad_len_in_bytes > 46) { + /* 3 x AES_BLOCK - 2 bytes for AAD len */ + imb_set_errno(state, IMB_ERR_JOB_AAD_LEN); + return 1; + } + if ((job->u.CCM.aad_len_in_bytes > 0) && + (job->u.CCM.aad == NULL)) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + /* M can be any even number from 4 to 16 */ + if (job->auth_tag_output_len_in_bytes < UINT64_C(4) || + job->auth_tag_output_len_in_bytes > UINT64_C(16) || + ((job->auth_tag_output_len_in_bytes & 1) != 0)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (cipher_mode != IMB_CIPHER_CCM) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + /* + * AES-CCM allows for only one message for + * cipher and authentication. + * AAD can be used to extend authentication over + * clear text fields. + */ + if (job->msg_len_to_cipher_in_bytes != + job->msg_len_to_hash_in_bytes) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->cipher_start_src_offset_in_bytes != + job->hash_start_src_offset_in_bytes) { + imb_set_errno(state, IMB_ERR_JOB_SRC_OFFSET); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_AES_CMAC: + case IMB_AUTH_AES_CMAC_BITLEN: + case IMB_AUTH_AES_CMAC_256: + /* + * WARNING: When using IMB_AUTH_AES_CMAC_BITLEN, length of + * message is passed in bits, using job->msg_len_to_hash_in_bits + * (unlike "normal" IMB_AUTH_AES_CMAC, where is passed in bytes, + * using job->msg_len_to_hash_in_bytes). + */ + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if ((job->u.CMAC._key_expanded == NULL) || + (job->u.CMAC._skey1 == NULL) || + (job->u.CMAC._skey2 == NULL)) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + /* T is 128 bits but 96 bits is also allowed due to + * IPsec use case (RFC 4494) and 32 bits for CMAC 3GPP. + * ACVP validation requires tag size of 8 bits. + */ + if (job->auth_tag_output_len_in_bytes < UINT64_C(1) || + job->auth_tag_output_len_in_bytes > UINT64_C(16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + break; + case IMB_AUTH_SHA_1: + case IMB_AUTH_SHA_224: + case IMB_AUTH_SHA_256: + case IMB_AUTH_SHA_384: + case IMB_AUTH_SHA_512: + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + break; + case IMB_AUTH_PON_CRC_BIP: + /* + * Authentication tag in PON is BIP 32-bit value only + * CRC is done together with cipher, + * its initial value is read from the source buffer and + * updated value put into the destination buffer. + * - msg_len_to_hash_in_bytes is aligned to 4 bytes + */ + if (((job->msg_len_to_hash_in_bytes & UINT64_C(3)) != 0) || + (job->msg_len_to_hash_in_bytes < UINT64_C(8)) || + (job->msg_len_to_hash_in_bytes > max_pon_len)) { + /* + * Length aligned to 4 bytes (and at least 8 bytes, + * including 8-byte XGEM header and no more + * than max length) + */ + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + /* 64-bits: + * - BIP 32-bits + * - CRC 32-bits + */ + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (cipher_mode != IMB_CIPHER_PON_AES_CNTR) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_ZUC_EIA3_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || + (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->u.ZUC_EIA3._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (job->u.ZUC_EIA3._iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_ZUC256_EIA3_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if ((job->msg_len_to_hash_in_bits < ZUC_MIN_BITLEN) || + (job->msg_len_to_hash_in_bits > ZUC_MAX_BITLEN)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->u.ZUC_EIA3._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (job->u.ZUC_EIA3._iv == NULL) { + /* If 25-byte IV is NULL, check 23-byte IV */ + if (job->u.ZUC_EIA3._iv23 == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + } + if ((job->auth_tag_output_len_in_bytes != 4) && + (job->auth_tag_output_len_in_bytes != 8) && + (job->auth_tag_output_len_in_bytes != 16)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_DOCSIS_CRC32: + /** + * Use only in combination with DOCSIS_SEC_BPI. + * Assumptions about Ethernet PDU carried over DOCSIS: + * - cipher_start_src_offset_in_bytes <= + * (hash_start_src_offset_in_bytes + 12) + * - msg_len_to_cipher_in_bytes <= + * (msg_len_to_hash_in_bytes - 12 + 4) + * - @note: in-place operation allowed only + * - authentication tag size is 4 bytes + * - @note: in encrypt direction, computed CRC value is put into + * the source buffer + * - encrypt chain order: hash, cipher + * - decrypt chain order: cipher, hash + */ + if (cipher_mode != IMB_CIPHER_DOCSIS_SEC_BPI) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->msg_len_to_cipher_in_bytes && + job->msg_len_to_hash_in_bytes) { + const uint64_t ciph_adjust = + IMB_DOCSIS_CRC32_MIN_ETH_PDU_SIZE - + 2 - /* ETH TYPE */ + IMB_DOCSIS_CRC32_TAG_SIZE; + + if ((job->msg_len_to_cipher_in_bytes + ciph_adjust) > + job->msg_len_to_hash_in_bytes) { + imb_set_errno(state, IMB_ERR_JOB_CIPH_LEN); + return 1; + } + if (job->cipher_start_src_offset_in_bytes < + (job->hash_start_src_offset_in_bytes + 12)) { + imb_set_errno(state, IMB_ERR_JOB_SRC_OFFSET); + return 1; + } + } + if (job->msg_len_to_hash_in_bytes > MB_MAX_LEN16) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + /* Ethernet FCS CRC is 32-bits */ + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if ((cipher_direction == IMB_DIR_ENCRYPT && + job->chain_order != IMB_ORDER_HASH_CIPHER) || + (cipher_direction == IMB_DIR_DECRYPT && + job->chain_order != IMB_ORDER_CIPHER_HASH)) { + imb_set_errno(state, IMB_ERR_JOB_CHAIN_ORDER); + return 1; + } + break; + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if ((job->msg_len_to_hash_in_bits == 0) || + (job->msg_len_to_hash_in_bits > SNOW3G_MAX_BITLEN)) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->u.SNOW3G_UIA2._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (job->u.SNOW3G_UIA2._iv == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_IV); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_KASUMI_UIA1: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + /* + * KASUMI-UIA1 needs to be at least 8 bytes + * (IV + direction bit + '1' + 0s to align to byte boundary) + */ + if ((job->msg_len_to_hash_in_bytes < + (IMB_KASUMI_BLOCK_SIZE + 1)) || + (job->msg_len_to_hash_in_bytes > + (KASUMI_MAX_LEN / BYTESIZE))) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_LEN); + return 1; + } + if (job->u.KASUMI_UIA1._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_KEY); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + break; + case IMB_AUTH_POLY1305: + if (job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->u.POLY1305._key == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH_KEY); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + break; + case IMB_AUTH_CHACHA20_POLY1305: + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->msg_len_to_hash_in_bytes != 0 && job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->u.CHACHA20_POLY1305.aad == NULL && + job->u.CHACHA20_POLY1305.aad_len_in_bytes > 0) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + break; + case IMB_AUTH_CHACHA20_POLY1305_SGL: + if (job->msg_len_to_hash_in_bytes != 0 && job->src == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SRC); + return 1; + } + if (job->msg_len_to_hash_in_bytes != 0 && job->dst == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_DST); + return 1; + } + if (cipher_mode != IMB_CIPHER_CHACHA20_POLY1305_SGL) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + if (job->u.CHACHA20_POLY1305.aad == NULL && + job->u.CHACHA20_POLY1305.aad_len_in_bytes > 0) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (job->u.CHACHA20_POLY1305.ctx == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_SGL_CTX); + return 1; + } + break; + case IMB_AUTH_SNOW_V_AEAD: + if ((job->u.SNOW_V_AEAD.aad_len_in_bytes > 0) && + (job->u.SNOW_V_AEAD.aad == NULL)) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AAD); + return 1; + } + if (job->auth_tag_output == NULL) { + imb_set_errno(state, IMB_ERR_JOB_NULL_AUTH); + return 1; + } + if (job->auth_tag_output_len_in_bytes != + auth_tag_len_ipsec[hash_alg]) { + imb_set_errno(state, IMB_ERR_JOB_AUTH_TAG_LEN); + return 1; + } + if (cipher_mode != IMB_CIPHER_SNOW_V_AEAD) { + imb_set_errno(state, IMB_ERR_CIPH_MODE); + return 1; + } + break; + default: + imb_set_errno(state, IMB_ERR_HASH_ALGO); + return 1; + } + return 0; +} + +#endif /* MB_MGR_JOB_CHECK_H */ -- GitLab From c1e6663a17e2b65fc734a6b63f6db102f8c6fe3e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 09:49:45 +0000 Subject: [PATCH 323/369] lib: [MB_MGR] cipher and hash synchronous burst API's moved to a separate file --- lib/include/mb_mgr_burst.h | 461 +++++++++++++++++++++++++++++++++++++ lib/include/mb_mgr_code.h | 425 +--------------------------------- 2 files changed, 462 insertions(+), 424 deletions(-) create mode 100644 lib/include/mb_mgr_burst.h diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h new file mode 100644 index 00000000..51863b83 --- /dev/null +++ b/lib/include/mb_mgr_burst.h @@ -0,0 +1,461 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef MB_MGR_BURST_H +#define MB_MGR_BURST_H + +/* synchronous cipher and hash burst API */ + +#include "intel-ipsec-mb.h" +#include "include/error.h" +#include "include/mb_mgr_job_check.h" /* is_job_invalid() */ + +__forceinline +uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + MB_MGR_AES_OOO *aes_ooo = NULL; + + IMB_JOB * (*submit_fn)(MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; + IMB_JOB * (*flush_fn)(MB_MGR_AES_OOO *state) = NULL; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CBC, IMB_AUTH_NULL, + IMB_DIR_ENCRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == 16) { + aes_ooo = state->aes128_ooo; + submit_fn = SUBMIT_JOB_AES128_ENC; + flush_fn = FLUSH_JOB_AES128_ENC; + } else if (key_size == 24) { + aes_ooo = state->aes192_ooo; + submit_fn = SUBMIT_JOB_AES192_ENC; + flush_fn = FLUSH_JOB_AES192_ENC; + } else { /* assume 32 */ + aes_ooo = state->aes256_ooo; + submit_fn = SUBMIT_JOB_AES256_ENC; + flush_fn = FLUSH_JOB_AES256_ENC; + } + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = submit_fn(aes_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while((job = flush_fn(aes_ooo)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + return completed_jobs; +} + +__forceinline +uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + void (*submit_fn) (const void *in, const uint8_t *IV, + const void *keys, void *out, + uint64_t len_bytes) = NULL; + (void) state; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CBC, IMB_AUTH_NULL, + IMB_DIR_DECRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == 16) + submit_fn = AES_CBC_DEC_128; + else if (key_size == 24) + submit_fn = AES_CBC_DEC_192; + else /* assume 32 */ + submit_fn = AES_CBC_DEC_256; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + + return completed_jobs; +} + +__forceinline +uint32_t submit_aes_ctr_burst(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_CNTR, IMB_AUTH_NULL, + IMB_DIR_ENCRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + +#ifdef AVX512 + if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { + void (*submit_fn_vaes) (IMB_JOB *job) = NULL; + + if (key_size == 16) + submit_fn_vaes = aes_cntr_128_submit_vaes_avx512; + else if (key_size == 24) + submit_fn_vaes = aes_cntr_192_submit_vaes_avx512; + else /* assume 32 */ + submit_fn_vaes = aes_cntr_256_submit_vaes_avx512; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn_vaes(job); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } else { +#endif + void (*submit_fn) (const void *in, const void *IV, + const void *keys, void *out, + uint64_t len_bytes, + uint64_t iv_len_bytes) = NULL; + + if (key_size == 16) + submit_fn = AES_CNTR_128; + else if (key_size == 24) + submit_fn = AES_CNTR_192; + else /* assume 32 */ + submit_fn = AES_CNTR_256; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + submit_fn(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } +#ifdef AVX512 + } +#endif + return completed_jobs; +} + +__forceinline +uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size, + const int run_check) +{ + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + + switch (cipher) { + case IMB_CIPHER_CBC: + if (dir == IMB_DIR_ENCRYPT) + return submit_aes_cbc_burst_enc(state, jobs, n_jobs, + key_size, run_check); + else + return submit_aes_cbc_burst_dec(state, jobs, n_jobs, + key_size, run_check); + case IMB_CIPHER_CNTR: + return submit_aes_ctr_burst(state, jobs, n_jobs, + key_size, run_check); + default: + break; + } + + /* unsupported cipher mode */ + imb_set_errno(state, IMB_ERR_CIPH_MODE); + + return 0; +} + +uint32_t +SUBMIT_CIPHER_BURST(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size) +{ + return submit_cipher_burst_and_check(state, jobs, n_jobs, + cipher, dir, key_size, 1); +} + +uint32_t +SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size) +{ + return submit_cipher_burst_and_check(state, jobs, n_jobs, + cipher, dir, key_size, 0); +} + +__forceinline +uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check, + const IMB_HASH_ALG hash_alg, + void *ooo_mgr, + IMB_JOB *(*submit_fn)(void *, IMB_JOB *), + IMB_JOB *(*flush_fn)(void *)) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, + IMB_CIPHER_NULL, + hash_alg, + IMB_DIR_ENCRYPT, + job->key_len_in_bytes)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = submit_fn(ooo_mgr, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = flush_fn(ooo_mgr)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + return completed_jobs; +} + +__forceinline +uint32_t submit_burst_hmac_sha_1(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_1, + (void *)state->hmac_sha_1_ooo, + (void *)SUBMIT_JOB_HMAC, + (void *)FLUSH_JOB_HMAC); +} + +__forceinline +uint32_t submit_burst_hmac_sha_224(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_224, + (void *)state->hmac_sha_224_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_224, + (void *)FLUSH_JOB_HMAC_SHA_224); + +} + +__forceinline +uint32_t submit_burst_hmac_sha_256(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_256, + (void *)state->hmac_sha_256_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_256, + (void *)FLUSH_JOB_HMAC_SHA_256); +} + +__forceinline +uint32_t submit_burst_hmac_sha_384(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_384, + (void *)state->hmac_sha_384_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_384, + (void *)FLUSH_JOB_HMAC_SHA_384); +} + +__forceinline +uint32_t submit_burst_hmac_sha_512(IMB_MGR *state, + IMB_JOB *jobs, + const uint32_t n_jobs, + const int run_check) +{ + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_512, + (void *)state->hmac_sha_512_ooo, + (void *)SUBMIT_JOB_HMAC_SHA_512, + (void *)FLUSH_JOB_HMAC_SHA_512); +} + +__forceinline +uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash, + const int run_check) +{ + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + if (jobs == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_JOB); + return 0; + } + } + + switch (hash) { + case IMB_AUTH_HMAC_SHA_1: + return submit_burst_hmac_sha_1(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_224: + return submit_burst_hmac_sha_224(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_256: + return submit_burst_hmac_sha_256(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_384: + return submit_burst_hmac_sha_384(state, jobs, + n_jobs, run_check); + case IMB_AUTH_HMAC_SHA_512: + return submit_burst_hmac_sha_512(state, jobs, + n_jobs, run_check); + default: + break; + } + + /* unsupported hash alg */ + imb_set_errno(state, IMB_ERR_HASH_ALGO); + + return 0; +} + +uint32_t +SUBMIT_HASH_BURST(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash) +{ + return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 1); +} + +uint32_t +SUBMIT_HASH_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash) +{ + return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 0); +} + +#endif /* MB_MGR_BURST_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 45ae310e..575fad28 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -51,6 +51,7 @@ #include "include/job_api_snowv.h" #include "include/job_api_kasumi.h" #include "include/mb_mgr_job_check.h" /* is_job_invalid() */ +#include "include/mb_mgr_burst.h" /* synchronous cipher/hash burst API */ #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -1402,428 +1403,4 @@ FLUSH_BURST(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs) return n_ret_jobs; } -__forceinline -uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_KEY_SIZE_BYTES key_size, - const int run_check) -{ - uint32_t i, completed_jobs = 0; - MB_MGR_AES_OOO *aes_ooo = NULL; - - IMB_JOB * (*submit_fn)(MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; - IMB_JOB * (*flush_fn)(MB_MGR_AES_OOO *state) = NULL; - - if (run_check) { - /* validate jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - /* validate job */ - if (is_job_invalid(state, job, - IMB_CIPHER_CBC, IMB_AUTH_NULL, - IMB_DIR_ENCRYPT, key_size)) { - job->status = IMB_STATUS_INVALID_ARGS; - return 0; - } - } - } - - if (key_size == 16) { - aes_ooo = state->aes128_ooo; - submit_fn = SUBMIT_JOB_AES128_ENC; - flush_fn = FLUSH_JOB_AES128_ENC; - } else if (key_size == 24) { - aes_ooo = state->aes192_ooo; - submit_fn = SUBMIT_JOB_AES192_ENC; - flush_fn = FLUSH_JOB_AES192_ENC; - } else { /* assume 32 */ - aes_ooo = state->aes256_ooo; - submit_fn = SUBMIT_JOB_AES256_ENC; - flush_fn = FLUSH_JOB_AES256_ENC; - } - - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - job = submit_fn(aes_ooo, job); - if (job != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - } - - if (completed_jobs != n_jobs) { - IMB_JOB *job = NULL; - - while((job = flush_fn(aes_ooo)) != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - } - - return completed_jobs; -} - -__forceinline -uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_KEY_SIZE_BYTES key_size, - const int run_check) -{ - uint32_t i, completed_jobs = 0; - void (*submit_fn) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = NULL; - (void) state; - - if (run_check) { - /* validate jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - /* validate job */ - if (is_job_invalid(state, job, - IMB_CIPHER_CBC, IMB_AUTH_NULL, - IMB_DIR_DECRYPT, key_size)) { - job->status = IMB_STATUS_INVALID_ARGS; - return 0; - } - } - } - - if (key_size == 16) - submit_fn = AES_CBC_DEC_128; - else if (key_size == 24) - submit_fn = AES_CBC_DEC_192; - else /* assume 32 */ - submit_fn = AES_CBC_DEC_256; - - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - submit_fn(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - - return completed_jobs; -} - -__forceinline -uint32_t submit_aes_ctr_burst(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_KEY_SIZE_BYTES key_size, - const int run_check) -{ - uint32_t i, completed_jobs = 0; - - if (run_check) { - /* validate jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - /* validate job */ - if (is_job_invalid(state, job, - IMB_CIPHER_CNTR, IMB_AUTH_NULL, - IMB_DIR_ENCRYPT, key_size)) { - job->status = IMB_STATUS_INVALID_ARGS; - return 0; - } - } - } - -#ifdef AVX512 - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - void (*submit_fn_vaes) (IMB_JOB *job) = NULL; - - if (key_size == 16) - submit_fn_vaes = aes_cntr_128_submit_vaes_avx512; - else if (key_size == 24) - submit_fn_vaes = aes_cntr_192_submit_vaes_avx512; - else /* assume 32 */ - submit_fn_vaes = aes_cntr_256_submit_vaes_avx512; - - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - submit_fn_vaes(job); - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - } else { -#endif - void (*submit_fn) (const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bytes, - uint64_t iv_len_bytes) = NULL; - - if (key_size == 16) - submit_fn = AES_CNTR_128; - else if (key_size == 24) - submit_fn = AES_CNTR_192; - else /* assume 32 */ - submit_fn = AES_CNTR_256; - - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - submit_fn(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } -#ifdef AVX512 - } -#endif - return completed_jobs; -} - -__forceinline -uint32_t submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size, - const int run_check) -{ - /* reset error status */ - imb_set_errno(state, 0); - - if (run_check) - if (jobs == NULL) { - imb_set_errno(state, IMB_ERR_NULL_BURST); - return 0; - } - - switch (cipher) { - case IMB_CIPHER_CBC: - if (dir == IMB_DIR_ENCRYPT) - return submit_aes_cbc_burst_enc(state, jobs, n_jobs, - key_size, run_check); - else - return submit_aes_cbc_burst_dec(state, jobs, n_jobs, - key_size, run_check); - case IMB_CIPHER_CNTR: - return submit_aes_ctr_burst(state, jobs, n_jobs, - key_size, run_check); - default: - break; - } - - /* unsupported cipher mode */ - imb_set_errno(state, IMB_ERR_CIPH_MODE); - - return 0; -} - -uint32_t -SUBMIT_CIPHER_BURST(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size) -{ - return submit_cipher_burst_and_check(state, jobs, n_jobs, - cipher, dir, key_size, 1); -} - -uint32_t -SUBMIT_CIPHER_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size) -{ - return submit_cipher_burst_and_check(state, jobs, n_jobs, - cipher, dir, key_size, 0); -} - -__forceinline -uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check, - const IMB_HASH_ALG hash_alg, - void *ooo_mgr, - IMB_JOB *(*submit_fn)(void *, IMB_JOB *), - IMB_JOB *(*flush_fn)(void *)) -{ - uint32_t i, completed_jobs = 0; - - if (run_check) { - /* validate jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - /* validate job */ - if (is_job_invalid(state, job, - IMB_CIPHER_NULL, - hash_alg, - IMB_DIR_ENCRYPT, - job->key_len_in_bytes)) { - job->status = IMB_STATUS_INVALID_ARGS; - return 0; - } - } - } - /* submit all jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - - job = submit_fn(ooo_mgr, job); - if (job != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - } - /* flush any outstanding jobs */ - if (completed_jobs != n_jobs) { - IMB_JOB *job = NULL; - - while ((job = flush_fn(ooo_mgr)) != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; - } - } - - return completed_jobs; -} - -__forceinline -uint32_t submit_burst_hmac_sha_1(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_1, - (void *)state->hmac_sha_1_ooo, - (void *)SUBMIT_JOB_HMAC, - (void *)FLUSH_JOB_HMAC); -} - -__forceinline -uint32_t submit_burst_hmac_sha_224(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_224, - (void *)state->hmac_sha_224_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_224, - (void *)FLUSH_JOB_HMAC_SHA_224); - -} - -__forceinline -uint32_t submit_burst_hmac_sha_256(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_256, - (void *)state->hmac_sha_256_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_256, - (void *)FLUSH_JOB_HMAC_SHA_256); -} - -__forceinline -uint32_t submit_burst_hmac_sha_384(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_384, - (void *)state->hmac_sha_384_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_384, - (void *)FLUSH_JOB_HMAC_SHA_384); -} - -__forceinline -uint32_t submit_burst_hmac_sha_512(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_512, - (void *)state->hmac_sha_512_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_512, - (void *)FLUSH_JOB_HMAC_SHA_512); -} - -__forceinline -uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash, - const int run_check) -{ - /* reset error status */ - imb_set_errno(state, 0); - - if (run_check) { - if (jobs == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_JOB); - return 0; - } - } - - switch (hash) { - case IMB_AUTH_HMAC_SHA_1: - return submit_burst_hmac_sha_1(state, jobs, - n_jobs, run_check); - case IMB_AUTH_HMAC_SHA_224: - return submit_burst_hmac_sha_224(state, jobs, - n_jobs, run_check); - case IMB_AUTH_HMAC_SHA_256: - return submit_burst_hmac_sha_256(state, jobs, - n_jobs, run_check); - case IMB_AUTH_HMAC_SHA_384: - return submit_burst_hmac_sha_384(state, jobs, - n_jobs, run_check); - case IMB_AUTH_HMAC_SHA_512: - return submit_burst_hmac_sha_512(state, jobs, - n_jobs, run_check); - default: - break; - } - - /* unsupported hash alg */ - imb_set_errno(state, IMB_ERR_HASH_ALGO); - - return 0; -} - -uint32_t -SUBMIT_HASH_BURST(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash) -{ - return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 1); -} - -uint32_t -SUBMIT_HASH_BURST_NOCHECK(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash) -{ - return submit_hash_burst_and_check(state, jobs, n_jobs, hash, 0); -} - #endif /* MB_MGR_CODE_H */ -- GitLab From 2884c9284c2ef5f36148aff83fcb47907a1308f8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 10:13:04 +0000 Subject: [PATCH 324/369] lib: [MB_MGR] chained cipher and hash asynchronous burst API's moved to a separate file --- lib/include/mb_mgr_burst_async.h | 270 +++++++++++++++++++++++++++++++ lib/include/mb_mgr_code.h | 250 +++------------------------- 2 files changed, 288 insertions(+), 232 deletions(-) create mode 100644 lib/include/mb_mgr_burst_async.h diff --git a/lib/include/mb_mgr_burst_async.h b/lib/include/mb_mgr_burst_async.h new file mode 100644 index 00000000..66937645 --- /dev/null +++ b/lib/include/mb_mgr_burst_async.h @@ -0,0 +1,270 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef MB_MGR_BURST_ASYNC_H +#define MB_MGR_BURST_ASYNC_H + +/* asynchronous burst API (chained cipher & hash) */ + +#include "intel-ipsec-mb.h" +#include "include/error.h" +#include "include/mb_mgr_job_check.h" /* is_job_invalid() */ + +__forceinline +void ADV_N_JOBS(int *ptr, const uint32_t n_jobs) +{ + *ptr += (sizeof(IMB_JOB) * n_jobs); + if (*ptr >= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB))) + *ptr -= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB)); +} + + +/* get number of jobs between job_offset and the end of the queue */ +__forceinline uint32_t +get_queue_sz_end(const int job_offset) +{ + return IMB_MAX_JOBS - (job_offset / sizeof(IMB_JOB)); +} + +__forceinline uint32_t +queue_sz_remaining(IMB_MGR *state) +{ + if (state->earliest_job < 0) + return IMB_MAX_JOBS; + + return IMB_MAX_JOBS - get_queue_sz(state); +} + +uint32_t +GET_NEXT_BURST(IMB_MGR *state, const uint32_t n_req_jobs, IMB_JOB **jobs) +{ + uint32_t i, num_jobs, n_ret_jobs, filled_jobs = 0; + IMB_JOB *job = NULL; + + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + if (n_req_jobs > IMB_MAX_BURST_SIZE) { + imb_set_errno(state, IMB_ERR_BURST_SIZE); + return 0; + } +#endif + /* set number of jobs to return */ + n_ret_jobs = queue_sz_remaining(state); + if (n_ret_jobs > n_req_jobs) + n_ret_jobs = n_req_jobs; + + /* start filling list from next available job */ + job = JOBS(state, state->next_job); + + /* check enough jobs available before end of queue */ + num_jobs = get_queue_sz_end(state->next_job); + + if (num_jobs < n_ret_jobs) { + /* fill jobs to the end of the queue */ + for (i = 0; i < num_jobs; i++) { + jobs[filled_jobs++] = job; + job++; + } + /* fill remaining jobs from beginning of queue */ + num_jobs = n_ret_jobs - num_jobs; + job = &state->jobs[0]; + } else + /* fill all jobs */ + num_jobs = n_ret_jobs; + + for (i = 0; i < num_jobs; i++) { + jobs[filled_jobs++] = job; + job++; + } + + return filled_jobs; +} + +__forceinline uint32_t +submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs, const int run_check) +{ + uint32_t i, n_ret_jobs = 0, num_jobs = n_jobs; + IMB_JOB *job = NULL; + + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + int job_offset = state->next_job; + + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } + if (n_jobs > IMB_MAX_BURST_SIZE) { + imb_set_errno(state, IMB_ERR_BURST_SIZE); + return 0; + } + /* check enough space in queue */ + if (queue_sz_remaining(state) < n_jobs) { + imb_set_errno(state, IMB_ERR_QUEUE_SPACE); + return 0; + } + + for (i = 0; i < n_jobs; i++) { + if (jobs[i] == NULL) { + imb_set_errno(state, IMB_ERR_NULL_JOB); + return 0; + } + if (jobs[i] != JOBS(state, job_offset)) { + imb_set_errno(state, IMB_ERR_BURST_OOO); + goto return_invalid_job; + } + ADV_JOBS(&job_offset); + + /* validate job */ + if (is_job_invalid(state, jobs[i], + jobs[i]->cipher_mode, + jobs[i]->hash_alg, + jobs[i]->cipher_direction, + jobs[i]->key_len_in_bytes)) { + goto return_invalid_job; + } + } + } + + /* state was previously empty */ + if (state->earliest_job < 0) + state->earliest_job = state->next_job; + + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + jobs[i]->status = IMB_STATUS_BEING_PROCESSED; + submit_new_job(state, jobs[i]); + } + ADV_N_JOBS(&state->next_job, n_jobs); + + /* + * return completed jobs + * - may need 2 passes if jobs wrap in queue + */ + num_jobs = get_queue_sz_end(state->earliest_job); + if (num_jobs > n_jobs) + num_jobs = n_jobs; + + /* start returning from earliest job */ + job = JOBS(state, state->earliest_job); + +return_jobs: + for (i = 0; i < num_jobs; i++) { + if (job->status < IMB_STATUS_COMPLETED) + goto return_jobs_done; + jobs[n_ret_jobs++] = job; + job++; + } + + /* check if all jobs returned + * if not, return remaining jobs from beginning of queue + */ + if (n_ret_jobs < n_jobs) { + num_jobs = n_jobs - num_jobs; + job = &state->jobs[0]; + goto return_jobs; + } + +return_jobs_done: + ADV_N_JOBS(&state->earliest_job, n_ret_jobs); + + if (state->earliest_job == state->next_job) { + state->earliest_job = -1; /* becomes empty */ + state->next_job = 0; + } + + return n_ret_jobs; + +return_invalid_job: + jobs[i]->status = IMB_STATUS_INVALID_ARGS; + jobs[0] = jobs[i]; + return 0; +} + +uint32_t +SUBMIT_BURST(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) +{ + return submit_burst_and_check(state, n_jobs, jobs, 1); +} + +uint32_t +SUBMIT_BURST_NOCHECK(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) +{ + return submit_burst_and_check(state, n_jobs, jobs, 0); +} + +uint32_t +FLUSH_BURST(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs) +{ + uint32_t i, max_ret_jobs, n_ret_jobs = 0; + + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (jobs == NULL) { + imb_set_errno(state, IMB_ERR_NULL_BURST); + return 0; + } +#endif + /* check if any jobs in queue */ + max_ret_jobs = queue_sz(state); + if (max_ret_jobs == 0) + return 0; + + /* set max number of jobs to return */ + if (max_ret_jobs > max_jobs) + max_ret_jobs = max_jobs; + + for (i = 0; i < max_ret_jobs; i++) { + IMB_JOB *job = JOBS(state, state->earliest_job); + + if (job->status < IMB_STATUS_COMPLETED) + complete_job(state, job); + + jobs[n_ret_jobs++] = job; + ADV_JOBS(&state->earliest_job); + } + + if (state->earliest_job == state->next_job) { + state->earliest_job = -1; /* becomes empty */ + state->next_job = 0; + } + + return n_ret_jobs; +} + +#endif /* MB_MGR_BURST_ASYNC_H */ diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 575fad28..9b7a427a 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -51,7 +51,6 @@ #include "include/job_api_snowv.h" #include "include/job_api_kasumi.h" #include "include/mb_mgr_job_check.h" /* is_job_invalid() */ -#include "include/mb_mgr_burst.h" /* synchronous cipher/hash burst API */ #ifdef LINUX #define BSWAP64 __builtin_bswap64 @@ -83,14 +82,6 @@ void ADV_JOBS(int *ptr) *ptr = 0; } -__forceinline -void ADV_N_JOBS(int *ptr, const uint32_t n_jobs) -{ - *ptr += (sizeof(IMB_JOB) * n_jobs); - if (*ptr >= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB))) - *ptr -= (int) (IMB_MAX_JOBS * sizeof(IMB_JOB)); -} - /* ========================================================================= */ /* Lower level "out of order" schedulers */ /* ========================================================================= */ @@ -1097,13 +1088,6 @@ FLUSH_JOB(IMB_MGR *state) return job; } -/* get number of jobs between job_offset and the end of the queue */ -__forceinline uint32_t -get_queue_sz_end(const int job_offset) -{ - return IMB_MAX_JOBS - (job_offset / sizeof(IMB_JOB)); -} - __forceinline uint32_t get_queue_sz(IMB_MGR *state) { @@ -1121,20 +1105,9 @@ queue_sz(IMB_MGR *state) return get_queue_sz(state); } -__forceinline uint32_t -queue_sz_remaining(IMB_MGR *state) -{ - if (state->earliest_job < 0) - return IMB_MAX_JOBS; - - return IMB_MAX_JOBS - get_queue_sz(state); -} - - /* ========================================================================= */ /* ========================================================================= */ - uint32_t QUEUE_SIZE(IMB_MGR *state) { @@ -1195,212 +1168,25 @@ GET_NEXT_JOB(IMB_MGR *state) return JOBS(state, state->next_job); } -uint32_t -GET_NEXT_BURST(IMB_MGR *state, const uint32_t n_req_jobs, IMB_JOB **jobs) -{ - uint32_t i, num_jobs, n_ret_jobs, filled_jobs = 0; - IMB_JOB *job = NULL; - - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(state, IMB_ERR_NULL_BURST); - return 0; - } - if (n_req_jobs > IMB_MAX_BURST_SIZE) { - imb_set_errno(state, IMB_ERR_BURST_SIZE); - return 0; - } -#endif - /* set number of jobs to return */ - n_ret_jobs = queue_sz_remaining(state); - if (n_ret_jobs > n_req_jobs) - n_ret_jobs = n_req_jobs; - - /* start filling list from next available job */ - job = JOBS(state, state->next_job); - - /* check enough jobs available before end of queue */ - num_jobs = get_queue_sz_end(state->next_job); - - if (num_jobs < n_ret_jobs) { - /* fill jobs to the end of the queue */ - for (i = 0; i < num_jobs; i++) { - jobs[filled_jobs++] = job; - job++; - } - /* fill remaining jobs from beginning of queue */ - num_jobs = n_ret_jobs - num_jobs; - job = &state->jobs[0]; - } else - /* fill all jobs */ - num_jobs = n_ret_jobs; - - for (i = 0; i < num_jobs; i++) { - jobs[filled_jobs++] = job; - job++; - } - - return filled_jobs; -} - -__forceinline uint32_t -submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, - IMB_JOB **jobs, const int run_check) -{ - uint32_t i, n_ret_jobs = 0, num_jobs = n_jobs; - IMB_JOB *job = NULL; - - /* reset error status */ - imb_set_errno(state, 0); - - if (run_check) { - int job_offset = state->next_job; - - if (jobs == NULL) { - imb_set_errno(state, IMB_ERR_NULL_BURST); - return 0; - } - if (n_jobs > IMB_MAX_BURST_SIZE) { - imb_set_errno(state, IMB_ERR_BURST_SIZE); - return 0; - } - /* check enough space in queue */ - if (queue_sz_remaining(state) < n_jobs) { - imb_set_errno(state, IMB_ERR_QUEUE_SPACE); - return 0; - } - - for (i = 0; i < n_jobs; i++) { - if (jobs[i] == NULL) { - imb_set_errno(state, IMB_ERR_NULL_JOB); - return 0; - } - if (jobs[i] != JOBS(state, job_offset)) { - imb_set_errno(state, IMB_ERR_BURST_OOO); - goto return_invalid_job; - } - ADV_JOBS(&job_offset); - - /* validate job */ - if (is_job_invalid(state, jobs[i], - jobs[i]->cipher_mode, - jobs[i]->hash_alg, - jobs[i]->cipher_direction, - jobs[i]->key_len_in_bytes)) { - goto return_invalid_job; - } - } - } - - /* state was previously empty */ - if (state->earliest_job < 0) - state->earliest_job = state->next_job; - - /* submit all jobs */ - for (i = 0; i < n_jobs; i++) { - jobs[i]->status = IMB_STATUS_BEING_PROCESSED; - submit_new_job(state, jobs[i]); - } - ADV_N_JOBS(&state->next_job, n_jobs); - - /* - * return completed jobs - * - may need 2 passes if jobs wrap in queue - */ - num_jobs = get_queue_sz_end(state->earliest_job); - if (num_jobs > n_jobs) - num_jobs = n_jobs; - - /* start returning from earliest job */ - job = JOBS(state, state->earliest_job); - -return_jobs: - for (i = 0; i < num_jobs; i++) { - if (job->status < IMB_STATUS_COMPLETED) - goto return_jobs_done; - jobs[n_ret_jobs++] = job; - job++; - } - - /* check if all jobs returned - * if not, return remaining jobs from beginning of queue - */ - if (n_ret_jobs < n_jobs) { - num_jobs = n_jobs - num_jobs; - job = &state->jobs[0]; - goto return_jobs; - } - -return_jobs_done: - ADV_N_JOBS(&state->earliest_job, n_ret_jobs); - - if (state->earliest_job == state->next_job) { - state->earliest_job = -1; /* becomes empty */ - state->next_job = 0; - } - - return n_ret_jobs; - -return_invalid_job: - jobs[i]->status = IMB_STATUS_INVALID_ARGS; - jobs[0] = jobs[i]; - return 0; -} - -uint32_t -SUBMIT_BURST(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) -{ - return submit_burst_and_check(state, n_jobs, jobs, 1); -} - -uint32_t -SUBMIT_BURST_NOCHECK(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs) -{ - return submit_burst_and_check(state, n_jobs, jobs, 0); -} - -uint32_t -FLUSH_BURST(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs) -{ - uint32_t i, max_ret_jobs, n_ret_jobs = 0; - - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (jobs == NULL) { - imb_set_errno(state, IMB_ERR_NULL_BURST); - return 0; - } -#endif - /* check if any jobs in queue */ - max_ret_jobs = queue_sz(state); - if (max_ret_jobs == 0) - return 0; - - /* set max number of jobs to return */ - if (max_ret_jobs > max_jobs) - max_ret_jobs = max_jobs; - - for (i = 0; i < max_ret_jobs; i++) { - IMB_JOB *job = JOBS(state, state->earliest_job); - - if (job->status < IMB_STATUS_COMPLETED) - complete_job(state, job); - - jobs[n_ret_jobs++] = job; - ADV_JOBS(&state->earliest_job); - } +/* ========================================================================= */ +/* + * Implements: + * GET_NEXT_BURST + * SUBMIT_BURST + * SUBMIT_BURST_NOCHECK + * FLUSH_BURST + */ +#include "include/mb_mgr_burst_async.h" /* asynchronous burst API */ - if (state->earliest_job == state->next_job) { - state->earliest_job = -1; /* becomes empty */ - state->next_job = 0; - } +/* ========================================================================= */ +/* + * Implements: + * SUBMIT_CIPHER_BURST + * SUBMIT_CIPHER_BURST_NOCHECK + * SUBMIT_HASH_BURST + * SUBMIT_HASH_BURST_NOCHECK + */ - return n_ret_jobs; -} +#include "include/mb_mgr_burst.h" /* synchronous cipher/hash burst API */ #endif /* MB_MGR_CODE_H */ -- GitLab From 847206585630b9b96941c71ceea999274d2d94b1 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 12:37:09 +0000 Subject: [PATCH 325/369] lib: [MB_MGR] JOB API's moved to a separate file --- lib/include/mb_mgr_code.h | 1110 +-------------------------------- lib/include/mb_mgr_job_api.h | 1127 ++++++++++++++++++++++++++++++++++ 2 files changed, 1137 insertions(+), 1100 deletions(-) create mode 100644 lib/include/mb_mgr_job_api.h diff --git a/lib/include/mb_mgr_code.h b/lib/include/mb_mgr_code.h index 9b7a427a..1d689355 100644 --- a/lib/include/mb_mgr_code.h +++ b/lib/include/mb_mgr_code.h @@ -28,44 +28,11 @@ #ifndef MB_MGR_CODE_H #define MB_MGR_CODE_H -/* - * This contains the bulk of the mb_mgr code, with #define's to build - * an SSE, AVX, AVX2 or AVX512 version (see mb_mgr_sse.c, mb_mgr_avx.c, etc.) - * - * get_next_job() returns a job object. This must be filled in and returned - * via submit_job() before get_next_job() is called again. - * - * submit_job() and flush_job() returns a job object. This job object ceases - * to be usable at the next call to get_next_job() - */ - #include -#include /* memcpy(), memset() */ -#include "include/clear_regs_mem.h" -#include "include/des.h" #include "intel-ipsec-mb.h" #include "include/error.h" -#include "include/snow3g_submit.h" -#include "include/job_api_gcm.h" -#include "include/job_api_snowv.h" -#include "include/job_api_kasumi.h" -#include "include/mb_mgr_job_check.h" /* is_job_invalid() */ - -#ifdef LINUX -#define BSWAP64 __builtin_bswap64 -#else -#define BSWAP64 _byteswap_uint64 -#endif -#define CRC(func, state, job) *((uint32_t *)job->auth_tag_output) = \ - func(state, job->src + job->hash_start_src_offset_in_bytes, \ - job->msg_len_to_hash_in_bytes) -/* - * JOBS() and ADV_JOBS() moved into mb_mgr_code.h - * get_next_job() and get_completed_job() API's are no longer inlines. - * For binary compatibility they have been made proper symbols. - */ __forceinline IMB_JOB *JOBS(IMB_MGR *state, const int offset) { @@ -82,1012 +49,6 @@ void ADV_JOBS(int *ptr) *ptr = 0; } -/* ========================================================================= */ -/* Lower level "out of order" schedulers */ -/* ========================================================================= */ - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES128_DEC(IMB_JOB *job) -{ - AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES192_DEC(IMB_JOB *job) -{ - AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES256_DEC(IMB_JOB *job) -{ - AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) -{ - AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_192_ENC(IMB_JOB *job) -{ - AES_ECB_ENC_192(job->src + job->cipher_start_src_offset_in_bytes, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_256_ENC(IMB_JOB *job) -{ - AES_ECB_ENC_256(job->src + job->cipher_start_src_offset_in_bytes, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_128_DEC(IMB_JOB *job) -{ - AES_ECB_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_192_DEC(IMB_JOB *job) -{ - AES_ECB_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) -{ - AES_ECB_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) -{ - AES_CBCS_1_9_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15), - job->cipher_fields.CBCS.next_iv); - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ========================================================================= */ -/* DOCSIS - it has to be below AES DEC */ -/* ========================================================================= */ - -#include "include/job_api_docsis.h" - -/* ========================================================================= */ -/* Custom hash / cipher */ -/* ========================================================================= */ - -__forceinline -IMB_JOB * -JOB_CUSTOM_CIPHER(IMB_JOB *job) -{ - if (!(job->status & IMB_STATUS_COMPLETED_CIPHER)) { - if (job->cipher_func(job)) - job->status = IMB_STATUS_INTERNAL_ERROR; - else - job->status |= IMB_STATUS_COMPLETED_CIPHER; - } - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_CUSTOM_CIPHER(IMB_JOB *job) -{ - return JOB_CUSTOM_CIPHER(job); -} - -__forceinline -IMB_JOB * -FLUSH_JOB_CUSTOM_CIPHER(IMB_JOB *job) -{ - return JOB_CUSTOM_CIPHER(job); -} - -__forceinline -IMB_JOB * -JOB_CUSTOM_HASH(IMB_JOB *job) -{ - if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { - if (job->hash_func(job)) - job->status = IMB_STATUS_INTERNAL_ERROR; - else - job->status |= IMB_STATUS_COMPLETED_AUTH; - } - return job; -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_CUSTOM_HASH(IMB_JOB *job) -{ - return JOB_CUSTOM_HASH(job); -} - -__forceinline -IMB_JOB * -FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) -{ - return JOB_CUSTOM_HASH(job); -} - -/* ========================================================================= */ -/* Cipher submit & flush functions */ -/* ========================================================================= */ -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) -{ - if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - - return SUBMIT_JOB_AES128_ENC(aes128_ooo, job); - } else if (24 == job->key_len_in_bytes) { - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - - return SUBMIT_JOB_AES192_ENC(aes192_ooo, job); - } else { /* assume 32 */ - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - - return SUBMIT_JOB_AES256_ENC(aes256_ooo, job); - } - } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR(job); - } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR_BIT(job); - } else if (IMB_CIPHER_ECB == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES_ECB_128_ENC(job); - } else if (24 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES_ECB_192_ENC(job); - } else { /* assume 32 */ - return SUBMIT_JOB_AES_ECB_256_ENC(job); - } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return submit_docsis_enc_job(state, job); - } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { - if (job->msg_len_to_cipher_in_bytes == 0) - return SUBMIT_JOB_PON_ENC_NO_CTR(job); - else - return SUBMIT_JOB_PON_ENC(job); - } else if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_ENC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_enc(state, job); - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { - return SUBMIT_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_DES == job->cipher_mode) { -#ifdef SUBMIT_JOB_DES_CBC_ENC - MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; - - return SUBMIT_JOB_DES_CBC_ENC(des_enc_ooo, job); -#else - return DES_CBC_ENC(job); -#endif /* SUBMIT_JOB_DES_CBC_ENC */ - } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_ENC_DEC(job); - } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_POLY1305(state, job); - } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { -#ifdef SUBMIT_JOB_DOCSIS_DES_ENC - MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; - - return SUBMIT_JOB_DOCSIS_DES_ENC(docsis_des_enc_ooo, - job); -#else - return DOCSIS_DES_ENC(job); -#endif /* SUBMIT_JOB_DOCSIS_DES_ENC */ - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { -#ifdef SUBMIT_JOB_3DES_CBC_ENC - MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; - - return SUBMIT_JOB_3DES_CBC_ENC(des3_enc_ooo, job); -#else - return DES3_CBC_ENC(job); -#endif - } else if (IMB_CIPHER_CCM == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return AES_CNTR_CCM_128(job); - } else { /* assume 32 */ - return AES_CNTR_CCM_256(job); - } - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - - return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); - } else { /* assume 32 */ - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = - state->zuc256_eea3_ooo; - - return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); - } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { -#ifdef SUBMIT_JOB_SNOW3G_UEA2 - return SUBMIT_JOB_SNOW3G_UEA2(state, job); -#else - return def_submit_snow3g_uea2_job(state, job); -#endif - } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { - return submit_kasumi_uea1_job(state, job); - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - - return SUBMIT_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo, job); - } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { - return SUBMIT_JOB_SNOW_V(job); - } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { - return submit_snow_v_aead_job(state, job); - } else { /* assume IMB_CIPHER_NULL */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) -{ - if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - - return FLUSH_JOB_AES128_ENC(aes128_ooo); - } else if (24 == job->key_len_in_bytes) { - MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - - return FLUSH_JOB_AES192_ENC(aes192_ooo); - } else { /* assume 32 */ - MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - - return FLUSH_JOB_AES256_ENC(aes256_ooo); - } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return flush_docsis_enc_job(state, job); -#ifdef FLUSH_JOB_DES_CBC_ENC - } else if (IMB_CIPHER_DES == job->cipher_mode) { - MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; - - return FLUSH_JOB_DES_CBC_ENC(des_enc_ooo); -#endif /* FLUSH_JOB_DES_CBC_ENC */ -#ifdef FLUSH_JOB_3DES_CBC_ENC - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { - MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; - - return FLUSH_JOB_3DES_CBC_ENC(des3_enc_ooo); -#endif /* FLUSH_JOB_3DES_CBC_ENC */ -#ifdef FLUSH_JOB_DOCSIS_DES_ENC - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { - MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; - - return FLUSH_JOB_DOCSIS_DES_ENC(docsis_des_enc_ooo); -#endif /* FLUSH_JOB_DOCSIS_DES_ENC */ - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { - return FLUSH_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - - return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); - } else { /* assume 32 */ - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = - state->zuc256_eea3_ooo; - - return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); - } - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { - MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; - - return FLUSH_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo); -#ifdef FLUSH_JOB_SNOW3G_UEA2 - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { - return FLUSH_JOB_SNOW3G_UEA2(state); -#endif - /** - * assume IMB_CIPHER_CNTR/CNTR_BITLEN, IMB_CIPHER_ECB, - * IMB_CIPHER_CCM, IMB_CIPHER_NULL or IMB_CIPHER_GCM - */ - } else { - return NULL; - } -} - -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) -{ - if (IMB_CIPHER_CBC == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES128_DEC(job); - } else if (24 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES192_DEC(job); - } else { /* assume 32 */ - return SUBMIT_JOB_AES256_DEC(job); - } - } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR(job); - } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR_BIT(job); - } else if (IMB_CIPHER_ECB == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES_ECB_128_DEC(job); - } else if (24 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES_ECB_192_DEC(job); - } else { /* assume 32 */ - return SUBMIT_JOB_AES_ECB_256_DEC(job); - } - } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { - return submit_docsis_dec_job(state, job); - } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { - if (job->msg_len_to_cipher_in_bytes == 0) - return SUBMIT_JOB_PON_DEC_NO_CTR(job); - else - return SUBMIT_JOB_PON_DEC(job); - } else if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_DEC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_dec(state, job); - } else if (IMB_CIPHER_DES == job->cipher_mode) { -#ifdef SUBMIT_JOB_DES_CBC_DEC - MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; - - return SUBMIT_JOB_DES_CBC_DEC(des_dec_ooo, job); -#else - (void) state; - return DES_CBC_DEC(job); -#endif /* SUBMIT_JOB_DES_CBC_DEC */ - } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_ENC_DEC(job); - } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_POLY1305(state, job); - } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { - return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); - } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { -#ifdef SUBMIT_JOB_DOCSIS_DES_DEC - MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - - return SUBMIT_JOB_DOCSIS_DES_DEC(docsis_des_dec_ooo, - job); -#else - return DOCSIS_DES_DEC(job); -#endif /* SUBMIT_JOB_DOCSIS_DES_DEC */ - } else if (IMB_CIPHER_DES3 == job->cipher_mode) { -#ifdef SUBMIT_JOB_3DES_CBC_DEC - MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; - - return SUBMIT_JOB_3DES_CBC_DEC(des3_dec_ooo, job); -#else - return DES3_CBC_DEC(job); -#endif - } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { - return SUBMIT_JOB_CUSTOM_CIPHER(job); - } else if (IMB_CIPHER_CCM == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - return AES_CNTR_CCM_128(job); - } else { /* assume 32 */ - return AES_CNTR_CCM_256(job); - } - } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - - return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); - } else { /* assume 32 */ - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = - state->zuc256_eea3_ooo; - - return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); - } - } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { -#ifdef SUBMIT_JOB_SNOW3G_UEA2 - return SUBMIT_JOB_SNOW3G_UEA2(state, job); -#else - return def_submit_snow3g_uea2_job(state, job); -#endif - } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { - return submit_kasumi_uea1_job(state, job); - } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { - return SUBMIT_JOB_AES128_CBCS_1_9_DEC(job); - } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { - return SUBMIT_JOB_SNOW_V(job); - } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { - return submit_snow_v_aead_job(state, job); - } else { - /* assume IMB_CIPHER_NULL */ - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) -{ -#ifdef FLUSH_JOB_SNOW3G_UEA2 - if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) - return FLUSH_JOB_SNOW3G_UEA2(state); -#endif - -#ifdef FLUSH_JOB_DES_CBC_DEC - if (IMB_CIPHER_DES == job->cipher_mode) { - MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; - - return FLUSH_JOB_DES_CBC_DEC(des_dec_ooo); - } -#endif /* FLUSH_JOB_DES_CBC_DEC */ - -#ifdef FLUSH_JOB_3DES_CBC_DEC - if (IMB_CIPHER_DES3 == job->cipher_mode) { - MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; - - return FLUSH_JOB_3DES_CBC_DEC(des3_dec_ooo); - } -#endif /* FLUSH_JOB_3DES_CBC_DEC */ - -#ifdef FLUSH_JOB_DOCSIS_DES_DEC - - if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { - MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; - - return FLUSH_JOB_DOCSIS_DES_DEC(docsis_des_dec_ooo); - } -#endif /* FLUSH_JOB_DOCSIS_DES_DEC */ - - if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { - if (16 == job->key_len_in_bytes) { - MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; - - return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); - } else { /* assume 32 */ - MB_MGR_ZUC_OOO *zuc256_eea3_ooo = - state->zuc256_eea3_ooo; - - return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); - } - } - - return NULL; -} - -/* ========================================================================= */ -/* Hash submit & flush functions */ -/* ========================================================================= */ - -__forceinline -IMB_JOB * -SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; - MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; - MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if defined (SSE) || defined (AVX512) - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; -#endif - - - switch (job->hash_alg) { - case IMB_AUTH_HMAC_SHA_1: - return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); - case IMB_AUTH_HMAC_SHA_224: - return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); - case IMB_AUTH_HMAC_SHA_256: - return SUBMIT_JOB_HMAC_SHA_256(hmac_sha_256_ooo, job); - case IMB_AUTH_HMAC_SHA_384: - return SUBMIT_JOB_HMAC_SHA_384(hmac_sha_384_ooo, job); - case IMB_AUTH_HMAC_SHA_512: - return SUBMIT_JOB_HMAC_SHA_512(hmac_sha_512_ooo, job); - case IMB_AUTH_AES_XCBC: - return SUBMIT_JOB_AES_XCBC(aes_xcbc_ooo, job); - case IMB_AUTH_MD5: - return SUBMIT_JOB_HMAC_MD5(hmac_md5_ooo, job); - case IMB_AUTH_CUSTOM: - return SUBMIT_JOB_CUSTOM_HASH(job); - case IMB_AUTH_AES_CCM: - if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES128_CCM_AUTH(aes_ccm_ooo, job); - } else { /* assume 32 */ - return SUBMIT_JOB_AES256_CCM_AUTH(aes256_ccm_ooo, job); - } - case IMB_AUTH_AES_CMAC: - /* - * CMAC OOO MGR assumes job len in bits - * (for CMAC length is provided in bytes) - */ - job->msg_len_to_hash_in_bits = - job->msg_len_to_hash_in_bytes * 8; - return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); - case IMB_AUTH_AES_CMAC_BITLEN: - return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); - case IMB_AUTH_AES_CMAC_256: - job->msg_len_to_hash_in_bits = - job->msg_len_to_hash_in_bytes * 8; - return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); - case IMB_AUTH_SHA_1: - return SUBMIT_JOB_SHA1(sha_1_ooo, job); - case IMB_AUTH_SHA_224: - return SUBMIT_JOB_SHA224(sha_224_ooo, job); - case IMB_AUTH_SHA_256: - return SUBMIT_JOB_SHA256(sha_256_ooo, job); - case IMB_AUTH_SHA_384: - return SUBMIT_JOB_SHA384(sha_384_ooo, job); - case IMB_AUTH_SHA_512: - return SUBMIT_JOB_SHA512(sha_512_ooo, job); - case IMB_AUTH_ZUC_EIA3_BITLEN: - return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); - case IMB_AUTH_ZUC256_EIA3_BITLEN: - return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, - job->auth_tag_output_len_in_bytes); - case IMB_AUTH_SNOW3G_UIA2_BITLEN: -#if defined (SSE) || defined (AVX512) - return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); -#else - IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) - job->u.SNOW3G_UIA2._key, - job->u.SNOW3G_UIA2._iv, - job->src + job->hash_start_src_offset_in_bytes, - job->msg_len_to_hash_in_bits, - job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; -#endif - case IMB_AUTH_KASUMI_UIA1: - IMB_KASUMI_F9_1_BUFFER(state, (const kasumi_key_sched_t *) - job->u.KASUMI_UIA1._key, - job->src + job->hash_start_src_offset_in_bytes, - (const uint32_t) job->msg_len_to_hash_in_bytes, - job->auth_tag_output); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_128: - process_gmac(state, job, IMB_KEY_128_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_192: - process_gmac(state, job, IMB_KEY_192_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_AES_GMAC_256: - process_gmac(state, job, IMB_KEY_256_BYTES); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_POLY1305: - POLY1305_MAC(job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_ETHERNET_FCS: - CRC(IMB_CRC32_ETHERNET_FCS, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_SCTP: - CRC(IMB_CRC32_SCTP, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: - CRC(IMB_CRC32_WIMAX_OFDMA_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC24_LTE_A: - CRC(IMB_CRC24_LTE_A, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC24_LTE_B: - CRC(IMB_CRC24_LTE_B, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC16_X25: - CRC(IMB_CRC16_X25, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC16_FP_DATA: - CRC(IMB_CRC16_FP_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC11_FP_HEADER: - CRC(IMB_CRC11_FP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC10_IUUP_DATA: - CRC(IMB_CRC10_IUUP_DATA, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: - CRC(IMB_CRC8_WIMAX_OFDMA_HCS, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC7_FP_HEADER: - CRC(IMB_CRC7_FP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_CRC6_IUUP_HEADER: - CRC(IMB_CRC6_IUUP_HEADER, state, job); - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - case IMB_AUTH_GHASH: - return process_ghash(state, job); - default: - /** - * assume IMB_AUTH_GCM, IMB_AUTH_PON_CRC_BIP, - * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL - */ - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - } -} - -__forceinline -IMB_JOB * -FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) -{ - MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; - MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; - MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; - MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; - MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; - MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; - MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; - MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; - MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; - MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; - MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; - MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; - MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; - MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; - MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; - MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if defined(SSE) || defined (AVX512) - MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; -#endif - - switch (job->hash_alg) { - case IMB_AUTH_HMAC_SHA_1: - return FLUSH_JOB_HMAC(hmac_sha_1_ooo); - case IMB_AUTH_HMAC_SHA_224: - return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); - case IMB_AUTH_HMAC_SHA_256: - return FLUSH_JOB_HMAC_SHA_256(hmac_sha_256_ooo); - case IMB_AUTH_HMAC_SHA_384: - return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); - case IMB_AUTH_HMAC_SHA_512: - return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); - case IMB_AUTH_SHA_1: - return FLUSH_JOB_SHA1(sha_1_ooo, job); - case IMB_AUTH_SHA_224: - return FLUSH_JOB_SHA224(sha_224_ooo, job); - case IMB_AUTH_SHA_256: - return FLUSH_JOB_SHA256(sha_256_ooo, job); - case IMB_AUTH_SHA_384: - return FLUSH_JOB_SHA384(sha_384_ooo, job); - case IMB_AUTH_SHA_512: - return FLUSH_JOB_SHA512(sha_512_ooo, job); - case IMB_AUTH_AES_XCBC: - return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); - case IMB_AUTH_MD5: - return FLUSH_JOB_HMAC_MD5(hmac_md5_ooo); - case IMB_AUTH_CUSTOM: - return FLUSH_JOB_CUSTOM_HASH(job); - case IMB_AUTH_AES_CCM: - if (16 == job->key_len_in_bytes) { - return FLUSH_JOB_AES128_CCM_AUTH(aes_ccm_ooo); - } else { /* assume 32 */ - return FLUSH_JOB_AES256_CCM_AUTH(aes256_ccm_ooo); - } - case IMB_AUTH_AES_CMAC: - case IMB_AUTH_AES_CMAC_BITLEN: - return FLUSH_JOB_AES128_CMAC_AUTH(aes_cmac_ooo); - case IMB_AUTH_AES_CMAC_256: - return FLUSH_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo); - case IMB_AUTH_ZUC_EIA3_BITLEN: - return FLUSH_JOB_ZUC_EIA3(zuc_eia3_ooo); - case IMB_AUTH_ZUC256_EIA3_BITLEN: - return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo, - job->auth_tag_output_len_in_bytes); -#if defined(SSE) || defined (AVX512) - case IMB_AUTH_SNOW3G_UIA2_BITLEN: - return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); -#endif - default: /* assume GCM or IMB_AUTH_NULL */ - if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { - job->status |= IMB_STATUS_COMPLETED_AUTH; - return job; - } - /* if HMAC is complete then return NULL */ - return NULL; - } -} - -/* ========================================================================= */ -/* Job submit & flush functions */ -/* ========================================================================= */ - -__forceinline -IMB_JOB *SUBMIT_JOB_AES(IMB_MGR *state, IMB_JOB *job) -{ - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = SUBMIT_JOB_AES_ENC(state, job); - else - job = SUBMIT_JOB_AES_DEC(state, job); - - return job; -} - -__forceinline -IMB_JOB *FLUSH_JOB_AES(IMB_MGR *state, IMB_JOB *job) -{ - if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = FLUSH_JOB_AES_ENC(state, job); - else - job = FLUSH_JOB_AES_DEC(state, job); - - return job; -} - -/* submit a half-completed job, based on the status */ -__forceinline -IMB_JOB *RESUBMIT_JOB(IMB_MGR *state, IMB_JOB *job) -{ - while (job != NULL && job->status < IMB_STATUS_COMPLETED) { - if (job->status == IMB_STATUS_COMPLETED_AUTH) - job = SUBMIT_JOB_AES(state, job); - else /* assumed job->status = IMB_STATUS_COMPLETED_CIPHER */ - job = SUBMIT_JOB_HASH(state, job); - } - - return job; -} - -__forceinline -IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) -{ - if (job->chain_order == IMB_ORDER_CIPHER_HASH) - job = SUBMIT_JOB_AES(state, job); - else - job = SUBMIT_JOB_HASH(state, job); - - job = RESUBMIT_JOB(state, job); - return job; -} - -__forceinline -uint32_t complete_job(IMB_MGR *state, IMB_JOB *job) -{ - uint32_t completed_jobs = 0; - - /** - * complete as many jobs as necessary - * until specified 'job' has completed - */ - if (job->chain_order == IMB_ORDER_CIPHER_HASH) { - /* while() loop optimized for cipher_hash order */ - while (job->status < IMB_STATUS_COMPLETED) { - IMB_JOB *tmp = FLUSH_JOB_AES(state, job); - - if (tmp == NULL) - tmp = FLUSH_JOB_HASH(state, job); - - (void) RESUBMIT_JOB(state, tmp); - completed_jobs++; - } - } else { - /* while() loop optimized for hash_cipher order */ - while (job->status < IMB_STATUS_COMPLETED) { - IMB_JOB *tmp = FLUSH_JOB_HASH(state, job); - - if (tmp == NULL) - tmp = FLUSH_JOB_AES(state, job); - - (void) RESUBMIT_JOB(state, tmp); - completed_jobs++; - } - } - - return completed_jobs; -} - -__forceinline -IMB_JOB * -submit_job_and_check(IMB_MGR *state, const int run_check) -{ - IMB_JOB *job = NULL; - - /* reset error status */ - imb_set_errno(state, 0); - - if (run_check) { - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } - } - -#ifndef LINUX - DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); - - SAVE_XMMS(xmm_save); -#endif - - job = JOBS(state, state->next_job); - - if (run_check) { - if (is_job_invalid(state, job, - job->cipher_mode, job->hash_alg, - job->cipher_direction, - job->key_len_in_bytes)) { - job->status = IMB_STATUS_INVALID_ARGS; - } else { - job->status = IMB_STATUS_BEING_PROCESSED; - job = submit_new_job(state, job); - } - } else { - job->status = IMB_STATUS_BEING_PROCESSED; - job = submit_new_job(state, job); - } - - if (state->earliest_job < 0) { - /* state was previously empty */ - if (job == NULL) - state->earliest_job = state->next_job; - ADV_JOBS(&state->next_job); - goto exit; - } - - ADV_JOBS(&state->next_job); - - if (state->earliest_job == state->next_job) { - /* Full */ - job = JOBS(state, state->earliest_job); - (void) complete_job(state, job); - ADV_JOBS(&state->earliest_job); - goto exit; - } - - /* not full */ - job = JOBS(state, state->earliest_job); - if (job->status < IMB_STATUS_COMPLETED) { - job = NULL; - goto exit; - } - - ADV_JOBS(&state->earliest_job); -exit: - -#ifndef LINUX - RESTORE_XMMS(xmm_save); -#endif - return job; -} - -IMB_JOB * -SUBMIT_JOB(IMB_MGR *state) -{ - return submit_job_and_check(state, 1); -} - -IMB_JOB * -SUBMIT_JOB_NOCHECK(IMB_MGR *state) -{ - return submit_job_and_check(state, 0); -} - -IMB_JOB * -FLUSH_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - IMB_JOB *job; -#ifndef LINUX - DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); -#endif - if (state->earliest_job < 0) - return NULL; /* empty */ - -#ifndef LINUX - SAVE_XMMS(xmm_save); -#endif - job = JOBS(state, state->earliest_job); - (void) complete_job(state, job); - - ADV_JOBS(&state->earliest_job); - - if (state->earliest_job == state->next_job) - state->earliest_job = -1; /* becomes empty */ - -#ifndef LINUX - RESTORE_XMMS(xmm_save); -#endif - return job; -} - __forceinline uint32_t get_queue_sz(IMB_MGR *state) { @@ -1106,67 +67,16 @@ queue_sz(IMB_MGR *state) } /* ========================================================================= */ -/* ========================================================================= */ - -uint32_t -QUEUE_SIZE(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return 0; - } -#endif - return queue_sz(state); -} - -IMB_JOB * -GET_COMPLETED_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - IMB_JOB *job; - - if (state->earliest_job < 0) - return NULL; - - job = JOBS(state, state->earliest_job); - if (job->status < IMB_STATUS_COMPLETED) - return NULL; - - ADV_JOBS(&state->earliest_job); - - if (state->earliest_job == state->next_job) - state->earliest_job = -1; - - return job; -} - -IMB_JOB * -GET_NEXT_JOB(IMB_MGR *state) -{ - /* reset error status */ - imb_set_errno(state, 0); - -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return NULL; - } -#endif - - return JOBS(state, state->next_job); -} +/* + * Implements: + * GET_NEXT_JOB + * GET_COMPLETED_JOB + * QUEUE_SIZE + * FLUSH_JOB + * SUBMIT_JOB_NOCHECK + * SUBMIT_JOB + */ +#include "include/mb_mgr_job_api.h" /* JOB API */ /* ========================================================================= */ /* diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h new file mode 100644 index 00000000..dfbe6839 --- /dev/null +++ b/lib/include/mb_mgr_job_api.h @@ -0,0 +1,1127 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef MB_MGR_JOB_API_H +#define MB_MGR_JOB_API_H + +/* + * This contains the bulk of the mb_mgr code, with #define's to build + * an SSE, AVX, AVX2 or AVX512 version (see mb_mgr_sse.c, mb_mgr_avx.c, etc.) + * + * get_next_job() returns a job object. This must be filled in and returned + * via submit_job() before get_next_job() is called again. + * + * submit_job() and flush_job() returns a job object. This job object ceases + * to be usable at the next call to get_next_job() + */ + +#include + +#include "include/clear_regs_mem.h" +#include "include/des.h" +#include "intel-ipsec-mb.h" +#include "include/error.h" +#include "include/snow3g_submit.h" +#include "include/job_api_gcm.h" +#include "include/job_api_snowv.h" +#include "include/job_api_kasumi.h" +#include "include/mb_mgr_job_check.h" /* is_job_invalid() */ + +#define CRC(func, state, job) *((uint32_t *)job->auth_tag_output) = \ + func(state, job->src + job->hash_start_src_offset_in_bytes, \ + job->msg_len_to_hash_in_bytes) + +/* ========================================================================= */ +/* Lower level "out of order" schedulers */ +/* ========================================================================= */ + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES128_DEC(IMB_JOB *job) +{ + AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES192_DEC(IMB_JOB *job) +{ + AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES256_DEC(IMB_JOB *job) +{ + AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) +{ + AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_192_ENC(IMB_JOB *job) +{ + AES_ECB_ENC_192(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_256_ENC(IMB_JOB *job) +{ + AES_ECB_ENC_256(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_128_DEC(IMB_JOB *job) +{ + AES_ECB_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_192_DEC(IMB_JOB *job) +{ + AES_ECB_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) +{ + AES_ECB_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) +{ + AES_CBCS_1_9_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & (~15), + job->cipher_fields.CBCS.next_iv); + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +/* ========================================================================= */ +/* DOCSIS - it has to be below AES DEC */ +/* ========================================================================= */ + +#include "include/job_api_docsis.h" + +/* ========================================================================= */ +/* Custom hash / cipher */ +/* ========================================================================= */ + +__forceinline +IMB_JOB * +JOB_CUSTOM_CIPHER(IMB_JOB *job) +{ + if (!(job->status & IMB_STATUS_COMPLETED_CIPHER)) { + if (job->cipher_func(job)) + job->status = IMB_STATUS_INTERNAL_ERROR; + else + job->status |= IMB_STATUS_COMPLETED_CIPHER; + } + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_CUSTOM_CIPHER(IMB_JOB *job) +{ + return JOB_CUSTOM_CIPHER(job); +} + +__forceinline +IMB_JOB * +FLUSH_JOB_CUSTOM_CIPHER(IMB_JOB *job) +{ + return JOB_CUSTOM_CIPHER(job); +} + +__forceinline +IMB_JOB * +JOB_CUSTOM_HASH(IMB_JOB *job) +{ + if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { + if (job->hash_func(job)) + job->status = IMB_STATUS_INTERNAL_ERROR; + else + job->status |= IMB_STATUS_COMPLETED_AUTH; + } + return job; +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_CUSTOM_HASH(IMB_JOB *job) +{ + return JOB_CUSTOM_HASH(job); +} + +__forceinline +IMB_JOB * +FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) +{ + return JOB_CUSTOM_HASH(job); +} + +/* ========================================================================= */ +/* Cipher submit & flush functions */ +/* ========================================================================= */ +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) +{ + if (IMB_CIPHER_CBC == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; + + return SUBMIT_JOB_AES128_ENC(aes128_ooo, job); + } else if (24 == job->key_len_in_bytes) { + MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; + + return SUBMIT_JOB_AES192_ENC(aes192_ooo, job); + } else { /* assume 32 */ + MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; + + return SUBMIT_JOB_AES256_ENC(aes256_ooo, job); + } + } else if (IMB_CIPHER_CNTR == job->cipher_mode) { + return SUBMIT_JOB_AES_CNTR(job); + } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { + return SUBMIT_JOB_AES_CNTR_BIT(job); + } else if (IMB_CIPHER_ECB == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES_ECB_128_ENC(job); + } else if (24 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES_ECB_192_ENC(job); + } else { /* assume 32 */ + return SUBMIT_JOB_AES_ECB_256_ENC(job); + } + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { + return submit_docsis_enc_job(state, job); + } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { + if (job->msg_len_to_cipher_in_bytes == 0) + return SUBMIT_JOB_PON_ENC_NO_CTR(job); + else + return SUBMIT_JOB_PON_ENC(job); + } else if (IMB_CIPHER_GCM == job->cipher_mode) { + return SUBMIT_JOB_AES_GCM_ENC(state, job); + } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { + return submit_gcm_sgl_enc(state, job); + } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + return SUBMIT_JOB_CUSTOM_CIPHER(job); + } else if (IMB_CIPHER_DES == job->cipher_mode) { +#ifdef SUBMIT_JOB_DES_CBC_ENC + MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; + + return SUBMIT_JOB_DES_CBC_ENC(des_enc_ooo, job); +#else + return DES_CBC_ENC(job); +#endif /* SUBMIT_JOB_DES_CBC_ENC */ + } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_ENC_DEC(job); + } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_POLY1305(state, job); + } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); + } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { +#ifdef SUBMIT_JOB_DOCSIS_DES_ENC + MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; + + return SUBMIT_JOB_DOCSIS_DES_ENC(docsis_des_enc_ooo, + job); +#else + return DOCSIS_DES_ENC(job); +#endif /* SUBMIT_JOB_DOCSIS_DES_ENC */ + } else if (IMB_CIPHER_DES3 == job->cipher_mode) { +#ifdef SUBMIT_JOB_3DES_CBC_ENC + MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; + + return SUBMIT_JOB_3DES_CBC_ENC(des3_enc_ooo, job); +#else + return DES3_CBC_ENC(job); +#endif + } else if (IMB_CIPHER_CCM == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + return AES_CNTR_CCM_128(job); + } else { /* assume 32 */ + return AES_CNTR_CCM_256(job); + } + } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; + + return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); + } else { /* assume 32 */ + MB_MGR_ZUC_OOO *zuc256_eea3_ooo = + state->zuc256_eea3_ooo; + + return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); + } + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { +#ifdef SUBMIT_JOB_SNOW3G_UEA2 + return SUBMIT_JOB_SNOW3G_UEA2(state, job); +#else + return def_submit_snow3g_uea2_job(state, job); +#endif + } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { + return submit_kasumi_uea1_job(state, job); + } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { + MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; + + return SUBMIT_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo, job); + } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { + return SUBMIT_JOB_SNOW_V(job); + } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { + return submit_snow_v_aead_job(state, job); + } else { /* assume IMB_CIPHER_NULL */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } +} + +__forceinline +IMB_JOB * +FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) +{ + if (IMB_CIPHER_CBC == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; + + return FLUSH_JOB_AES128_ENC(aes128_ooo); + } else if (24 == job->key_len_in_bytes) { + MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; + + return FLUSH_JOB_AES192_ENC(aes192_ooo); + } else { /* assume 32 */ + MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; + + return FLUSH_JOB_AES256_ENC(aes256_ooo); + } + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { + return flush_docsis_enc_job(state, job); +#ifdef FLUSH_JOB_DES_CBC_ENC + } else if (IMB_CIPHER_DES == job->cipher_mode) { + MB_MGR_DES_OOO *des_enc_ooo = state->des_enc_ooo; + + return FLUSH_JOB_DES_CBC_ENC(des_enc_ooo); +#endif /* FLUSH_JOB_DES_CBC_ENC */ +#ifdef FLUSH_JOB_3DES_CBC_ENC + } else if (IMB_CIPHER_DES3 == job->cipher_mode) { + MB_MGR_DES_OOO *des3_enc_ooo = state->des3_enc_ooo; + + return FLUSH_JOB_3DES_CBC_ENC(des3_enc_ooo); +#endif /* FLUSH_JOB_3DES_CBC_ENC */ +#ifdef FLUSH_JOB_DOCSIS_DES_ENC + } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + MB_MGR_DES_OOO *docsis_des_enc_ooo = state->docsis_des_enc_ooo; + + return FLUSH_JOB_DOCSIS_DES_ENC(docsis_des_enc_ooo); +#endif /* FLUSH_JOB_DOCSIS_DES_ENC */ + } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + return FLUSH_JOB_CUSTOM_CIPHER(job); + } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; + + return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); + } else { /* assume 32 */ + MB_MGR_ZUC_OOO *zuc256_eea3_ooo = + state->zuc256_eea3_ooo; + + return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); + } + } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { + MB_MGR_AES_OOO *aes128_cbcs_ooo = state->aes128_cbcs_ooo; + + return FLUSH_JOB_AES128_CBCS_1_9_ENC(aes128_cbcs_ooo); +#ifdef FLUSH_JOB_SNOW3G_UEA2 + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { + return FLUSH_JOB_SNOW3G_UEA2(state); +#endif + /** + * assume IMB_CIPHER_CNTR/CNTR_BITLEN, IMB_CIPHER_ECB, + * IMB_CIPHER_CCM, IMB_CIPHER_NULL or IMB_CIPHER_GCM + */ + } else { + return NULL; + } +} + +__forceinline +IMB_JOB * +SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) +{ + if (IMB_CIPHER_CBC == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES128_DEC(job); + } else if (24 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES192_DEC(job); + } else { /* assume 32 */ + return SUBMIT_JOB_AES256_DEC(job); + } + } else if (IMB_CIPHER_CNTR == job->cipher_mode) { + return SUBMIT_JOB_AES_CNTR(job); + } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { + return SUBMIT_JOB_AES_CNTR_BIT(job); + } else if (IMB_CIPHER_ECB == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES_ECB_128_DEC(job); + } else if (24 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES_ECB_192_DEC(job); + } else { /* assume 32 */ + return SUBMIT_JOB_AES_ECB_256_DEC(job); + } + } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { + return submit_docsis_dec_job(state, job); + } else if (IMB_CIPHER_PON_AES_CNTR == job->cipher_mode) { + if (job->msg_len_to_cipher_in_bytes == 0) + return SUBMIT_JOB_PON_DEC_NO_CTR(job); + else + return SUBMIT_JOB_PON_DEC(job); + } else if (IMB_CIPHER_GCM == job->cipher_mode) { + return SUBMIT_JOB_AES_GCM_DEC(state, job); + } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { + return submit_gcm_sgl_dec(state, job); + } else if (IMB_CIPHER_DES == job->cipher_mode) { +#ifdef SUBMIT_JOB_DES_CBC_DEC + MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; + + return SUBMIT_JOB_DES_CBC_DEC(des_dec_ooo, job); +#else + (void) state; + return DES_CBC_DEC(job); +#endif /* SUBMIT_JOB_DES_CBC_DEC */ + } else if (IMB_CIPHER_CHACHA20 == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_ENC_DEC(job); + } else if (IMB_CIPHER_CHACHA20_POLY1305 == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_POLY1305(state, job); + } else if (IMB_CIPHER_CHACHA20_POLY1305_SGL == job->cipher_mode) { + return SUBMIT_JOB_CHACHA20_POLY1305_SGL(state, job); + } else if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { +#ifdef SUBMIT_JOB_DOCSIS_DES_DEC + MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; + + return SUBMIT_JOB_DOCSIS_DES_DEC(docsis_des_dec_ooo, + job); +#else + return DOCSIS_DES_DEC(job); +#endif /* SUBMIT_JOB_DOCSIS_DES_DEC */ + } else if (IMB_CIPHER_DES3 == job->cipher_mode) { +#ifdef SUBMIT_JOB_3DES_CBC_DEC + MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; + + return SUBMIT_JOB_3DES_CBC_DEC(des3_dec_ooo, job); +#else + return DES3_CBC_DEC(job); +#endif + } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { + return SUBMIT_JOB_CUSTOM_CIPHER(job); + } else if (IMB_CIPHER_CCM == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + return AES_CNTR_CCM_128(job); + } else { /* assume 32 */ + return AES_CNTR_CCM_256(job); + } + } else if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; + + return SUBMIT_JOB_ZUC_EEA3(zuc_eea3_ooo, job); + } else { /* assume 32 */ + MB_MGR_ZUC_OOO *zuc256_eea3_ooo = + state->zuc256_eea3_ooo; + + return SUBMIT_JOB_ZUC256_EEA3(zuc256_eea3_ooo, job); + } + } else if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) { +#ifdef SUBMIT_JOB_SNOW3G_UEA2 + return SUBMIT_JOB_SNOW3G_UEA2(state, job); +#else + return def_submit_snow3g_uea2_job(state, job); +#endif + } else if (IMB_CIPHER_KASUMI_UEA1_BITLEN == job->cipher_mode) { + return submit_kasumi_uea1_job(state, job); + } else if (IMB_CIPHER_CBCS_1_9 == job->cipher_mode) { + return SUBMIT_JOB_AES128_CBCS_1_9_DEC(job); + } else if (IMB_CIPHER_SNOW_V == job->cipher_mode) { + return SUBMIT_JOB_SNOW_V(job); + } else if (IMB_CIPHER_SNOW_V_AEAD == job->cipher_mode) { + return submit_snow_v_aead_job(state, job); + } else { + /* assume IMB_CIPHER_NULL */ + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; + } +} + +__forceinline +IMB_JOB * +FLUSH_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) +{ +#ifdef FLUSH_JOB_SNOW3G_UEA2 + if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) + return FLUSH_JOB_SNOW3G_UEA2(state); +#endif + +#ifdef FLUSH_JOB_DES_CBC_DEC + if (IMB_CIPHER_DES == job->cipher_mode) { + MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; + + return FLUSH_JOB_DES_CBC_DEC(des_dec_ooo); + } +#endif /* FLUSH_JOB_DES_CBC_DEC */ + +#ifdef FLUSH_JOB_3DES_CBC_DEC + if (IMB_CIPHER_DES3 == job->cipher_mode) { + MB_MGR_DES_OOO *des3_dec_ooo = state->des3_dec_ooo; + + return FLUSH_JOB_3DES_CBC_DEC(des3_dec_ooo); + } +#endif /* FLUSH_JOB_3DES_CBC_DEC */ + +#ifdef FLUSH_JOB_DOCSIS_DES_DEC + + if (IMB_CIPHER_DOCSIS_DES == job->cipher_mode) { + MB_MGR_DES_OOO *docsis_des_dec_ooo = state->docsis_des_dec_ooo; + + return FLUSH_JOB_DOCSIS_DES_DEC(docsis_des_dec_ooo); + } +#endif /* FLUSH_JOB_DOCSIS_DES_DEC */ + + if (IMB_CIPHER_ZUC_EEA3 == job->cipher_mode) { + if (16 == job->key_len_in_bytes) { + MB_MGR_ZUC_OOO *zuc_eea3_ooo = state->zuc_eea3_ooo; + + return FLUSH_JOB_ZUC_EEA3(zuc_eea3_ooo); + } else { /* assume 32 */ + MB_MGR_ZUC_OOO *zuc256_eea3_ooo = + state->zuc256_eea3_ooo; + + return FLUSH_JOB_ZUC256_EEA3(zuc256_eea3_ooo); + } + } + + return NULL; +} + +/* ========================================================================= */ +/* Hash submit & flush functions */ +/* ========================================================================= */ + +__forceinline +IMB_JOB * +SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; + MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; + MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; + MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; + MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; + MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; + MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; + MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; + MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#if defined (SSE) || defined (AVX512) + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif + + + switch (job->hash_alg) { + case IMB_AUTH_HMAC_SHA_1: + return SUBMIT_JOB_HMAC(hmac_sha_1_ooo, job); + case IMB_AUTH_HMAC_SHA_224: + return SUBMIT_JOB_HMAC_SHA_224(hmac_sha_224_ooo, job); + case IMB_AUTH_HMAC_SHA_256: + return SUBMIT_JOB_HMAC_SHA_256(hmac_sha_256_ooo, job); + case IMB_AUTH_HMAC_SHA_384: + return SUBMIT_JOB_HMAC_SHA_384(hmac_sha_384_ooo, job); + case IMB_AUTH_HMAC_SHA_512: + return SUBMIT_JOB_HMAC_SHA_512(hmac_sha_512_ooo, job); + case IMB_AUTH_AES_XCBC: + return SUBMIT_JOB_AES_XCBC(aes_xcbc_ooo, job); + case IMB_AUTH_MD5: + return SUBMIT_JOB_HMAC_MD5(hmac_md5_ooo, job); + case IMB_AUTH_CUSTOM: + return SUBMIT_JOB_CUSTOM_HASH(job); + case IMB_AUTH_AES_CCM: + if (16 == job->key_len_in_bytes) { + return SUBMIT_JOB_AES128_CCM_AUTH(aes_ccm_ooo, job); + } else { /* assume 32 */ + return SUBMIT_JOB_AES256_CCM_AUTH(aes256_ccm_ooo, job); + } + case IMB_AUTH_AES_CMAC: + /* + * CMAC OOO MGR assumes job len in bits + * (for CMAC length is provided in bytes) + */ + job->msg_len_to_hash_in_bits = + job->msg_len_to_hash_in_bytes * 8; + return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); + case IMB_AUTH_AES_CMAC_BITLEN: + return SUBMIT_JOB_AES128_CMAC_AUTH(aes_cmac_ooo, job); + case IMB_AUTH_AES_CMAC_256: + job->msg_len_to_hash_in_bits = + job->msg_len_to_hash_in_bytes * 8; + return SUBMIT_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo, job); + case IMB_AUTH_SHA_1: + return SUBMIT_JOB_SHA1(sha_1_ooo, job); + case IMB_AUTH_SHA_224: + return SUBMIT_JOB_SHA224(sha_224_ooo, job); + case IMB_AUTH_SHA_256: + return SUBMIT_JOB_SHA256(sha_256_ooo, job); + case IMB_AUTH_SHA_384: + return SUBMIT_JOB_SHA384(sha_384_ooo, job); + case IMB_AUTH_SHA_512: + return SUBMIT_JOB_SHA512(sha_512_ooo, job); + case IMB_AUTH_ZUC_EIA3_BITLEN: + return SUBMIT_JOB_ZUC_EIA3(zuc_eia3_ooo, job); + case IMB_AUTH_ZUC256_EIA3_BITLEN: + return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, + job->auth_tag_output_len_in_bytes); + case IMB_AUTH_SNOW3G_UIA2_BITLEN: +#if defined (SSE) || defined (AVX512) + return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); +#else + IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) + job->u.SNOW3G_UIA2._key, + job->u.SNOW3G_UIA2._iv, + job->src + job->hash_start_src_offset_in_bytes, + job->msg_len_to_hash_in_bits, + job->auth_tag_output); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; +#endif + case IMB_AUTH_KASUMI_UIA1: + IMB_KASUMI_F9_1_BUFFER(state, (const kasumi_key_sched_t *) + job->u.KASUMI_UIA1._key, + job->src + job->hash_start_src_offset_in_bytes, + (const uint32_t) job->msg_len_to_hash_in_bytes, + job->auth_tag_output); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_128: + process_gmac(state, job, IMB_KEY_128_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_192: + process_gmac(state, job, IMB_KEY_192_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_AES_GMAC_256: + process_gmac(state, job, IMB_KEY_256_BYTES); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_POLY1305: + POLY1305_MAC(job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_ETHERNET_FCS: + CRC(IMB_CRC32_ETHERNET_FCS, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_SCTP: + CRC(IMB_CRC32_SCTP, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC32_WIMAX_OFDMA_DATA: + CRC(IMB_CRC32_WIMAX_OFDMA_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC24_LTE_A: + CRC(IMB_CRC24_LTE_A, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC24_LTE_B: + CRC(IMB_CRC24_LTE_B, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC16_X25: + CRC(IMB_CRC16_X25, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC16_FP_DATA: + CRC(IMB_CRC16_FP_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC11_FP_HEADER: + CRC(IMB_CRC11_FP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC10_IUUP_DATA: + CRC(IMB_CRC10_IUUP_DATA, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC8_WIMAX_OFDMA_HCS: + CRC(IMB_CRC8_WIMAX_OFDMA_HCS, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC7_FP_HEADER: + CRC(IMB_CRC7_FP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_CRC6_IUUP_HEADER: + CRC(IMB_CRC6_IUUP_HEADER, state, job); + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + case IMB_AUTH_GHASH: + return process_ghash(state, job); + default: + /** + * assume IMB_AUTH_GCM, IMB_AUTH_PON_CRC_BIP, + * IMB_AUTH_SNOW_V_AEAD or IMB_AUTH_NULL + */ + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + } +} + +__forceinline +IMB_JOB * +FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_HMAC_SHA_1_OOO *hmac_sha_1_ooo = state->hmac_sha_1_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_224_ooo = state->hmac_sha_224_ooo; + MB_MGR_HMAC_SHA_256_OOO *hmac_sha_256_ooo = state->hmac_sha_256_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_384_ooo = state->hmac_sha_384_ooo; + MB_MGR_HMAC_SHA_512_OOO *hmac_sha_512_ooo = state->hmac_sha_512_ooo; + MB_MGR_HMAC_MD5_OOO *hmac_md5_ooo = state->hmac_md5_ooo; + MB_MGR_AES_XCBC_OOO *aes_xcbc_ooo = state->aes_xcbc_ooo; + MB_MGR_CCM_OOO *aes_ccm_ooo = state->aes_ccm_ooo; + MB_MGR_CCM_OOO *aes256_ccm_ooo = state->aes256_ccm_ooo; + MB_MGR_CMAC_OOO *aes_cmac_ooo = state->aes_cmac_ooo; + MB_MGR_CMAC_OOO *aes256_cmac_ooo = state->aes256_cmac_ooo; + MB_MGR_ZUC_OOO *zuc_eia3_ooo = state->zuc_eia3_ooo; + MB_MGR_ZUC_OOO *zuc256_eia3_ooo = state->zuc256_eia3_ooo; + MB_MGR_SHA_1_OOO *sha_1_ooo = state->sha_1_ooo; + MB_MGR_SHA_256_OOO *sha_224_ooo = state->sha_224_ooo; + MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; + MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; + MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; +#if defined(SSE) || defined (AVX512) + MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; +#endif + + switch (job->hash_alg) { + case IMB_AUTH_HMAC_SHA_1: + return FLUSH_JOB_HMAC(hmac_sha_1_ooo); + case IMB_AUTH_HMAC_SHA_224: + return FLUSH_JOB_HMAC_SHA_224(hmac_sha_224_ooo); + case IMB_AUTH_HMAC_SHA_256: + return FLUSH_JOB_HMAC_SHA_256(hmac_sha_256_ooo); + case IMB_AUTH_HMAC_SHA_384: + return FLUSH_JOB_HMAC_SHA_384(hmac_sha_384_ooo); + case IMB_AUTH_HMAC_SHA_512: + return FLUSH_JOB_HMAC_SHA_512(hmac_sha_512_ooo); + case IMB_AUTH_SHA_1: + return FLUSH_JOB_SHA1(sha_1_ooo, job); + case IMB_AUTH_SHA_224: + return FLUSH_JOB_SHA224(sha_224_ooo, job); + case IMB_AUTH_SHA_256: + return FLUSH_JOB_SHA256(sha_256_ooo, job); + case IMB_AUTH_SHA_384: + return FLUSH_JOB_SHA384(sha_384_ooo, job); + case IMB_AUTH_SHA_512: + return FLUSH_JOB_SHA512(sha_512_ooo, job); + case IMB_AUTH_AES_XCBC: + return FLUSH_JOB_AES_XCBC(aes_xcbc_ooo); + case IMB_AUTH_MD5: + return FLUSH_JOB_HMAC_MD5(hmac_md5_ooo); + case IMB_AUTH_CUSTOM: + return FLUSH_JOB_CUSTOM_HASH(job); + case IMB_AUTH_AES_CCM: + if (16 == job->key_len_in_bytes) { + return FLUSH_JOB_AES128_CCM_AUTH(aes_ccm_ooo); + } else { /* assume 32 */ + return FLUSH_JOB_AES256_CCM_AUTH(aes256_ccm_ooo); + } + case IMB_AUTH_AES_CMAC: + case IMB_AUTH_AES_CMAC_BITLEN: + return FLUSH_JOB_AES128_CMAC_AUTH(aes_cmac_ooo); + case IMB_AUTH_AES_CMAC_256: + return FLUSH_JOB_AES256_CMAC_AUTH(aes256_cmac_ooo); + case IMB_AUTH_ZUC_EIA3_BITLEN: + return FLUSH_JOB_ZUC_EIA3(zuc_eia3_ooo); + case IMB_AUTH_ZUC256_EIA3_BITLEN: + return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo, + job->auth_tag_output_len_in_bytes); +#if defined(SSE) || defined (AVX512) + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); +#endif + default: /* assume GCM or IMB_AUTH_NULL */ + if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { + job->status |= IMB_STATUS_COMPLETED_AUTH; + return job; + } + /* if HMAC is complete then return NULL */ + return NULL; + } +} + +/* ========================================================================= */ +/* Job submit & flush functions */ +/* ========================================================================= */ + +__forceinline +IMB_JOB *SUBMIT_JOB_AES(IMB_MGR *state, IMB_JOB *job) +{ + if (job->cipher_direction == IMB_DIR_ENCRYPT) + job = SUBMIT_JOB_AES_ENC(state, job); + else + job = SUBMIT_JOB_AES_DEC(state, job); + + return job; +} + +__forceinline +IMB_JOB *FLUSH_JOB_AES(IMB_MGR *state, IMB_JOB *job) +{ + if (job->cipher_direction == IMB_DIR_ENCRYPT) + job = FLUSH_JOB_AES_ENC(state, job); + else + job = FLUSH_JOB_AES_DEC(state, job); + + return job; +} + +/* submit a half-completed job, based on the status */ +__forceinline +IMB_JOB *RESUBMIT_JOB(IMB_MGR *state, IMB_JOB *job) +{ + while (job != NULL && job->status < IMB_STATUS_COMPLETED) { + if (job->status == IMB_STATUS_COMPLETED_AUTH) + job = SUBMIT_JOB_AES(state, job); + else /* assumed job->status = IMB_STATUS_COMPLETED_CIPHER */ + job = SUBMIT_JOB_HASH(state, job); + } + + return job; +} + +__forceinline +IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) +{ + if (job->chain_order == IMB_ORDER_CIPHER_HASH) + job = SUBMIT_JOB_AES(state, job); + else + job = SUBMIT_JOB_HASH(state, job); + + job = RESUBMIT_JOB(state, job); + return job; +} + +__forceinline +uint32_t complete_job(IMB_MGR *state, IMB_JOB *job) +{ + uint32_t completed_jobs = 0; + + /** + * complete as many jobs as necessary + * until specified 'job' has completed + */ + if (job->chain_order == IMB_ORDER_CIPHER_HASH) { + /* while() loop optimized for cipher_hash order */ + while (job->status < IMB_STATUS_COMPLETED) { + IMB_JOB *tmp = FLUSH_JOB_AES(state, job); + + if (tmp == NULL) + tmp = FLUSH_JOB_HASH(state, job); + + (void) RESUBMIT_JOB(state, tmp); + completed_jobs++; + } + } else { + /* while() loop optimized for hash_cipher order */ + while (job->status < IMB_STATUS_COMPLETED) { + IMB_JOB *tmp = FLUSH_JOB_HASH(state, job); + + if (tmp == NULL) + tmp = FLUSH_JOB_AES(state, job); + + (void) RESUBMIT_JOB(state, tmp); + completed_jobs++; + } + } + + return completed_jobs; +} + +__forceinline +IMB_JOB * +submit_job_and_check(IMB_MGR *state, const int run_check) +{ + IMB_JOB *job = NULL; + + /* reset error status */ + imb_set_errno(state, 0); + + if (run_check) { + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return NULL; + } + } + +#ifndef LINUX + DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); + + SAVE_XMMS(xmm_save); +#endif + + job = JOBS(state, state->next_job); + + if (run_check) { + if (is_job_invalid(state, job, + job->cipher_mode, job->hash_alg, + job->cipher_direction, + job->key_len_in_bytes)) { + job->status = IMB_STATUS_INVALID_ARGS; + } else { + job->status = IMB_STATUS_BEING_PROCESSED; + job = submit_new_job(state, job); + } + } else { + job->status = IMB_STATUS_BEING_PROCESSED; + job = submit_new_job(state, job); + } + + if (state->earliest_job < 0) { + /* state was previously empty */ + if (job == NULL) + state->earliest_job = state->next_job; + ADV_JOBS(&state->next_job); + goto exit; + } + + ADV_JOBS(&state->next_job); + + if (state->earliest_job == state->next_job) { + /* Full */ + job = JOBS(state, state->earliest_job); + (void) complete_job(state, job); + ADV_JOBS(&state->earliest_job); + goto exit; + } + + /* not full */ + job = JOBS(state, state->earliest_job); + if (job->status < IMB_STATUS_COMPLETED) { + job = NULL; + goto exit; + } + + ADV_JOBS(&state->earliest_job); +exit: + +#ifndef LINUX + RESTORE_XMMS(xmm_save); +#endif + return job; +} + +IMB_JOB * +SUBMIT_JOB(IMB_MGR *state) +{ + return submit_job_and_check(state, 1); +} + +IMB_JOB * +SUBMIT_JOB_NOCHECK(IMB_MGR *state) +{ + return submit_job_and_check(state, 0); +} + +IMB_JOB * +FLUSH_JOB(IMB_MGR *state) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return NULL; + } +#endif + IMB_JOB *job; +#ifndef LINUX + DECLARE_ALIGNED(imb_uint128_t xmm_save[10], 16); +#endif + if (state->earliest_job < 0) + return NULL; /* empty */ + +#ifndef LINUX + SAVE_XMMS(xmm_save); +#endif + job = JOBS(state, state->earliest_job); + (void) complete_job(state, job); + + ADV_JOBS(&state->earliest_job); + + if (state->earliest_job == state->next_job) + state->earliest_job = -1; /* becomes empty */ + +#ifndef LINUX + RESTORE_XMMS(xmm_save); +#endif + return job; +} + +/* ========================================================================= */ +/* ========================================================================= */ + +uint32_t +QUEUE_SIZE(IMB_MGR *state) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return 0; + } +#endif + return queue_sz(state); +} + +IMB_JOB * +GET_COMPLETED_JOB(IMB_MGR *state) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return NULL; + } +#endif + IMB_JOB *job; + + if (state->earliest_job < 0) + return NULL; + + job = JOBS(state, state->earliest_job); + if (job->status < IMB_STATUS_COMPLETED) + return NULL; + + ADV_JOBS(&state->earliest_job); + + if (state->earliest_job == state->next_job) + state->earliest_job = -1; + + return job; +} + +IMB_JOB * +GET_NEXT_JOB(IMB_MGR *state) +{ + /* reset error status */ + imb_set_errno(state, 0); + +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return NULL; + } +#endif + + return JOBS(state, state->next_job); +} + +#endif /* MB_MGR_JOB_API_H */ -- GitLab From 09e807f4aacc756ca0423faf8ba2899e9985f8a7 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 12:56:19 +0000 Subject: [PATCH 326/369] lib: [SHA] moved SHA prototypes from sha_mb_mgr.h to adequate architecture header files --- lib/include/arch_avx2_type1.h | 5 ++++ lib/include/arch_avx512_type1.h | 20 ++++++++----- lib/include/arch_avx_type1.h | 17 +++++++---- lib/include/arch_sse_type1.h | 5 ++++ lib/include/arch_sse_type2.h | 5 ++++ lib/include/sha_mb_mgr.h | 53 ++++----------------------------- 6 files changed, 43 insertions(+), 62 deletions(-) diff --git a/lib/include/arch_avx2_type1.h b/lib/include/arch_avx2_type1.h index 6df5ef99..edb73f9a 100644 --- a/lib/include/arch_avx2_type1.h +++ b/lib/include/arch_avx2_type1.h @@ -33,6 +33,11 @@ #include "intel-ipsec-mb.h" #include "ipsec_ooo_mgr.h" +/* SHA */ +void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); + /* moved from MB MGR */ IMB_JOB *submit_job_zuc_eea3_avx2(MB_MGR_ZUC_OOO *state, IMB_JOB *job); diff --git a/lib/include/arch_avx512_type1.h b/lib/include/arch_avx512_type1.h index fc7a3adb..cf551921 100644 --- a/lib/include/arch_avx512_type1.h +++ b/lib/include/arch_avx512_type1.h @@ -70,10 +70,10 @@ IMB_JOB *submit_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, IMB_JOB *flush_job_zuc256_eea3_no_gfni_avx512(MB_MGR_ZUC_OOO *state); IMB_JOB *submit_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - IMB_JOB *job, - const uint64_t tag_sz); + IMB_JOB *job, + const uint64_t tag_sz); IMB_JOB *flush_job_zuc256_eia3_no_gfni_avx512(MB_MGR_ZUC_OOO *state, - const uint64_t tag_sz); + const uint64_t tag_sz); IMB_JOB *submit_job_sha1_avx512(MB_MGR_SHA_1_OOO *state, IMB_JOB *job); @@ -114,23 +114,23 @@ void aes_cmac_256_subkey_gen_avx512(const void *key_exp, void *key1, void *key2); IMB_JOB *submit_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_avx512(MB_MGR_HMAC_SHA_1_OOO *state); IMB_JOB *submit_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_224_avx512(MB_MGR_HMAC_SHA_256_OOO *state); IMB_JOB *submit_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_256_avx512(MB_MGR_HMAC_SHA_256_OOO *state); IMB_JOB *submit_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_384_avx512(MB_MGR_HMAC_SHA_512_OOO *state); IMB_JOB *submit_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state); void poly1305_mac_plain_avx512(IMB_JOB *job); @@ -151,6 +151,10 @@ submit_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB * flush_job_aes_docsis256_enc_crc32_avx512(MB_MGR_DOCSIS_AES_OOO *state); +/* SHA */ +void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +void call_sha512_x8_avx512_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); #endif /* IMB_ASM_AVX512_T1_H */ diff --git a/lib/include/arch_avx_type1.h b/lib/include/arch_avx_type1.h index 4f41568d..8a11a0f4 100644 --- a/lib/include/arch_avx_type1.h +++ b/lib/include/arch_avx_type1.h @@ -182,27 +182,27 @@ IMB_JOB *snow_v_avx(IMB_JOB *job); IMB_JOB *snow_v_aead_init_avx(IMB_JOB *job); IMB_JOB *submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state); IMB_JOB *submit_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_224_avx(MB_MGR_HMAC_SHA_256_OOO *state); IMB_JOB *submit_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_256_avx(MB_MGR_HMAC_SHA_256_OOO *state); IMB_JOB *submit_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_384_avx(MB_MGR_HMAC_SHA_512_OOO *state); IMB_JOB *submit_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_sha_512_avx(MB_MGR_HMAC_SHA_512_OOO *state); IMB_JOB *submit_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state, - IMB_JOB *job); + IMB_JOB *job); IMB_JOB *flush_job_hmac_md5_avx(MB_MGR_HMAC_MD5_OOO *state); IMB_JOB *submit_job_aes128_cmac_auth_avx(MB_MGR_CMAC_OOO *state, @@ -245,4 +245,9 @@ uint32_t crc6_iuup_header_avx(const void *msg, const uint64_t len); uint32_t crc32_wimax_ofdma_data_avx(const void *msg, const uint64_t len); uint32_t crc8_wimax_ofdma_hcs_avx(const void *msg, const uint64_t len); +/* SHA */ +void call_sha1_mult_avx_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +void call_sha512_x2_avx_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); + #endif /* IMB_ASM_AVX_T1_H */ diff --git a/lib/include/arch_sse_type1.h b/lib/include/arch_sse_type1.h index eeb0d895..baf13691 100644 --- a/lib/include/arch_sse_type1.h +++ b/lib/include/arch_sse_type1.h @@ -114,6 +114,11 @@ uint32_t crc8_wimax_ofdma_hcs_sse(const void *msg, const uint64_t len); uint32_t ethernet_fcs_sse_local(const void *msg, const uint64_t len, const void *tag_ouput); +/* SHA */ +void call_sha1_mult_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +void call_sha512_x2_sse_from_c(SHA512_ARGS *args, uint64_t size_in_blocks); + /* moved from MB MGR */ IMB_JOB *submit_job_aes128_enc_sse(MB_MGR_AES_OOO *state, IMB_JOB *job); diff --git a/lib/include/arch_sse_type2.h b/lib/include/arch_sse_type2.h index 609fb5d9..fcdf8f51 100644 --- a/lib/include/arch_sse_type2.h +++ b/lib/include/arch_sse_type2.h @@ -33,6 +33,11 @@ #include "intel-ipsec-mb.h" #include "ipsec_ooo_mgr.h" +/* SHA */ +void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, uint32_t size_in_blocks); +void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); +void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, uint32_t size_in_blocks); + /* Moved from MB MGR */ IMB_JOB *submit_job_hmac_ni_sse(MB_MGR_HMAC_SHA_1_OOO *state, diff --git a/lib/include/sha_mb_mgr.h b/lib/include/sha_mb_mgr.h index aaebfb7c..8b4ce721 100644 --- a/lib/include/sha_mb_mgr.h +++ b/lib/include/sha_mb_mgr.h @@ -28,54 +28,11 @@ #include "include/sha_generic.h" #include "ipsec_ooo_mgr.h" #include "constants.h" - -extern void call_sha1_mult_sse_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha1_mult_avx_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha1_x8_avx2_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha_256_mult_sse_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha_256_mult_avx_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha256_oct_avx2_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha256_x16_avx512_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha1_ni_x2_sse_from_c(SHA1_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha224_ni_x2_sse_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha256_ni_x2_sse_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha256_ni_x1_sse_from_c(SHA256_ARGS *args, - uint32_t size_in_blocks); - -extern void call_sha512_x2_sse_from_c(SHA512_ARGS *args, - uint64_t size_in_blocks); - -extern void call_sha512_x2_avx_from_c(SHA512_ARGS *args, - uint64_t size_in_blocks); - -extern void call_sha512_x4_avx2_from_c(SHA512_ARGS *args, - uint64_t size_in_blocks); - -extern void call_sha512_x8_avx512_from_c(SHA512_ARGS *args, - uint64_t size_in_blocks); +#include "include/arch_sse_type1.h" +#include "include/arch_sse_type2.h" +#include "include/arch_avx_type1.h" +#include "include/arch_avx2_type1.h" +#include "include/arch_avx512_type1.h" __forceinline void copy_bswap4_array_mb(void *dst, const void *src, const size_t num, -- GitLab From 724df2a8ab08bdffede23858168a295c574d33ef Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 13:56:24 +0000 Subject: [PATCH 327/369] lib: [MB_MGR] add CBC mode specifier and rename [SUBMIT|FLUSH]_JOB_AES[128|192|256]_[ENC|DEC] macros into [SUBMIT|FLUSH]_JOB_AES_CBC_[128|192|256]_[ENC|DEC] --- lib/avx/mb_mgr_avx.c | 18 +++---- lib/avx2_t1/mb_mgr_avx2.c | 18 +++---- lib/avx512_t1/mb_mgr_avx512_t1.c | 18 +++---- lib/avx512_t2/mb_mgr_avx512_t2.c | 18 +++---- lib/include/docsis_common.h | 26 +++++----- lib/include/mb_mgr_burst.h | 12 ++--- lib/include/mb_mgr_job_api.h | 82 +++++++++--------------------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 18 +++---- lib/sse_t1/mb_mgr_sse_t1.c | 18 +++---- lib/sse_t2/mb_mgr_sse_t2.c | 18 +++---- lib/sse_t3/mb_mgr_sse_t3.c | 18 +++---- 11 files changed, 116 insertions(+), 148 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index df9a0de4..b4d0da8b 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -50,15 +50,15 @@ #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx #define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 85391b9a..5e3e1466 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -57,17 +57,17 @@ #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx2 #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx2 diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index 90a7c391..269d71c1 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -193,17 +193,17 @@ submit_job_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) #define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx512 /* AES-CBC */ -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx #define AES_CBC_DEC_128 aes_cbc_dec_128_avx #define AES_CBC_DEC_192 aes_cbc_dec_192_avx diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index 0017a215..d8faf42d 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -196,17 +196,17 @@ vaes_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) #define SUBMIT_JOB_AES_GCM_ENC vaes_submit_gcm_enc_avx512 /* AES-CBC */ -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_vaes_avx512 -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_vaes_avx512 -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_vaes_avx512 +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_vaes_avx512 -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_vaes_avx512 -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_vaes_avx512 -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_vaes_avx512 +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_vaes_avx512 -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_vaes_avx512 -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_vaes_avx512 -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_vaes_avx512 +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_vaes_avx512 +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_vaes_avx512 #define AES_CBC_DEC_128 aes_cbc_dec_128_vaes_avx512 #define AES_CBC_DEC_192 aes_cbc_dec_192_vaes_avx512 diff --git a/lib/include/docsis_common.h b/lib/include/docsis_common.h index b92dc6c7..ca57ab95 100644 --- a/lib/include/docsis_common.h +++ b/lib/include/docsis_common.h @@ -31,9 +31,9 @@ * JOB submit and flush helper functions to be used from mb_mgr_code.h * * @note These need to be defined prior to including this file: - * ETHERNET_FCS, AES_CFB_ONE, SUBMIT_JOB_AES128_DEC and - * SUBMIT_JOB_AES128_ENC, SUBMIT_JOB_AES256_DEC and - * SUBMIT_JOB_AES256_DEC. + * ETHERNET_FCS, AES_CFB_ONE, SUBMIT_JOB_AES_CBC_128_DEC and + * SUBMIT_JOB_AES_CBC_128_ENC, SUBMIT_JOB_AES_CBC_256_ENC and + * SUBMIT_JOB_AES_CBC_256_DEC. * * @note The file defines the following: * DOCSIS_LAST_BLOCK, DOCSIS_FIRST_BLOCK, @@ -180,17 +180,17 @@ SUBMIT_JOB_DOCSIS_SEC_ENC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, if (key_size == 16) { if (job->msg_len_to_cipher_in_bytes >= IMB_AES_BLOCK_SIZE) { - tmp = SUBMIT_JOB_AES128_ENC((MB_MGR_AES_OOO *)state, - job); + MB_MGR_AES_OOO *aes_mgr = (MB_MGR_AES_OOO *)state; + tmp = SUBMIT_JOB_AES_CBC_128_ENC(aes_mgr, job); return DOCSIS_LAST_BLOCK(tmp, 16); } else return DOCSIS_FIRST_BLOCK(job, 16); } else { /* Key length = 32 */ if (job->msg_len_to_cipher_in_bytes >= IMB_AES_BLOCK_SIZE) { - tmp = SUBMIT_JOB_AES256_ENC((MB_MGR_AES_OOO *)state, - job); + MB_MGR_AES_OOO *aes_mgr = (MB_MGR_AES_OOO *)state; + tmp = SUBMIT_JOB_AES_CBC_256_ENC(aes_mgr, job); return DOCSIS_LAST_BLOCK(tmp, 32); } else return DOCSIS_FIRST_BLOCK(job, 32); @@ -225,11 +225,11 @@ FLUSH_JOB_DOCSIS_SEC_ENC(MB_MGR_DOCSIS_AES_OOO *state, const uint64_t key_size) IMB_JOB *tmp; if (key_size == 16) { - tmp = FLUSH_JOB_AES128_ENC((MB_MGR_AES_OOO *)state); + tmp = FLUSH_JOB_AES_CBC_128_ENC((MB_MGR_AES_OOO *)state); return DOCSIS_LAST_BLOCK(tmp, 16); } else { /* 32 */ - tmp = FLUSH_JOB_AES256_ENC((MB_MGR_AES_OOO *)state); + tmp = FLUSH_JOB_AES_CBC_256_ENC((MB_MGR_AES_OOO *)state); return DOCSIS_LAST_BLOCK(tmp, 32); } @@ -266,13 +266,13 @@ SUBMIT_JOB_DOCSIS_SEC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, if (key_size == 16) { if (job->msg_len_to_cipher_in_bytes >= IMB_AES_BLOCK_SIZE) { DOCSIS_LAST_BLOCK(job, 16); - return SUBMIT_JOB_AES128_DEC(job); + return SUBMIT_JOB_AES_CBC_128_DEC(job); } else return DOCSIS_FIRST_BLOCK(job, 16); } else { /* 32 */ if (job->msg_len_to_cipher_in_bytes >= IMB_AES_BLOCK_SIZE) { DOCSIS_LAST_BLOCK(job, 32); - return SUBMIT_JOB_AES256_DEC(job); + return SUBMIT_JOB_AES_CBC_256_DEC(job); } else return DOCSIS_FIRST_BLOCK(job, 32); } @@ -367,9 +367,9 @@ SUBMIT_JOB_DOCSIS_SEC_CRC_DEC(MB_MGR_DOCSIS_AES_OOO *state, IMB_JOB *job, if (job->msg_len_to_cipher_in_bytes >= IMB_AES_BLOCK_SIZE) { DOCSIS_LAST_BLOCK(job, key_size); if (key_size == 16) - job = SUBMIT_JOB_AES128_DEC(job); + job = SUBMIT_JOB_AES_CBC_128_DEC(job); else /* 32 */ - job = SUBMIT_JOB_AES256_DEC(job); + job = SUBMIT_JOB_AES_CBC_256_DEC(job); } else { job = DOCSIS_FIRST_BLOCK(job, key_size); } diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 51863b83..77b09d0e 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -64,16 +64,16 @@ uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, if (key_size == 16) { aes_ooo = state->aes128_ooo; - submit_fn = SUBMIT_JOB_AES128_ENC; - flush_fn = FLUSH_JOB_AES128_ENC; + submit_fn = SUBMIT_JOB_AES_CBC_128_ENC; + flush_fn = FLUSH_JOB_AES_CBC_128_ENC; } else if (key_size == 24) { aes_ooo = state->aes192_ooo; - submit_fn = SUBMIT_JOB_AES192_ENC; - flush_fn = FLUSH_JOB_AES192_ENC; + submit_fn = SUBMIT_JOB_AES_CBC_192_ENC; + flush_fn = FLUSH_JOB_AES_CBC_192_ENC; } else { /* assume 32 */ aes_ooo = state->aes256_ooo; - submit_fn = SUBMIT_JOB_AES256_ENC; - flush_fn = FLUSH_JOB_AES256_ENC; + submit_fn = SUBMIT_JOB_AES_CBC_256_ENC; + flush_fn = FLUSH_JOB_AES_CBC_256_ENC; } for (i = 0; i < n_jobs; i++) { diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index dfbe6839..5b8739e1 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -59,9 +59,7 @@ /* Lower level "out of order" schedulers */ /* ========================================================================= */ -__forceinline -IMB_JOB * -SUBMIT_JOB_AES128_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_CBC_128_DEC(IMB_JOB *job) { AES_CBC_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, job->iv, @@ -72,9 +70,7 @@ SUBMIT_JOB_AES128_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES192_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_CBC_192_DEC(IMB_JOB *job) { AES_CBC_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, job->iv, @@ -85,9 +81,7 @@ SUBMIT_JOB_AES192_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES256_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_CBC_256_DEC(IMB_JOB *job) { AES_CBC_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, job->iv, @@ -98,9 +92,7 @@ SUBMIT_JOB_AES256_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) { AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes, job->enc_keys, @@ -110,9 +102,7 @@ SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_192_ENC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_192_ENC(IMB_JOB *job) { AES_ECB_ENC_192(job->src + job->cipher_start_src_offset_in_bytes, job->enc_keys, @@ -122,9 +112,7 @@ SUBMIT_JOB_AES_ECB_192_ENC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_256_ENC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_256_ENC(IMB_JOB *job) { AES_ECB_ENC_256(job->src + job->cipher_start_src_offset_in_bytes, job->enc_keys, @@ -134,9 +122,7 @@ SUBMIT_JOB_AES_ECB_256_ENC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_128_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_128_DEC(IMB_JOB *job) { AES_ECB_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, job->dec_keys, @@ -146,9 +132,7 @@ SUBMIT_JOB_AES_ECB_128_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_192_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_192_DEC(IMB_JOB *job) { AES_ECB_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, job->dec_keys, @@ -158,9 +142,7 @@ SUBMIT_JOB_AES_ECB_192_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) { AES_ECB_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, job->dec_keys, @@ -170,9 +152,7 @@ SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) +__forceinline IMB_JOB * SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) { AES_CBCS_1_9_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, job->iv, @@ -194,9 +174,7 @@ SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) /* Custom hash / cipher */ /* ========================================================================= */ -__forceinline -IMB_JOB * -JOB_CUSTOM_CIPHER(IMB_JOB *job) +__forceinline IMB_JOB *JOB_CUSTOM_CIPHER(IMB_JOB *job) { if (!(job->status & IMB_STATUS_COMPLETED_CIPHER)) { if (job->cipher_func(job)) @@ -207,23 +185,17 @@ JOB_CUSTOM_CIPHER(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_CUSTOM_CIPHER(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CUSTOM_CIPHER(IMB_JOB *job) { return JOB_CUSTOM_CIPHER(job); } -__forceinline -IMB_JOB * -FLUSH_JOB_CUSTOM_CIPHER(IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CUSTOM_CIPHER(IMB_JOB *job) { return JOB_CUSTOM_CIPHER(job); } -__forceinline -IMB_JOB * -JOB_CUSTOM_HASH(IMB_JOB *job) +__forceinline IMB_JOB *JOB_CUSTOM_HASH(IMB_JOB *job) { if (!(job->status & IMB_STATUS_COMPLETED_AUTH)) { if (job->hash_func(job)) @@ -234,16 +206,12 @@ JOB_CUSTOM_HASH(IMB_JOB *job) return job; } -__forceinline -IMB_JOB * -SUBMIT_JOB_CUSTOM_HASH(IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CUSTOM_HASH(IMB_JOB *job) { return JOB_CUSTOM_HASH(job); } -__forceinline -IMB_JOB * -FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) { return JOB_CUSTOM_HASH(job); } @@ -259,15 +227,15 @@ SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) if (16 == job->key_len_in_bytes) { MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - return SUBMIT_JOB_AES128_ENC(aes128_ooo, job); + return SUBMIT_JOB_AES_CBC_128_ENC(aes128_ooo, job); } else if (24 == job->key_len_in_bytes) { MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - return SUBMIT_JOB_AES192_ENC(aes192_ooo, job); + return SUBMIT_JOB_AES_CBC_192_ENC(aes192_ooo, job); } else { /* assume 32 */ MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - return SUBMIT_JOB_AES256_ENC(aes256_ooo, job); + return SUBMIT_JOB_AES_CBC_256_ENC(aes256_ooo, job); } } else if (IMB_CIPHER_CNTR == job->cipher_mode) { return SUBMIT_JOB_AES_CNTR(job); @@ -372,15 +340,15 @@ FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) if (16 == job->key_len_in_bytes) { MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; - return FLUSH_JOB_AES128_ENC(aes128_ooo); + return FLUSH_JOB_AES_CBC_128_ENC(aes128_ooo); } else if (24 == job->key_len_in_bytes) { MB_MGR_AES_OOO *aes192_ooo = state->aes192_ooo; - return FLUSH_JOB_AES192_ENC(aes192_ooo); + return FLUSH_JOB_AES_CBC_192_ENC(aes192_ooo); } else { /* assume 32 */ MB_MGR_AES_OOO *aes256_ooo = state->aes256_ooo; - return FLUSH_JOB_AES256_ENC(aes256_ooo); + return FLUSH_JOB_AES_CBC_256_ENC(aes256_ooo); } } else if (IMB_CIPHER_DOCSIS_SEC_BPI == job->cipher_mode) { return flush_docsis_enc_job(state, job); @@ -438,11 +406,11 @@ SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) { if (IMB_CIPHER_CBC == job->cipher_mode) { if (16 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES128_DEC(job); + return SUBMIT_JOB_AES_CBC_128_DEC(job); } else if (24 == job->key_len_in_bytes) { - return SUBMIT_JOB_AES192_DEC(job); + return SUBMIT_JOB_AES_CBC_192_DEC(job); } else { /* assume 32 */ - return SUBMIT_JOB_AES256_DEC(job); + return SUBMIT_JOB_AES_CBC_256_DEC(job); } } else if (IMB_CIPHER_CNTR == job->cipher_mode) { return SUBMIT_JOB_AES_CNTR(job); diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index a9b7d1b6..08d0b27e 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -52,15 +52,15 @@ #define SAVE_XMMS save_xmms #define RESTORE_XMMS restore_xmms -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse_no_aesni -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse_no_aesni -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse_no_aesni -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse_no_aesni -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse_no_aesni -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse_no_aesni -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse_no_aesni -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse_no_aesni -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_sse_no_aesni +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_sse_no_aesni +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_sse_no_aesni +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_sse_no_aesni +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_sse_no_aesni #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_sse_no_aesni #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_sse_no_aesni #define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_sse_no_aesni diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index 55d2994d..ccb1e5a0 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -189,17 +189,17 @@ submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse /* AES-CBC */ -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_sse +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_sse -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_sse +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_sse -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_sse +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_sse #define AES_CBC_DEC_128 aes_cbc_dec_128_sse #define AES_CBC_DEC_192 aes_cbc_dec_192_sse diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index ecd4ff5f..ca4e69b7 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -189,17 +189,17 @@ submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse /* AES-CBC */ -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_sse -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_sse +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_sse +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_sse -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_sse -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_sse +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_sse +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_sse -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_sse -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_sse +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_sse +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_sse #define AES_CBC_DEC_128 aes_cbc_dec_128_sse #define AES_CBC_DEC_192 aes_cbc_dec_192_sse diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index 9a6cb1a0..4302463f 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -190,17 +190,17 @@ submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse /* AES-CBC */ -#define SUBMIT_JOB_AES128_ENC submit_job_aes128_enc_x8_sse -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_sse -#define FLUSH_JOB_AES128_ENC flush_job_aes128_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_sse +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_x8_sse -#define SUBMIT_JOB_AES192_ENC submit_job_aes192_enc_x8_sse -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_sse -#define FLUSH_JOB_AES192_ENC flush_job_aes192_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_sse +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_x8_sse -#define SUBMIT_JOB_AES256_ENC submit_job_aes256_enc_x8_sse -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_sse -#define FLUSH_JOB_AES256_ENC flush_job_aes256_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_x8_sse +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_sse +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_x8_sse #define AES_CBC_DEC_128 aes_cbc_dec_128_by8_sse #define AES_CBC_DEC_192 aes_cbc_dec_192_by8_sse -- GitLab From 9cb6b1445a5e26a5c5a25ff4005101483d507e91 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 14:00:23 +0000 Subject: [PATCH 328/369] lib: [MB_MGR] in job submit functions check for GCM before other ciphers --- lib/include/mb_mgr_job_api.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index 5b8739e1..f96159a6 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -223,7 +223,11 @@ __forceinline IMB_JOB * SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) { - if (IMB_CIPHER_CBC == job->cipher_mode) { + if (IMB_CIPHER_GCM == job->cipher_mode) { + return SUBMIT_JOB_AES_GCM_ENC(state, job); + } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { + return submit_gcm_sgl_enc(state, job); + } else if (IMB_CIPHER_CBC == job->cipher_mode) { if (16 == job->key_len_in_bytes) { MB_MGR_AES_OOO *aes128_ooo = state->aes128_ooo; @@ -256,10 +260,6 @@ SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_PON_ENC_NO_CTR(job); else return SUBMIT_JOB_PON_ENC(job); - } else if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_ENC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_enc(state, job); } else if (IMB_CIPHER_CUSTOM == job->cipher_mode) { return SUBMIT_JOB_CUSTOM_CIPHER(job); } else if (IMB_CIPHER_DES == job->cipher_mode) { @@ -404,7 +404,11 @@ __forceinline IMB_JOB * SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) { - if (IMB_CIPHER_CBC == job->cipher_mode) { + if (IMB_CIPHER_GCM == job->cipher_mode) { + return SUBMIT_JOB_AES_GCM_DEC(state, job); + } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { + return submit_gcm_sgl_dec(state, job); + } else if (IMB_CIPHER_CBC == job->cipher_mode) { if (16 == job->key_len_in_bytes) { return SUBMIT_JOB_AES_CBC_128_DEC(job); } else if (24 == job->key_len_in_bytes) { @@ -431,10 +435,6 @@ SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_PON_DEC_NO_CTR(job); else return SUBMIT_JOB_PON_DEC(job); - } else if (IMB_CIPHER_GCM == job->cipher_mode) { - return SUBMIT_JOB_AES_GCM_DEC(state, job); - } else if (IMB_CIPHER_GCM_SGL == job->cipher_mode) { - return submit_gcm_sgl_dec(state, job); } else if (IMB_CIPHER_DES == job->cipher_mode) { #ifdef SUBMIT_JOB_DES_CBC_DEC MB_MGR_DES_OOO *des_dec_ooo = state->des_dec_ooo; -- GitLab From bcdc9f31e3f802c26bf2504be182cb4cf144f21c Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 15:12:40 +0000 Subject: [PATCH 329/369] lib: [MB_MGR] rename [SUBMIT|FLUSH]_JOB_AES_[ENC|DEC] macros into [SUBMIT|FLUSH]_JOB_CIPHER_[ENC|DEC] --- lib/avx/mb_mgr_avx.c | 8 +++--- lib/avx2_t1/mb_mgr_avx2.c | 12 ++++---- lib/avx512_t1/mb_mgr_avx512_t1.c | 6 ++-- lib/avx512_t2/mb_mgr_avx512_t2.c | 6 ++-- lib/include/mb_mgr_job_api.h | 46 +++++++++++++++--------------- lib/no-aesni/mb_mgr_sse_no_aesni.c | 6 ++-- lib/sse_t1/mb_mgr_sse_t1.c | 6 ++-- lib/sse_t2/mb_mgr_sse_t2.c | 6 ++-- lib/sse_t3/mb_mgr_sse_t3.c | 6 ++-- 9 files changed, 51 insertions(+), 51 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index b4d0da8b..c3459e7c 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -137,10 +137,10 @@ #define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx #define QUEUE_SIZE queue_size_avx -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX -#define FLUSH_JOB_AES_DEC FLUSH_JOB_AES_DEC_AVX +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX +#define FLUSH_JOB_CIPHER_DEC FLUSH_JOB_CIPHER_DEC_AVX #define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 5e3e1466..32870475 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -142,14 +142,14 @@ #define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 #define FLUSH_JOB_SHA512 flush_job_sha512_avx2 -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx #define QUEUE_SIZE queue_size_avx2 -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX2 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX2 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX2 +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX2 #define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 #define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index 269d71c1..c84efdb0 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -80,9 +80,9 @@ #define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512 /* Cipher encrypt / decrypt */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512 +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX512 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX512 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX512 /* AES-GCM */ #define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index d8faf42d..65b06934 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -85,9 +85,9 @@ #define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX512_T2 /* Cipher encrypt / decrypt */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_AVX512_T2 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_AVX512_T2 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_AVX512_T2 +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX512_T2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX512_T2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX512_T2 /* AES-GCM */ #define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index f96159a6..67e6cc40 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -56,7 +56,7 @@ job->msg_len_to_hash_in_bytes) /* ========================================================================= */ -/* Lower level "out of order" schedulers */ +/* AES-CBC */ /* ========================================================================= */ __forceinline IMB_JOB *SUBMIT_JOB_AES_CBC_128_DEC(IMB_JOB *job) @@ -92,6 +92,10 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_CBC_256_DEC(IMB_JOB *job) return job; } +/* ========================================================================= */ +/* AES-ECB */ +/* ========================================================================= */ + __forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_128_ENC(IMB_JOB *job) { AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes, @@ -152,6 +156,10 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_ECB_256_DEC(IMB_JOB *job) return job; } +/* ========================================================================= */ +/* AES-CBCS */ +/* ========================================================================= */ + __forceinline IMB_JOB * SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) { AES_CBCS_1_9_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, @@ -219,9 +227,7 @@ __forceinline IMB_JOB *FLUSH_JOB_CUSTOM_HASH(IMB_JOB *job) /* ========================================================================= */ /* Cipher submit & flush functions */ /* ========================================================================= */ -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) { if (IMB_CIPHER_GCM == job->cipher_mode) { return SUBMIT_JOB_AES_GCM_ENC(state, job); @@ -332,9 +338,7 @@ SUBMIT_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) } } -__forceinline -IMB_JOB * -FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) { if (IMB_CIPHER_CBC == job->cipher_mode) { if (16 == job->key_len_in_bytes) { @@ -400,9 +404,7 @@ FLUSH_JOB_AES_ENC(IMB_MGR *state, IMB_JOB *job) } } -__forceinline -IMB_JOB * -SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) { if (IMB_CIPHER_GCM == job->cipher_mode) { return SUBMIT_JOB_AES_GCM_DEC(state, job); @@ -507,9 +509,7 @@ SUBMIT_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) } } -__forceinline -IMB_JOB * -FLUSH_JOB_AES_DEC(IMB_MGR *state, IMB_JOB *job) +__forceinline IMB_JOB *FLUSH_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) { #ifdef FLUSH_JOB_SNOW3G_UEA2 if (IMB_CIPHER_SNOW3G_UEA2_BITLEN == job->cipher_mode) @@ -825,23 +825,23 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) /* ========================================================================= */ __forceinline -IMB_JOB *SUBMIT_JOB_AES(IMB_MGR *state, IMB_JOB *job) +IMB_JOB *SUBMIT_JOB_CIPHER(IMB_MGR *state, IMB_JOB *job) { if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = SUBMIT_JOB_AES_ENC(state, job); + job = SUBMIT_JOB_CIPHER_ENC(state, job); else - job = SUBMIT_JOB_AES_DEC(state, job); + job = SUBMIT_JOB_CIPHER_DEC(state, job); return job; } __forceinline -IMB_JOB *FLUSH_JOB_AES(IMB_MGR *state, IMB_JOB *job) +IMB_JOB *FLUSH_JOB_CIPHER(IMB_MGR *state, IMB_JOB *job) { if (job->cipher_direction == IMB_DIR_ENCRYPT) - job = FLUSH_JOB_AES_ENC(state, job); + job = FLUSH_JOB_CIPHER_ENC(state, job); else - job = FLUSH_JOB_AES_DEC(state, job); + job = FLUSH_JOB_CIPHER_DEC(state, job); return job; } @@ -852,7 +852,7 @@ IMB_JOB *RESUBMIT_JOB(IMB_MGR *state, IMB_JOB *job) { while (job != NULL && job->status < IMB_STATUS_COMPLETED) { if (job->status == IMB_STATUS_COMPLETED_AUTH) - job = SUBMIT_JOB_AES(state, job); + job = SUBMIT_JOB_CIPHER(state, job); else /* assumed job->status = IMB_STATUS_COMPLETED_CIPHER */ job = SUBMIT_JOB_HASH(state, job); } @@ -864,7 +864,7 @@ __forceinline IMB_JOB *submit_new_job(IMB_MGR *state, IMB_JOB *job) { if (job->chain_order == IMB_ORDER_CIPHER_HASH) - job = SUBMIT_JOB_AES(state, job); + job = SUBMIT_JOB_CIPHER(state, job); else job = SUBMIT_JOB_HASH(state, job); @@ -884,7 +884,7 @@ uint32_t complete_job(IMB_MGR *state, IMB_JOB *job) if (job->chain_order == IMB_ORDER_CIPHER_HASH) { /* while() loop optimized for cipher_hash order */ while (job->status < IMB_STATUS_COMPLETED) { - IMB_JOB *tmp = FLUSH_JOB_AES(state, job); + IMB_JOB *tmp = FLUSH_JOB_CIPHER(state, job); if (tmp == NULL) tmp = FLUSH_JOB_HASH(state, job); @@ -898,7 +898,7 @@ uint32_t complete_job(IMB_MGR *state, IMB_JOB *job) IMB_JOB *tmp = FLUSH_JOB_HASH(state, job); if (tmp == NULL) - tmp = FLUSH_JOB_AES(state, job); + tmp = FLUSH_JOB_CIPHER(state, job); (void) RESUBMIT_JOB(state, tmp); completed_jobs++; diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 08d0b27e..0c414305 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -179,9 +179,9 @@ /* ====================================================================== */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_SSE +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_SSE +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE #define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_SSE #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index ccb1e5a0..a60cc6e3 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -76,9 +76,9 @@ #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE /* Cipher encrypt / decrypt */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_SSE +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_SSE +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE /* AES-GCM */ #define AES_GCM_DEC_128 aes_gcm_dec_128_sse diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index ca4e69b7..84a4175e 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -76,9 +76,9 @@ #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE_T2 /* Cipher encrypt / decrypt */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE_T2 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE_T2 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE_T2 +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_SSE_T2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_SSE_T2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE_T2 /* AES-GCM */ #define AES_GCM_DEC_128 aes_gcm_dec_128_sse diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index 4302463f..aeae4488 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -77,9 +77,9 @@ #define FLUSH_JOB_HASH FLUSH_JOB_HASH_SSE_T3 /* Cipher encrypt / decrypt */ -#define SUBMIT_JOB_AES_ENC SUBMIT_JOB_AES_ENC_SSE_T3 -#define FLUSH_JOB_AES_ENC FLUSH_JOB_AES_ENC_SSE_T3 -#define SUBMIT_JOB_AES_DEC SUBMIT_JOB_AES_DEC_SSE_T3 +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_SSE_T3 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_SSE_T3 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE_T3 /* AES-GCM */ #define AES_GCM_DEC_128 aes_gcm_dec_128_sse -- GitLab From 062722150e1edfa5da1a325928dc01fc91298071 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 24 Aug 2022 15:45:45 +0000 Subject: [PATCH 330/369] lib: [MB_MGR] move GCM submit functions from architecture managers to common mb_mgr_job_api.h --- lib/avx/mb_mgr_avx.c | 102 --------------------------- lib/avx2_t1/mb_mgr_avx2.c | 102 --------------------------- lib/avx512_t1/mb_mgr_avx512_t1.c | 97 -------------------------- lib/avx512_t2/mb_mgr_avx512_t2.c | 107 ++--------------------------- lib/include/mb_mgr_job_api.h | 91 ++++++++++++++++++++++++ lib/no-aesni/mb_mgr_sse_no_aesni.c | 100 --------------------------- lib/sse_t1/mb_mgr_sse_t1.c | 97 -------------------------- lib/sse_t2/mb_mgr_sse_t2.c | 97 -------------------------- lib/sse_t3/mb_mgr_sse_t3.c | 97 -------------------------- 9 files changed, 97 insertions(+), 793 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index c3459e7c..06503398 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -101,13 +101,6 @@ #define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx_gen2 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx_gen2 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx_gen2 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx_gen2 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx_gen2 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx_gen2 - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 @@ -223,101 +216,6 @@ ethernet_fcs_avx_local(const void *msg, const uint64_t len, /* ====================================================================== */ -/* - * GCM submit / flush API for AVX arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_avx(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_avx(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -/* ====================================================================== */ - IMB_DLL_LOCAL IMB_JOB * submit_job_aes_cntr_avx(IMB_JOB *job) { diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 32870475..55519787 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -111,13 +111,6 @@ #define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx_gen4 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx_gen4 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx_gen4 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx_gen4 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx_gen4 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx_gen4 - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 @@ -264,101 +257,6 @@ static void /* ====================================================================== */ -/* - * GCM submit / flush API for AVX2 arch - */ -static IMB_JOB * -submit_job_aes_gcm_dec_avx2(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_avx2(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -/* ====================================================================== */ - static void reset_ooo_mgrs(IMB_MGR *state) { diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index c84efdb0..9b1c6994 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -85,13 +85,6 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX512 /* AES-GCM */ -#define AES_GCM_DEC_128 aes_gcm_dec_128_avx512 -#define AES_GCM_ENC_128 aes_gcm_enc_128_avx512 -#define AES_GCM_DEC_192 aes_gcm_dec_192_avx512 -#define AES_GCM_ENC_192 aes_gcm_enc_192_avx512 -#define AES_GCM_DEC_256 aes_gcm_dec_256_avx512 -#define AES_GCM_ENC_256 aes_gcm_enc_256_avx512 - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx512 #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx512 #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx512 @@ -99,96 +92,6 @@ #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx512 #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx512 -static IMB_JOB * -submit_job_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - #define SUBMIT_JOB_AES_GCM_DEC submit_job_gcm_dec_avx512 #define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx512 diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index 65b06934..068202d0 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -90,107 +90,12 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX512_T2 /* AES-GCM */ -#define AES_GCM_DEC_128_VAES aes_gcm_dec_128_vaes_avx512 -#define AES_GCM_ENC_128_VAES aes_gcm_enc_128_vaes_avx512 -#define AES_GCM_DEC_192_VAES aes_gcm_dec_192_vaes_avx512 -#define AES_GCM_ENC_192_VAES aes_gcm_enc_192_vaes_avx512 -#define AES_GCM_DEC_256_VAES aes_gcm_dec_256_vaes_avx512 -#define AES_GCM_ENC_256_VAES aes_gcm_enc_256_vaes_avx512 - -#define AES_GCM_DEC_IV_128_VAES aes_gcm_dec_var_iv_128_vaes_avx512 -#define AES_GCM_ENC_IV_128_VAES aes_gcm_enc_var_iv_128_vaes_avx512 -#define AES_GCM_DEC_IV_192_VAES aes_gcm_dec_var_iv_192_vaes_avx512 -#define AES_GCM_ENC_IV_192_VAES aes_gcm_enc_var_iv_192_vaes_avx512 -#define AES_GCM_DEC_IV_256_VAES aes_gcm_dec_var_iv_256_vaes_avx512 -#define AES_GCM_ENC_IV_256_VAES aes_gcm_enc_var_iv_256_vaes_avx512 - -static IMB_JOB * -vaes_submit_gcm_dec_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_DEC_IV_128_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_DEC_IV_192_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_DEC_IV_256_VAES(job->dec_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -vaes_submit_gcm_enc_avx512(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) - AES_GCM_ENC_IV_128_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_GCM_ENC_IV_192_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - else /* assume 32 bytes */ - AES_GCM_ENC_IV_256_VAES(job->enc_keys, &ctx, - job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - - job->status = IMB_STATUS_COMPLETED; - return job; -} +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_vaes_avx512 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_vaes_avx512 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_vaes_avx512 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_vaes_avx512 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_vaes_avx512 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_vaes_avx512 #define SUBMIT_JOB_AES_GCM_DEC vaes_submit_gcm_dec_avx512 #define SUBMIT_JOB_AES_GCM_ENC vaes_submit_gcm_enc_avx512 diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index 67e6cc40..5df2e5d2 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -178,6 +178,97 @@ __forceinline IMB_JOB * SUBMIT_JOB_AES128_CBCS_1_9_DEC(IMB_JOB *job) #include "include/job_api_docsis.h" +/* ========================================================================= */ +/* AES-GCM */ +/* ========================================================================= */ +__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_DEC(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_128(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_DEC_IV_192(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_DEC_IV_256(job->dec_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + +__forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_ENC(IMB_MGR *state, IMB_JOB *job) +{ + DECLARE_ALIGNED(struct gcm_context_data ctx, 16); + (void) state; + + if (16 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_128(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else if (24 == job->key_len_in_bytes) { + AES_GCM_ENC_IV_192(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } else { /* assume 32 bytes */ + AES_GCM_ENC_IV_256(job->enc_keys, + &ctx, job->dst, + job->src + + job->cipher_start_src_offset_in_bytes, + job->msg_len_to_cipher_in_bytes, + job->iv, job->iv_len_in_bytes, + job->u.GCM.aad, + job->u.GCM.aad_len_in_bytes, + job->auth_tag_output, + job->auth_tag_output_len_in_bytes); + } + + job->status = IMB_STATUS_COMPLETED; + return job; +} + /* ========================================================================= */ /* Custom hash / cipher */ /* ========================================================================= */ diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 0c414305..85907921 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -139,13 +139,6 @@ #define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_sse_no_aesni #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_sse_no_aesni -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse_no_aesni -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse_no_aesni -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse_no_aesni -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse_no_aesni -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse_no_aesni -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse_no_aesni - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse_no_aesni #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse_no_aesni #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse_no_aesni @@ -230,99 +223,6 @@ /* ====================================================================== */ -/* - * GCM submit / flush API for SSE arch without AESNI - */ -static IMB_JOB * -submit_job_aes_gcm_dec_sse_no_aesni(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_sse_no_aesni(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - IMB_DLL_LOCAL IMB_JOB * submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job) { diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index a60cc6e3..8e0e8510 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -81,13 +81,6 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE /* AES-GCM */ -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse @@ -95,96 +88,6 @@ #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse -static IMB_JOB * -submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index 84a4175e..2ee61a81 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -81,13 +81,6 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE_T2 /* AES-GCM */ -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse @@ -95,96 +88,6 @@ #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse -static IMB_JOB * -submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index aeae4488..a7111173 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -82,13 +82,6 @@ #define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_SSE_T3 /* AES-GCM */ -#define AES_GCM_DEC_128 aes_gcm_dec_128_sse -#define AES_GCM_ENC_128 aes_gcm_enc_128_sse -#define AES_GCM_DEC_192 aes_gcm_dec_192_sse -#define AES_GCM_ENC_192 aes_gcm_enc_192_sse -#define AES_GCM_DEC_256 aes_gcm_dec_256_sse -#define AES_GCM_ENC_256 aes_gcm_enc_256_sse - #define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_sse #define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_sse #define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_sse @@ -96,96 +89,6 @@ #define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_sse #define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_sse -static IMB_JOB * -submit_job_aes_gcm_dec_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_128(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_DEC_IV_192(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_DEC_IV_256(job->dec_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - -static IMB_JOB * -submit_job_aes_gcm_enc_sse(IMB_MGR *state, IMB_JOB *job) -{ - DECLARE_ALIGNED(struct gcm_context_data ctx, 16); - (void) state; - - if (16 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_128(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else if (24 == job->key_len_in_bytes) { - AES_GCM_ENC_IV_192(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } else { /* assume 32 bytes */ - AES_GCM_ENC_IV_256(job->enc_keys, - &ctx, job->dst, - job->src + - job->cipher_start_src_offset_in_bytes, - job->msg_len_to_cipher_in_bytes, - job->iv, job->iv_len_in_bytes, - job->u.GCM.aad, - job->u.GCM.aad_len_in_bytes, - job->auth_tag_output, - job->auth_tag_output_len_in_bytes); - } - - job->status = IMB_STATUS_COMPLETED; - return job; -} - #define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_sse #define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_sse -- GitLab From 8a96ae866ffa28cd7d21b0cb61e41cf0e4e170a4 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 25 Aug 2022 09:12:08 +0000 Subject: [PATCH 331/369] lib: [MB_MGR] move AES-CTR and AES-CTR_BIT submit functions to mb_mgr_job_api.h --- lib/avx/mb_mgr_avx.c | 76 +++--------------------- lib/avx2_t1/mb_mgr_avx2.c | 12 ++-- lib/avx512_t1/mb_mgr_avx512_t1.c | 12 ++-- lib/avx512_t2/mb_mgr_avx512_t2.c | 38 ++---------- lib/include/mb_mgr_burst.h | 85 ++++++++++++++------------ lib/include/mb_mgr_job_api.h | 95 ++++++++++++++++++++++++++++-- lib/no-aesni/mb_mgr_sse_no_aesni.c | 75 ++--------------------- lib/sse_t1/mb_mgr_sse_t1.c | 79 +++---------------------- lib/sse_t2/mb_mgr_sse_t2.c | 78 +++--------------------- lib/sse_t3/mb_mgr_sse_t3.c | 78 +++--------------------- 10 files changed, 191 insertions(+), 437 deletions(-) diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx/mb_mgr_avx.c index 06503398..a9372bdc 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx/mb_mgr_avx.c @@ -66,9 +66,6 @@ #define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx #define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx - #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx #define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx @@ -82,9 +79,12 @@ #define AES_CBC_DEC_192 aes_cbc_dec_192_avx #define AES_CBC_DEC_256 aes_cbc_dec_256_avx -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx @@ -216,69 +216,7 @@ ethernet_fcs_avx_local(const void *msg, const uint64_t len, /* ====================================================================== */ -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_avx(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_bit_avx(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_avx(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static void -reset_ooo_mgrs(IMB_MGR *state) +static void reset_ooo_mgrs(IMB_MGR *state) { /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 8); diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 55519787..990f7edc 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -76,9 +76,6 @@ #define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx2 #define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx2 -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx - #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 #define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 @@ -92,9 +89,12 @@ #define AES_CBC_DEC_192 aes_cbc_dec_192_avx #define AES_CBC_DEC_256 aes_cbc_dec_256_avx -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx diff --git a/lib/avx512_t1/mb_mgr_avx512_t1.c b/lib/avx512_t1/mb_mgr_avx512_t1.c index 9b1c6994..78c53f35 100644 --- a/lib/avx512_t1/mb_mgr_avx512_t1.c +++ b/lib/avx512_t1/mb_mgr_avx512_t1.c @@ -134,12 +134,12 @@ #define AES_ECB_DEC_256 aes_ecb_dec_256_avx /* AES-CTR */ -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_avx -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_avx - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx /* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx diff --git a/lib/avx512_t2/mb_mgr_avx512_t2.c b/lib/avx512_t2/mb_mgr_avx512_t2.c index 068202d0..8b94a712 100644 --- a/lib/avx512_t2/mb_mgr_avx512_t2.c +++ b/lib/avx512_t2/mb_mgr_avx512_t2.c @@ -139,38 +139,12 @@ #define AES_ECB_DEC_256 aes_ecb_dec_256_vaes_avx512 /* AES-CTR */ -#define SUBMIT_JOB_AES_CNTR vaes_submit_cntr_avx512 -#define SUBMIT_JOB_AES_CNTR_BIT vaes_submit_cntr_bit_avx512 - -#define AES_CNTR_128 aes_cntr_128_avx -#define AES_CNTR_192 aes_cntr_192_avx -#define AES_CNTR_256 aes_cntr_256_avx - -static IMB_JOB * vaes_submit_cntr_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_256_submit_vaes_avx512(job); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static IMB_JOB * vaes_submit_cntr_bit_avx512(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_submit_vaes_avx512(job); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_submit_vaes_avx512(job); - else /* assume 32 bytes */ - aes_cntr_bit_256_submit_vaes_avx512(job); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} +#define SUBMIT_JOB_AES_CTR_128 aes_cntr_128_submit_vaes_avx512 +#define SUBMIT_JOB_AES_CTR_192 aes_cntr_192_submit_vaes_avx512 +#define SUBMIT_JOB_AES_CTR_256 aes_cntr_256_submit_vaes_avx512 +#define SUBMIT_JOB_AES_CTR_128_BIT aes_cntr_bit_128_submit_vaes_avx512 +#define SUBMIT_JOB_AES_CTR_192_BIT aes_cntr_bit_192_submit_vaes_avx512 +#define SUBMIT_JOB_AES_CTR_256_BIT aes_cntr_bit_256_submit_vaes_avx512 /* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_vaes_avx512 diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 77b09d0e..a3d0696c 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -155,9 +155,9 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, const IMB_KEY_SIZE_BYTES key_size, const int run_check) { - uint32_t i, completed_jobs = 0; - if (run_check) { + uint32_t i; + /* validate jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -172,55 +172,66 @@ uint32_t submit_aes_ctr_burst(IMB_MGR *state, } } -#ifdef AVX512 - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - void (*submit_fn_vaes) (IMB_JOB *job) = NULL; - - if (key_size == 16) - submit_fn_vaes = aes_cntr_128_submit_vaes_avx512; - else if (key_size == 24) - submit_fn_vaes = aes_cntr_192_submit_vaes_avx512; - else /* assume 32 */ - submit_fn_vaes = aes_cntr_256_submit_vaes_avx512; + if (key_size == IMB_KEY_128_BYTES) { + uint32_t i; for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - submit_fn_vaes(job); +#ifdef SUBMIT_JOB_AES_CTR_128 + SUBMIT_JOB_AES_CTR_128(job); +#else + AES_CTR_128(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); +#endif job->status = IMB_STATUS_COMPLETED; - completed_jobs++; } - } else { + } else if (key_size == IMB_KEY_192_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + +#ifdef SUBMIT_JOB_AES_CTR_192 + SUBMIT_JOB_AES_CTR_192(job); +#else + AES_CTR_192(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); #endif - void (*submit_fn) (const void *in, const void *IV, - const void *keys, void *out, - uint64_t len_bytes, - uint64_t iv_len_bytes) = NULL; - - if (key_size == 16) - submit_fn = AES_CNTR_128; - else if (key_size == 24) - submit_fn = AES_CNTR_192; - else /* assume 32 */ - submit_fn = AES_CNTR_256; + job->status = IMB_STATUS_COMPLETED; + } + } else /* assume 256-bit key */ { + uint32_t i; for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; - submit_fn(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); +#ifdef SUBMIT_JOB_AES_CTR_256 + SUBMIT_JOB_AES_CTR_256(job); +#else + AES_CTR_256(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); +#endif job->status = IMB_STATUS_COMPLETED; - completed_jobs++; } -#ifdef AVX512 } -#endif - return completed_jobs; + + return n_jobs; } __forceinline diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index 5df2e5d2..a728dfb1 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -268,6 +268,93 @@ __forceinline IMB_JOB *SUBMIT_JOB_AES_GCM_ENC(IMB_MGR *state, IMB_JOB *job) job->status = IMB_STATUS_COMPLETED; return job; } +/* ========================================================================= */ +/* AES-CTR */ +/* ========================================================================= */ +__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR(IMB_JOB *job) +{ + if (IMB_KEY_128_BYTES == job->key_len_in_bytes) { +#ifdef SUBMIT_JOB_AES_CTR_128 + SUBMIT_JOB_AES_CTR_128(job); +#else + AES_CTR_128(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); +#endif + } else if (IMB_KEY_192_BYTES == job->key_len_in_bytes) { +#ifdef SUBMIT_JOB_AES_CTR_192 + SUBMIT_JOB_AES_CTR_192(job); +#else + AES_CTR_192(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); +#endif + } else /* assume 256-bit key */ { +#ifdef SUBMIT_JOB_AES_CTR_256 + SUBMIT_JOB_AES_CTR_256(job); +#else + AES_CTR_256(job->src + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bytes, + job->iv_len_in_bytes); +#endif + } + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} + +__forceinline IMB_JOB *SUBMIT_JOB_AES_CTR_BIT(IMB_JOB *job) +{ + if (IMB_KEY_128_BYTES == job->key_len_in_bytes) { +#ifdef SUBMIT_JOB_AES_CTR_128_BIT + SUBMIT_JOB_AES_CTR_128_BIT(job); +#else + AES_CTR_128_BIT(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); +#endif + } else if (IMB_KEY_192_BYTES == job->key_len_in_bytes) { +#ifdef SUBMIT_JOB_AES_CTR_192_BIT + SUBMIT_JOB_AES_CTR_192_BIT(job); +#else + AES_CTR_192_BIT(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); +#endif + } else /* assume 256-bit key */ { +#ifdef SUBMIT_JOB_AES_CTR_256_BIT + SUBMIT_JOB_AES_CTR_256_BIT(job); +#else + AES_CTR_256_BIT(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->enc_keys, + job->dst, + job->msg_len_to_cipher_in_bits, + job->iv_len_in_bytes); +#endif + } + + job->status |= IMB_STATUS_COMPLETED_CIPHER; + return job; +} /* ========================================================================= */ /* Custom hash / cipher */ @@ -339,9 +426,9 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_ENC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_AES_CBC_256_ENC(aes256_ooo, job); } } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR(job); + return SUBMIT_JOB_AES_CTR(job); } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR_BIT(job); + return SUBMIT_JOB_AES_CTR_BIT(job); } else if (IMB_CIPHER_ECB == job->cipher_mode) { if (16 == job->key_len_in_bytes) { return SUBMIT_JOB_AES_ECB_128_ENC(job); @@ -510,9 +597,9 @@ __forceinline IMB_JOB *SUBMIT_JOB_CIPHER_DEC(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_AES_CBC_256_DEC(job); } } else if (IMB_CIPHER_CNTR == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR(job); + return SUBMIT_JOB_AES_CTR(job); } else if (IMB_CIPHER_CNTR_BITLEN == job->cipher_mode) { - return SUBMIT_JOB_AES_CNTR_BIT(job); + return SUBMIT_JOB_AES_CTR_BIT(job); } else if (IMB_CIPHER_ECB == job->cipher_mode) { if (16 == job->key_len_in_bytes) { return SUBMIT_JOB_AES_ECB_128_DEC(job); diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 85907921..268576aa 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -104,9 +104,6 @@ #define SUBMIT_JOB_SHA512 submit_job_sha512_sse #define FLUSH_JOB_SHA512 flush_job_sha512_sse -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse_no_aesni -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse_no_aesni - #define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_sse_no_aesni #define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_sse_no_aesni #define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_sse_no_aesni @@ -120,9 +117,12 @@ #define AES_CBC_DEC_192 aes_cbc_dec_192_sse_no_aesni #define AES_CBC_DEC_256 aes_cbc_dec_256_sse_no_aesni -#define AES_CNTR_128 aes_cntr_128_sse_no_aesni -#define AES_CNTR_192 aes_cntr_192_sse_no_aesni -#define AES_CNTR_256 aes_cntr_256_sse_no_aesni +#define AES_CTR_128 aes_cntr_128_sse_no_aesni +#define AES_CTR_192 aes_cntr_192_sse_no_aesni +#define AES_CTR_256 aes_cntr_256_sse_no_aesni +#define AES_CTR_128_BIT aes_cntr_bit_128_sse_no_aesni +#define AES_CTR_192_BIT aes_cntr_bit_192_sse_no_aesni +#define AES_CTR_256_BIT aes_cntr_bit_256_sse_no_aesni #define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse_no_aesni #define AES_CNTR_CCM_256 aes_cntr_ccm_256_sse_no_aesni @@ -220,69 +220,6 @@ #define ETHERNET_FCS ethernet_fcs_sse_no_aesni_local - -/* ====================================================================== */ - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_sse_no_aesni(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -IMB_DLL_LOCAL IMB_JOB * -submit_job_aes_cntr_bit_sse_no_aesni(IMB_JOB *job) -{ - const uint64_t offset = job->cipher_start_src_offset_in_bytes; - - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse_no_aesni(job->src + offset, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse_no_aesni(job->src + offset, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse_no_aesni(job->src + offset, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - /* ====================================================================== */ static void diff --git a/lib/sse_t1/mb_mgr_sse_t1.c b/lib/sse_t1/mb_mgr_sse_t1.c index 8e0e8510..e6cd7953 100644 --- a/lib/sse_t1/mb_mgr_sse_t1.c +++ b/lib/sse_t1/mb_mgr_sse_t1.c @@ -130,12 +130,12 @@ #define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse /* AES-CTR */ -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse - -#define AES_CNTR_128 aes_cntr_128_sse -#define AES_CNTR_192 aes_cntr_192_sse -#define AES_CNTR_256 aes_cntr_256_sse +#define AES_CTR_128 aes_cntr_128_sse +#define AES_CTR_192 aes_cntr_192_sse +#define AES_CTR_256 aes_cntr_256_sse +#define AES_CTR_128_BIT aes_cntr_bit_128_sse +#define AES_CTR_192_BIT aes_cntr_bit_192_sse +#define AES_CTR_256_BIT aes_cntr_bit_256_sse /* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse @@ -254,72 +254,7 @@ flush_snow3g_uea2_job_sse(IMB_MGR *state) /* ====================================================================== */ -static IMB_JOB * -submit_job_aes_cntr_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static IMB_JOB * -submit_job_aes_cntr_bit_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ====================================================================== */ - - -static void -reset_ooo_mgrs(IMB_MGR *state) +static void reset_ooo_mgrs(IMB_MGR *state) { /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 4); diff --git a/lib/sse_t2/mb_mgr_sse_t2.c b/lib/sse_t2/mb_mgr_sse_t2.c index 2ee61a81..9bed6aa9 100644 --- a/lib/sse_t2/mb_mgr_sse_t2.c +++ b/lib/sse_t2/mb_mgr_sse_t2.c @@ -130,12 +130,12 @@ #define AES_ECB_DEC_256 aes_ecb_dec_256_by4_sse /* AES-CTR */ -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse - -#define AES_CNTR_128 aes_cntr_128_sse -#define AES_CNTR_192 aes_cntr_192_sse -#define AES_CNTR_256 aes_cntr_256_sse +#define AES_CTR_128 aes_cntr_128_sse +#define AES_CTR_192 aes_cntr_192_sse +#define AES_CTR_256 aes_cntr_256_sse +#define AES_CTR_128_BIT aes_cntr_bit_128_sse +#define AES_CTR_192_BIT aes_cntr_bit_192_sse +#define AES_CTR_256_BIT aes_cntr_bit_256_sse /* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse @@ -254,71 +254,7 @@ flush_snow3g_uea2_job_sse(IMB_MGR *state) /* ====================================================================== */ -static IMB_JOB * -submit_job_aes_cntr_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static IMB_JOB * -submit_job_aes_cntr_bit_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) +static void reset_ooo_mgrs(IMB_MGR *state) { /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 4); diff --git a/lib/sse_t3/mb_mgr_sse_t3.c b/lib/sse_t3/mb_mgr_sse_t3.c index a7111173..418dea2e 100644 --- a/lib/sse_t3/mb_mgr_sse_t3.c +++ b/lib/sse_t3/mb_mgr_sse_t3.c @@ -131,12 +131,12 @@ #define AES_ECB_DEC_256 aes_ecb_dec_256_by8_sse /* AES-CTR */ -#define SUBMIT_JOB_AES_CNTR submit_job_aes_cntr_sse -#define SUBMIT_JOB_AES_CNTR_BIT submit_job_aes_cntr_bit_sse - -#define AES_CNTR_128 aes_cntr_128_sse -#define AES_CNTR_192 aes_cntr_192_sse -#define AES_CNTR_256 aes_cntr_256_sse +#define AES_CTR_128 aes_cntr_128_sse +#define AES_CTR_192 aes_cntr_192_sse +#define AES_CTR_256 aes_cntr_256_sse +#define AES_CTR_128_BIT aes_cntr_bit_128_sse +#define AES_CTR_192_BIT aes_cntr_bit_192_sse +#define AES_CTR_256_BIT aes_cntr_bit_256_sse /* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_sse @@ -255,71 +255,7 @@ flush_snow3g_uea2_job_sse(IMB_MGR *state) /* ====================================================================== */ -static IMB_JOB * -submit_job_aes_cntr_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - AES_CNTR_128(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - AES_CNTR_192(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - AES_CNTR_256(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bytes, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -static IMB_JOB * -submit_job_aes_cntr_bit_sse(IMB_JOB *job) -{ - if (16 == job->key_len_in_bytes) - aes_cntr_bit_128_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else if (24 == job->key_len_in_bytes) - aes_cntr_bit_192_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - else /* assume 32 bytes */ - aes_cntr_bit_256_sse(job->src + - job->cipher_start_src_offset_in_bytes, - job->iv, - job->enc_keys, - job->dst, - job->msg_len_to_cipher_in_bits, - job->iv_len_in_bytes); - - job->status |= IMB_STATUS_COMPLETED_CIPHER; - return job; -} - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) +static void reset_ooo_mgrs(IMB_MGR *state) { /* Init AES out-of-order fields */ ooo_mgr_aes_reset(state->aes128_ooo, 8); -- GitLab From 194e5a2590b15710641c5a03ef5111e88f54fd27 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 25 Aug 2022 10:10:04 +0000 Subject: [PATCH 332/369] lib: [MB_MGR] replace function pointers in AES-CBC cipher burst API - cleaned up some no longer needed `endbranch64` instructions in AES-CBC, AES-CTR, AES-ECB and SHA --- lib/avx/aes128_cbc_dec_by8_avx.asm | 2 - lib/avx/aes128_cntr_by8_avx.asm | 2 - lib/avx/aes128_ecb_by8_avx.asm | 3 - lib/avx/aes192_cbc_dec_by8_avx.asm | 2 - lib/avx/aes192_cntr_by8_avx.asm | 2 - lib/avx/aes256_cbc_dec_by8_avx.asm | 2 - lib/avx/aes256_cntr_by8_avx.asm | 2 - lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm | 3 - lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm | 3 - lib/avx2_t1/sha1_x8_avx2.asm | 3 +- lib/avx2_t1/sha256_oct_avx2.asm | 3 - lib/avx2_t1/sha512_x4_avx2.asm | 3 - lib/avx512_t1/sha1_x16_avx512.asm | 2 - lib/avx512_t1/sha256_x16_avx512.asm | 2 - lib/avx512_t1/sha512_x8_avx512.asm | 2 - .../aes_cbc_dec_by16_vaes_avx512.asm | 5 +- lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm | 14 +- .../aes_cntr_api_by16_vaes_avx512.asm | 3 - .../aes_cntr_bit_api_by16_vaes_avx512.asm | 3 - lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc | 3 +- .../aes_cntr_ccm_api_by16_vaes_avx512.asm | 2 - .../aes_cntr_pon_api_by16_vaes_avx512.asm | 3 - lib/avx512_t2/aes_ecb_vaes_avx512.asm | 8 +- .../mb_mgr_aes128_cbc_enc_flush_avx512.asm | 2 - .../mb_mgr_aes128_cbc_enc_submit_avx512.asm | 2 - lib/include/mb_mgr_burst.h | 162 ++++++++++++------ lib/sse_t1/aes128_cbc_enc_x4_sse.asm | 2 - lib/sse_t1/aes128_cntr_by8_sse.asm | 6 +- lib/sse_t1/aes192_cntr_by8_sse.asm | 3 +- lib/sse_t1/aes256_cntr_by8_sse.asm | 6 +- .../mb_mgr_aes128_cbc_enc_flush_x4_sse.asm | 4 - .../mb_mgr_aes128_cbc_enc_submit_x4_sse.asm | 3 - lib/sse_t1/sha1_x4_sse.asm | 1 - lib/sse_t1/zuc_x4_sse.asm | 1 - lib/sse_t2/sha1_ni_x1_sse.asm | 1 - lib/sse_t2/sha1_ni_x2_sse.asm | 1 - lib/sse_t2/sha256_ni_x2_sse.asm | 1 - lib/sse_t3/aes128_cbc_dec_by8_sse.asm | 3 - lib/sse_t3/aes128_cbc_enc_x8_sse.asm | 2 - lib/sse_t3/aes128_ecb_by8_sse.asm | 3 - lib/sse_t3/aes192_cbc_dec_by8_sse.asm | 3 +- lib/sse_t3/aes256_cbc_dec_by8_sse.asm | 3 +- 42 files changed, 126 insertions(+), 160 deletions(-) diff --git a/lib/avx/aes128_cbc_dec_by8_avx.asm b/lib/avx/aes128_cbc_dec_by8_avx.asm index 3cfaa81c..c50d7898 100644 --- a/lib/avx/aes128_cbc_dec_by8_avx.asm +++ b/lib/avx/aes128_cbc_dec_by8_avx.asm @@ -31,7 +31,6 @@ %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -216,7 +215,6 @@ mksection .text MKGLOBAL(AES_CBC_DEC_128_X8,function,internal) AES_CBC_DEC_128_X8: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %else diff --git a/lib/avx/aes128_cntr_by8_avx.asm b/lib/avx/aes128_cntr_by8_avx.asm index a5459096..a74b8599 100644 --- a/lib/avx/aes128_cntr_by8_avx.asm +++ b/lib/avx/aes128_cntr_by8_avx.asm @@ -31,7 +31,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -600,7 +599,6 @@ aes_cntr_ccm_128_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_128_avx,function,internal) aes_cntr_128_avx: - endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_128_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/aes128_ecb_by8_avx.asm b/lib/avx/aes128_ecb_by8_avx.asm index 97152c61..65cd3972 100644 --- a/lib/avx/aes128_ecb_by8_avx.asm +++ b/lib/avx/aes128_ecb_by8_avx.asm @@ -30,7 +30,6 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/aes_common.asm" -%include "include/cet.inc" %ifdef LINUX %define IN rdi @@ -155,13 +154,11 @@ mksection .text align 16 MKGLOBAL(AES_ECB_ENC,function,internal) AES_ECB_ENC: - endbranch64 AES_ECB ENC ret align 16 MKGLOBAL(AES_ECB_DEC,function,internal) AES_ECB_DEC: - endbranch64 AES_ECB DEC ret diff --git a/lib/avx/aes192_cbc_dec_by8_avx.asm b/lib/avx/aes192_cbc_dec_by8_avx.asm index dabecd1d..13f85efb 100644 --- a/lib/avx/aes192_cbc_dec_by8_avx.asm +++ b/lib/avx/aes192_cbc_dec_by8_avx.asm @@ -30,7 +30,6 @@ ; XMM registers are clobbered. Saving/restoring must be done at a higher level %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -231,7 +230,6 @@ mksection .text ;; aes_cbc_dec_192_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(aes_cbc_dec_192_avx,function,internal) aes_cbc_dec_192_avx: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/avx/aes192_cntr_by8_avx.asm b/lib/avx/aes192_cntr_by8_avx.asm index 017b837b..c6e928c0 100644 --- a/lib/avx/aes192_cntr_by8_avx.asm +++ b/lib/avx/aes192_cntr_by8_avx.asm @@ -30,7 +30,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -496,7 +495,6 @@ aes_cntr_ccm_192_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_192_avx,function,internal) aes_cntr_192_avx: - endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_192_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/aes256_cbc_dec_by8_avx.asm b/lib/avx/aes256_cbc_dec_by8_avx.asm index 44f76834..36aa5438 100644 --- a/lib/avx/aes256_cbc_dec_by8_avx.asm +++ b/lib/avx/aes256_cbc_dec_by8_avx.asm @@ -30,7 +30,6 @@ ; XMM registers are clobbered. Saving/restoring must be done at a higher level %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define CONCAT(a,b) a %+ b %define VMOVDQ vmovdqu @@ -247,7 +246,6 @@ mksection .text ;; aes_cbc_dec_256_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(aes_cbc_dec_256_avx,function,internal) aes_cbc_dec_256_avx: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/avx/aes256_cntr_by8_avx.asm b/lib/avx/aes256_cntr_by8_avx.asm index bde04636..3d5d3b6a 100644 --- a/lib/avx/aes256_cntr_by8_avx.asm +++ b/lib/avx/aes256_cntr_by8_avx.asm @@ -31,7 +31,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -594,7 +593,6 @@ aes_cntr_ccm_256_avx: ;; UINT64 iv_len) MKGLOBAL(aes_cntr_256_avx,function,internal) aes_cntr_256_avx: - endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_256_avx(void *in, void *IV, void *keys, void *out, UINT64 num_bits, diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm b/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm index 5f6c6167..c0a68a4f 100644 --- a/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %ifndef AES_CBC_ENC_X8 @@ -116,8 +115,6 @@ endstruc ; arg 2 : job MKGLOBAL(FLUSH_JOB_AES_ENC,function,internal) FLUSH_JOB_AES_ENC: - endbranch64 - mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm b/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm index c563589e..e44dadeb 100644 --- a/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" @@ -89,8 +88,6 @@ endstruc ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_ENC,function,internal) SUBMIT_JOB_AES_ENC: - endbranch64 - mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/avx2_t1/sha1_x8_avx2.asm b/lib/avx2_t1/sha1_x8_avx2.asm index cf5b86aa..04b288dd 100644 --- a/lib/avx2_t1/sha1_x8_avx2.asm +++ b/lib/avx2_t1/sha1_x8_avx2.asm @@ -40,7 +40,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/transpose_avx2.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + mksection .rodata default rel align 32 @@ -366,7 +366,6 @@ align 32 ; arg 2 : rdx : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_x8_avx2,function,internal) sha1_x8_avx2: - endbranch64 sub rsp, FRAMESZ ;; Initialize digests diff --git a/lib/avx2_t1/sha256_oct_avx2.asm b/lib/avx2_t1/sha256_oct_avx2.asm index 61f440fd..f56b7256 100644 --- a/lib/avx2_t1/sha256_oct_avx2.asm +++ b/lib/avx2_t1/sha256_oct_avx2.asm @@ -42,7 +42,6 @@ %include "include/os.asm" ;%define DO_DBGPRINT %include "include/dbgprint.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %include "include/transpose_avx2.asm" %include "include/clear_regs.asm" @@ -480,7 +479,6 @@ endstruc MKGLOBAL(sha256_oct_avx2,function,internal) align 16 sha256_oct_avx2: - endbranch64 ; general registers preserved in outer calling routine ; outer calling routine saves all the XMM registers sub rsp, FRAMESZ @@ -568,7 +566,6 @@ lloop: jmp Lrounds_16_xx align 16 Lrounds_16_xx: - endbranch64 %rep 16 ROUND_16_XX T1, i %assign i (i+1) diff --git a/lib/avx2_t1/sha512_x4_avx2.asm b/lib/avx2_t1/sha512_x4_avx2.asm index 88f12566..f3c042e9 100644 --- a/lib/avx2_t1/sha512_x4_avx2.asm +++ b/lib/avx2_t1/sha512_x4_avx2.asm @@ -47,7 +47,6 @@ %include "include/dbgprint.asm" %include "include/mb_mgr_datastruct.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" mksection .rodata default rel align 64 @@ -382,7 +381,6 @@ endstruc MKGLOBAL(sha512_x4_avx2,function,internal) align 32 sha512_x4_avx2: - endbranch64 ; general registers preserved in outer calling routine ; outer calling routine saves all the XMM registers @@ -449,7 +447,6 @@ lloop: jmp Lrounds_16_xx align 16 Lrounds_16_xx: - endbranch64 %rep 16 ROUND_16_XX T1, i %assign i (i+1) diff --git a/lib/avx512_t1/sha1_x16_avx512.asm b/lib/avx512_t1/sha1_x16_avx512.asm index 2822535e..501be9b8 100644 --- a/lib/avx512_t1/sha1_x16_avx512.asm +++ b/lib/avx512_t1/sha1_x16_avx512.asm @@ -44,7 +44,6 @@ %include "include/transpose_avx512.asm" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" mksection .rodata default rel align 64 @@ -297,7 +296,6 @@ align 64 ; arg 2 : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_x16_avx512,function,internal) sha1_x16_avx512: - endbranch64 ;; Initialize digests vmovdqu32 A, [state + 0*SHA1_DIGEST_ROW_SIZE] vmovdqu32 B, [state + 1*SHA1_DIGEST_ROW_SIZE] diff --git a/lib/avx512_t1/sha256_x16_avx512.asm b/lib/avx512_t1/sha256_x16_avx512.asm index f38b49a6..fb060b9d 100644 --- a/lib/avx512_t1/sha256_x16_avx512.asm +++ b/lib/avx512_t1/sha256_x16_avx512.asm @@ -44,7 +44,6 @@ %include "include/transpose_avx512.asm" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; re-use K256 from sha256_oct_avx2.asm extern K256 @@ -646,7 +645,6 @@ mksection .text MKGLOBAL(sha256_x16_avx512,function,internal) align 64 sha256_x16_avx512: - endbranch64 mov rax, rsp sub rsp, STACK_SPACE and rsp, ~63 ; align stack to multiple of 64 diff --git a/lib/avx512_t1/sha512_x8_avx512.asm b/lib/avx512_t1/sha512_x8_avx512.asm index af95b8f0..24c03f8a 100644 --- a/lib/avx512_t1/sha512_x8_avx512.asm +++ b/lib/avx512_t1/sha512_x8_avx512.asm @@ -45,7 +45,6 @@ %include "include/mb_mgr_datastruct.asm" %include "include/transpose_avx512.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define APPEND(a,b) a %+ b %ifdef LINUX @@ -491,7 +490,6 @@ mksection .text MKGLOBAL(sha512_x8_avx512,function,internal) align 64 sha512_x8_avx512: - endbranch64 mov rax, rsp sub rsp, STACK_SPACE and rsp, ~63 ; align stack to multiple of 64 diff --git a/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm index b81784f7..39ee837b 100644 --- a/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm @@ -29,7 +29,7 @@ %include "include/reg_sizes.asm" %include "include/aes_common.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + %define zIV zmm0 %define zBLK_0_3 zmm1 %define zBLK_4_7 zmm2 @@ -471,7 +471,6 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_dec_128_vaes_avx512,function,internal) aes_cbc_dec_128_vaes_avx512: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif @@ -490,7 +489,6 @@ aes_cbc_dec_128_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_dec_192_vaes_avx512,function,internal) aes_cbc_dec_192_vaes_avx512: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif @@ -509,7 +507,6 @@ aes_cbc_dec_192_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_dec_256_vaes_avx512,function,internal) aes_cbc_dec_256_vaes_avx512: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm index c7d0ce77..c916a83d 100644 --- a/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbc_enc_vaes_avx512.asm @@ -31,7 +31,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + struc STACK _gpr_save: resq 4 _lane_masks: resw 16 @@ -1037,7 +1037,6 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_128_vaes_avx512,function,internal) aes_cbc_enc_128_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE @@ -1048,7 +1047,6 @@ aes_cbc_enc_128_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_192_vaes_avx512,function,internal) aes_cbc_enc_192_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 11, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE @@ -1059,7 +1057,6 @@ aes_cbc_enc_192_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_256_vaes_avx512,function,internal) aes_cbc_enc_256_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 13, MAC_TYPE_NONE, SUBMIT FUNC_RESTORE @@ -1070,7 +1067,6 @@ aes_cbc_enc_256_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes128_cbc_mac_vaes_avx512,function,internal) aes128_cbc_mac_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE @@ -1081,7 +1077,6 @@ aes128_cbc_mac_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes256_cbc_mac_vaes_avx512,function,internal) aes256_cbc_mac_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 13, MAC_TYPE_CBC, SUBMIT FUNC_RESTORE @@ -1092,7 +1087,6 @@ aes256_cbc_mac_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_xcbc_mac_128_vaes_avx512,function,internal) aes_xcbc_mac_128_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_XCBC, SUBMIT FUNC_RESTORE @@ -1105,7 +1099,6 @@ aes_xcbc_mac_128_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_128_flush_vaes_avx512,function,internal) aes_cbc_enc_128_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_NONE, FLUSH FUNC_RESTORE @@ -1118,7 +1111,6 @@ aes_cbc_enc_128_flush_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_192_flush_vaes_avx512,function,internal) aes_cbc_enc_192_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 11, MAC_TYPE_NONE, FLUSH FUNC_RESTORE @@ -1131,7 +1123,6 @@ aes_cbc_enc_192_flush_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbc_enc_256_flush_vaes_avx512,function,internal) aes_cbc_enc_256_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 13, MAC_TYPE_NONE, FLUSH FUNC_RESTORE @@ -1144,7 +1135,6 @@ aes_cbc_enc_256_flush_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes128_cbc_mac_flush_vaes_avx512,function,internal) aes128_cbc_mac_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_CBC, FLUSH FUNC_RESTORE @@ -1157,7 +1147,6 @@ aes128_cbc_mac_flush_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes256_cbc_mac_flush_vaes_avx512,function,internal) aes256_cbc_mac_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 13, MAC_TYPE_CBC, FLUSH FUNC_RESTORE @@ -1170,7 +1159,6 @@ aes256_cbc_mac_flush_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_xcbc_mac_128_flush_vaes_avx512,function,internal) aes_xcbc_mac_128_flush_vaes_avx512: - endbranch64 FUNC_SAVE CBC_ENC 9, MAC_TYPE_XCBC, FLUSH FUNC_RESTORE diff --git a/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm index 1f551e02..62df455b 100644 --- a/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_api_by16_vaes_avx512.asm @@ -34,7 +34,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_128_submit_vaes_avx512,function,internal) aes_cntr_128_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR ;; arg1 - [in] job ;; arg2 - [in] NROUNDS @@ -49,7 +48,6 @@ aes_cntr_128_submit_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_192_submit_vaes_avx512,function,internal) aes_cntr_192_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR ;; arg1 - [in] job ;; arg2 - [in] NROUNDS @@ -64,7 +62,6 @@ aes_cntr_192_submit_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_256_submit_vaes_avx512,function,internal) aes_cntr_256_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR ;; arg1 - [in] job ;; arg2 - [in] NROUNDS diff --git a/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm index ef1df535..431a8119 100644 --- a/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_bit_api_by16_vaes_avx512.asm @@ -34,7 +34,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_bit_128_submit_vaes_avx512,function,internal) aes_cntr_bit_128_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR_BIT ;; arg1 - [in] job ;; arg2 - [in] NROUNDS @@ -49,7 +48,6 @@ aes_cntr_bit_128_submit_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_bit_192_submit_vaes_avx512,function,internal) aes_cntr_bit_192_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR_BIT ;; arg1 - [in] job ;; arg2 - [in] NROUNDS @@ -64,7 +62,6 @@ aes_cntr_bit_192_submit_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_bit_256_submit_vaes_avx512,function,internal) aes_cntr_bit_256_submit_vaes_avx512: - endbranch64 FUNC_SAVE CNTR_BIT ;; arg1 - [in] job ;; arg2 - [in] NROUNDS diff --git a/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc b/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc index 6e3fc34a..8e859aaa 100644 --- a/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc +++ b/lib/avx512_t2/aes_cntr_by16_vaes_avx512.inc @@ -35,11 +35,10 @@ %include "include/mb_mgr_datastruct.asm" %include "include/imb_job.asm" %include "include/memcpy.asm" -%include "include/cet.inc" %include "include/aes_common.asm" %include "include/const.inc" %include "include/clear_regs.asm" -%include "include/cet.inc" + mksection .rodata default rel diff --git a/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm index f28dcf72..6dca7572 100644 --- a/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_ccm_api_by16_vaes_avx512.asm @@ -34,7 +34,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_ccm_128_vaes_avx512,function,internal) aes_cntr_ccm_128_vaes_avx512: - endbranch64 FUNC_SAVE CNTR ;; arg1 - [in] job ;; arg2 - [in] NROUNDS @@ -49,7 +48,6 @@ aes_cntr_ccm_128_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_ccm_256_vaes_avx512,function,internal) aes_cntr_ccm_256_vaes_avx512: - endbranch64 FUNC_SAVE CNTR ;; arg1 - [in] job ;; arg2 - [in] NROUNDS diff --git a/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm index 78754018..74d59b4c 100644 --- a/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cntr_pon_api_by16_vaes_avx512.asm @@ -35,7 +35,6 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_pon_enc_128_vaes_avx512,function,internal) aes_cntr_pon_enc_128_vaes_avx512: - endbranch64 CNTR_PON_ENC_DEC ENCRYPT ret @@ -45,8 +44,6 @@ aes_cntr_pon_enc_128_vaes_avx512: ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cntr_pon_dec_128_vaes_avx512,function,internal) aes_cntr_pon_dec_128_vaes_avx512: - endbranch64 - CNTR_PON_ENC_DEC DECRYPT ret diff --git a/lib/avx512_t2/aes_ecb_vaes_avx512.asm b/lib/avx512_t2/aes_ecb_vaes_avx512.asm index 8159c5c1..a4287a30 100644 --- a/lib/avx512_t2/aes_ecb_vaes_avx512.asm +++ b/lib/avx512_t2/aes_ecb_vaes_avx512.asm @@ -45,7 +45,7 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/aes_common.asm" -%include "include/cet.inc" + %define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx512 %define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx512 %define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx512 @@ -203,14 +203,12 @@ align 16 align 16 MKGLOBAL(AES_ECB_ENC_128,function,internal) AES_ECB_ENC_128: - endbranch64 AES_ECB 10, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_128,function,internal) AES_ECB_DEC_128: - endbranch64 AES_ECB 10, DEC ret @@ -220,14 +218,12 @@ AES_ECB_DEC_128: align 16 MKGLOBAL(AES_ECB_ENC_192,function,internal) AES_ECB_ENC_192: - endbranch64 AES_ECB 12, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_192,function,internal) AES_ECB_DEC_192: - endbranch64 AES_ECB 12, DEC ret @@ -237,14 +233,12 @@ AES_ECB_DEC_192: align 16 MKGLOBAL(AES_ECB_ENC_256,function,internal) AES_ECB_ENC_256: - endbranch64 AES_ECB 14, ENC ret align 16 MKGLOBAL(AES_ECB_DEC_256,function,internal) AES_ECB_DEC_256: - endbranch64 AES_ECB 14, DEC ret diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm index 48dd36bf..90bfd104 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm @@ -30,7 +30,6 @@ %include "include/mb_mgr_datastruct.asm" %include "include/constants.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %include "include/clear_regs.asm" %ifndef AES_CBC_ENC_X16 @@ -156,7 +155,6 @@ endstruc ; arg 2 : job MKGLOBAL(FLUSH_JOB_AES_ENC,function,internal) FLUSH_JOB_AES_ENC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm index 98b3952d..2acf75f7 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" @@ -141,7 +140,6 @@ endstruc ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_ENC,function,internal) SUBMIT_JOB_AES_ENC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index a3d0696c..250d77ac 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -41,13 +41,11 @@ uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, const IMB_KEY_SIZE_BYTES key_size, const int run_check) { - uint32_t i, completed_jobs = 0; - MB_MGR_AES_OOO *aes_ooo = NULL; - - IMB_JOB * (*submit_fn)(MB_MGR_AES_OOO *state, IMB_JOB *job) = NULL; - IMB_JOB * (*flush_fn)(MB_MGR_AES_OOO *state) = NULL; + uint32_t completed_jobs = 0; if (run_check) { + uint32_t i; + /* validate jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -62,36 +60,74 @@ uint32_t submit_aes_cbc_burst_enc(IMB_MGR *state, } } - if (key_size == 16) { - aes_ooo = state->aes128_ooo; - submit_fn = SUBMIT_JOB_AES_CBC_128_ENC; - flush_fn = FLUSH_JOB_AES_CBC_128_ENC; - } else if (key_size == 24) { - aes_ooo = state->aes192_ooo; - submit_fn = SUBMIT_JOB_AES_CBC_192_ENC; - flush_fn = FLUSH_JOB_AES_CBC_192_ENC; - } else { /* assume 32 */ - aes_ooo = state->aes256_ooo; - submit_fn = SUBMIT_JOB_AES_CBC_256_ENC; - flush_fn = FLUSH_JOB_AES_CBC_256_ENC; - } + if (key_size == IMB_KEY_128_BYTES) { + MB_MGR_AES_OOO *aes_ooo = state->aes128_ooo; + uint32_t i; - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; - job = submit_fn(aes_ooo, job); - if (job != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; + job = SUBMIT_JOB_AES_CBC_128_ENC(aes_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } } - } - if (completed_jobs != n_jobs) { - IMB_JOB *job = NULL; + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; - while((job = flush_fn(aes_ooo)) != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; + while((job = FLUSH_JOB_AES_CBC_128_ENC(aes_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (key_size == IMB_KEY_192_BYTES) { + MB_MGR_AES_OOO *aes_ooo = state->aes192_ooo; + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_AES_CBC_192_ENC(aes_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while((job = FLUSH_JOB_AES_CBC_192_ENC(aes_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else { /* assume 256-bit key */ + MB_MGR_AES_OOO *aes_ooo = state->aes256_ooo; + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_AES_CBC_256_ENC(aes_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while((job = FLUSH_JOB_AES_CBC_256_ENC(aes_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } } } @@ -105,13 +141,11 @@ uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, const IMB_KEY_SIZE_BYTES key_size, const int run_check) { - uint32_t i, completed_jobs = 0; - void (*submit_fn) (const void *in, const uint8_t *IV, - const void *keys, void *out, - uint64_t len_bytes) = NULL; (void) state; if (run_check) { + uint32_t i; + /* validate jobs */ for (i = 0; i < n_jobs; i++) { IMB_JOB *job = &jobs[i]; @@ -126,26 +160,54 @@ uint32_t submit_aes_cbc_burst_dec(IMB_MGR *state, } } - if (key_size == 16) - submit_fn = AES_CBC_DEC_128; - else if (key_size == 24) - submit_fn = AES_CBC_DEC_192; - else /* assume 32 */ - submit_fn = AES_CBC_DEC_256; + if (key_size == IMB_KEY_128_BYTES) { + uint32_t i; - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_CBC_DEC_128(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & + (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else if (key_size == IMB_KEY_192_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_CBC_DEC_192(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & + (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else /* assume 256-bit key */ { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; - submit_fn(job->src + job->cipher_start_src_offset_in_bytes, - job->iv, - job->dec_keys, - job->dst, - job->msg_len_to_cipher_in_bytes & (~15)); - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; + AES_CBC_DEC_256(job->src + + job->cipher_start_src_offset_in_bytes, + job->iv, + job->dec_keys, + job->dst, + job->msg_len_to_cipher_in_bytes & + (~15)); + job->status = IMB_STATUS_COMPLETED; + } } - return completed_jobs; + return n_jobs; } __forceinline diff --git a/lib/sse_t1/aes128_cbc_enc_x4_sse.asm b/lib/sse_t1/aes128_cbc_enc_x4_sse.asm index d84d9164..a2713c2c 100644 --- a/lib/sse_t1/aes128_cbc_enc_x4_sse.asm +++ b/lib/sse_t1/aes128_cbc_enc_x4_sse.asm @@ -32,7 +32,6 @@ %include "include/os.asm" %include "include/mb_mgr_datastruct.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define MOVDQ movdqu ;; assume buffers not aligned %macro pxor2 2 @@ -378,7 +377,6 @@ mksection .text MKGLOBAL(FUNC,function,internal) FUNC: - endbranch64 %ifdef ARG_OUT AES_CBC_X4 MODE, OFFSET, ARG_IV, ARG_KEYS, ARG_IN, ARG_OUT %else diff --git a/lib/sse_t1/aes128_cntr_by8_sse.asm b/lib/sse_t1/aes128_cntr_by8_sse.asm index d0b0ebc7..7005d563 100644 --- a/lib/sse_t1/aes128_cntr_by8_sse.asm +++ b/lib/sse_t1/aes128_cntr_by8_sse.asm @@ -31,7 +31,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES128 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -561,21 +560,22 @@ align 32 jmp %%bswap_iv %endmacro -align 32 %ifdef CNTR_CCM_SSE ; IMB_JOB * aes_cntr_ccm_128_sse(IMB_JOB *job) ; arg 1 : job +align 32 MKGLOBAL(AES_CNTR_CCM_128,function,internal) AES_CNTR_CCM_128: DO_CNTR CCM %else ;; aes_cntr_128_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) +align 32 MKGLOBAL(AES_CNTR_128,function,internal) AES_CNTR_128: - endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_128_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) +align 32 MKGLOBAL(AES_CNTR_BIT_128,function,internal) AES_CNTR_BIT_128: DO_CNTR CNTR_BIT diff --git a/lib/sse_t1/aes192_cntr_by8_sse.asm b/lib/sse_t1/aes192_cntr_by8_sse.asm index ec894769..d0a861dd 100644 --- a/lib/sse_t1/aes192_cntr_by8_sse.asm +++ b/lib/sse_t1/aes192_cntr_by8_sse.asm @@ -30,7 +30,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES192 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -497,9 +496,9 @@ align 32 ;; aes_cntr_192_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) MKGLOBAL(AES_CNTR_192,function,internal) AES_CNTR_192: - endbranch64 DO_CNTR CNTR +align 32 ;; aes_cntr_bit_192_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) MKGLOBAL(AES_CNTR_BIT_192,function,internal) AES_CNTR_BIT_192: diff --git a/lib/sse_t1/aes256_cntr_by8_sse.asm b/lib/sse_t1/aes256_cntr_by8_sse.asm index 3964bc10..c9ca9877 100644 --- a/lib/sse_t1/aes256_cntr_by8_sse.asm +++ b/lib/sse_t1/aes256_cntr_by8_sse.asm @@ -31,7 +31,6 @@ %include "include/const.inc" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" ; routine to do AES256 CNTR enc/decrypt "by8" ; XMM registers are clobbered. Saving/restoring must be done at a higher level @@ -589,21 +588,22 @@ align 32 jmp %%_bswap_iv %endmacro -align 32 %ifdef CNTR_CCM_SSE ; IMB_JOB * aes_cntr_ccm_256_(IMB_JOB *job) ; arg 1 : job +align 32 MKGLOBAL(AES_CNTR_CCM_256,function,internal) AES_CNTR_CCM_256: DO_CNTR CCM %else ;; aes_cntr_256_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bytes, UINT64 iv_len) +align 32 MKGLOBAL(AES_CNTR_256,function,internal) AES_CNTR_256: - endbranch64 DO_CNTR CNTR ;; aes_cntr_bit_256_sse(void *in, void *IV, void *keys, void *out, UINT64 num_bits, UINT64 iv_len) +align 32 MKGLOBAL(AES_CNTR_BIT_256,function,internal) AES_CNTR_BIT_256: DO_CNTR CNTR_BIT diff --git a/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm index 4c7a5146..35362ca7 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbc_enc_flush_x4_sse.asm @@ -30,7 +30,6 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %ifndef NUM_LANES %define NUM_LANES 4 @@ -120,8 +119,6 @@ endstruc ; arg 2 : job MKGLOBAL(FLUSH_JOB_AES_ENC,function,internal) FLUSH_JOB_AES_ENC: - endbranch64 - mov rax, rsp sub rsp, STACK_size and rsp, -16 @@ -226,7 +223,6 @@ APPEND(skip_clear_,I): %endif return: - endbranch64 mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm index d692dede..0c67da09 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbc_enc_submit_x4_sse.asm @@ -31,7 +31,6 @@ %include "include/reg_sizes.asm" %include "include/const.inc" -%include "include/cet.inc" %ifndef NUM_LANES %define NUM_LANES 4 %endif @@ -91,7 +90,6 @@ mksection .text ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_ENC,function,internal) SUBMIT_JOB_AES_ENC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -16 @@ -176,7 +174,6 @@ len_is_0: %endif return: - endbranch64 mov rbx, [rsp + _gpr_save + 8*0] mov rbp, [rsp + _gpr_save + 8*1] mov r12, [rsp + _gpr_save + 8*2] diff --git a/lib/sse_t1/sha1_x4_sse.asm b/lib/sse_t1/sha1_x4_sse.asm index ac0b131b..b648c5ff 100644 --- a/lib/sse_t1/sha1_x4_sse.asm +++ b/lib/sse_t1/sha1_x4_sse.asm @@ -29,7 +29,6 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %include "include/clear_regs.asm" diff --git a/lib/sse_t1/zuc_x4_sse.asm b/lib/sse_t1/zuc_x4_sse.asm index 55aaf3fe..c1410580 100644 --- a/lib/sse_t1/zuc_x4_sse.asm +++ b/lib/sse_t1/zuc_x4_sse.asm @@ -30,7 +30,6 @@ %include "include/zuc_sbox.inc" %include "include/memcpy.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/const.inc" %ifndef ZUC_CIPHER_4 diff --git a/lib/sse_t2/sha1_ni_x1_sse.asm b/lib/sse_t2/sha1_ni_x1_sse.asm index 2e7eb8a1..9fdb2e73 100644 --- a/lib/sse_t2/sha1_ni_x1_sse.asm +++ b/lib/sse_t2/sha1_ni_x1_sse.asm @@ -42,7 +42,6 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %ifdef LINUX diff --git a/lib/sse_t2/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm index f32f1956..8d1c30e7 100644 --- a/lib/sse_t2/sha1_ni_x2_sse.asm +++ b/lib/sse_t2/sha1_ni_x2_sse.asm @@ -42,7 +42,6 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %ifdef LINUX diff --git a/lib/sse_t2/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm index e80ae407..c9aafac1 100644 --- a/lib/sse_t2/sha256_ni_x2_sse.asm +++ b/lib/sse_t2/sha256_ni_x2_sse.asm @@ -41,7 +41,6 @@ %include "include/os.asm" ;%define DO_DBGPRINT %include "include/dbgprint.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %include "include/clear_regs.asm" diff --git a/lib/sse_t3/aes128_cbc_dec_by8_sse.asm b/lib/sse_t3/aes128_cbc_dec_by8_sse.asm index 31c535dc..862edf5a 100644 --- a/lib/sse_t3/aes128_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes128_cbc_dec_by8_sse.asm @@ -43,7 +43,6 @@ ; %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %ifndef AES_CBC_DEC_128 %define AES_CBC_DEC_128 aes_cbc_dec_128_by8_sse @@ -225,7 +224,6 @@ mksection .text align 32 MKGLOBAL(AES_CBC_DEC_128,function,internal) AES_CBC_DEC_128: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif @@ -307,7 +305,6 @@ mult_of_8_blks: movdqa xkey10, [p_keys + 10*16] main_loop2: - endbranch64 main_loop3: ; num_bytes is a multiple of 8 and >0 diff --git a/lib/sse_t3/aes128_cbc_enc_x8_sse.asm b/lib/sse_t3/aes128_cbc_enc_x8_sse.asm index 06f7b4b0..2888cf23 100644 --- a/lib/sse_t3/aes128_cbc_enc_x8_sse.asm +++ b/lib/sse_t3/aes128_cbc_enc_x8_sse.asm @@ -32,7 +32,6 @@ %include "include/os.asm" %include "include/mb_mgr_datastruct.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %macro PXOR2 2 movdqu XTMP, %2 @@ -119,7 +118,6 @@ aes128_cbc_mac_x8_sse: MKGLOBAL(aes_cbc_enc_128_x8_sse,function,internal) aes_cbc_enc_128_x8_sse: %endif - endbranch64 sub rsp, STACK_size mov [GPR_SAVE_AREA + 8*0], rbp %ifdef CBC_MAC diff --git a/lib/sse_t3/aes128_ecb_by8_sse.asm b/lib/sse_t3/aes128_ecb_by8_sse.asm index 84ff93e7..9c6a5aed 100644 --- a/lib/sse_t3/aes128_ecb_by8_sse.asm +++ b/lib/sse_t3/aes128_ecb_by8_sse.asm @@ -30,7 +30,6 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/aes_common.asm" -%include "include/cet.inc" %ifdef LINUX %define IN rdi @@ -155,13 +154,11 @@ mksection .text align 16 MKGLOBAL(AES_ECB_ENC,function,internal) AES_ECB_ENC: - endbranch64 AES_ECB ENC ret align 16 MKGLOBAL(AES_ECB_DEC,function,internal) AES_ECB_DEC: - endbranch64 AES_ECB DEC ret diff --git a/lib/sse_t3/aes192_cbc_dec_by8_sse.asm b/lib/sse_t3/aes192_cbc_dec_by8_sse.asm index 2451c3a8..c04f546b 100644 --- a/lib/sse_t3/aes192_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes192_cbc_dec_by8_sse.asm @@ -43,7 +43,7 @@ ; %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + %ifndef AES_CBC_DEC_192 %define AES_CBC_DEC_192 aes_cbc_dec_192_by8_sse %endif @@ -248,7 +248,6 @@ align 32 ;; AES_CBC_DEC_192(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(AES_CBC_DEC_192,function,internal) AES_CBC_DEC_192: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif diff --git a/lib/sse_t3/aes256_cbc_dec_by8_sse.asm b/lib/sse_t3/aes256_cbc_dec_by8_sse.asm index 5dfe6ea7..bc0fb718 100644 --- a/lib/sse_t3/aes256_cbc_dec_by8_sse.asm +++ b/lib/sse_t3/aes256_cbc_dec_by8_sse.asm @@ -44,7 +44,7 @@ %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + %ifndef AES_CBC_DEC_256 %define AES_CBC_DEC_256 aes_cbc_dec_256_by8_sse %endif @@ -265,7 +265,6 @@ align 32 ;; AES_CBC_DEC_256(void *in, void *IV, void *keys, void *out, UINT64 num_bytes) MKGLOBAL(AES_CBC_DEC_256,function,internal) AES_CBC_DEC_256: - endbranch64 %ifndef LINUX mov num_bytes, [rsp + 8*5] %endif -- GitLab From e15b734f0f7999071b66b4464a3c14f5064afd87 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 25 Aug 2022 14:45:33 +0000 Subject: [PATCH 333/369] avx2: [MB_MGR] split into type-1 and type-2 manager implementations --- lib/Makefile | 2 + lib/avx2_t1/mb_mgr_avx2.c | 499 +++-------------------------------- lib/avx2_t1/mb_mgr_avx2_t1.c | 450 +++++++++++++++++++++++++++++++ lib/avx2_t2/mb_mgr_avx2_t2.c | 451 +++++++++++++++++++++++++++++++ lib/include/ipsec_ooo_mgr.h | 91 +++++-- lib/intel-ipsec-mb.h | 3 + lib/libIPSec_MB.def | 174 +++++++----- lib/win_x64.mak | 2 + 8 files changed, 1132 insertions(+), 540 deletions(-) create mode 100644 lib/avx2_t1/mb_mgr_avx2_t1.c create mode 100644 lib/avx2_t2/mb_mgr_avx2_t2.c diff --git a/lib/Makefile b/lib/Makefile index c3db3749..838d6a1c 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -248,6 +248,8 @@ SAFE_OPTIONS_MSG2="All safe options enabled by default." c_lib_objs := \ mb_mgr_avx.o \ mb_mgr_avx2.o \ + mb_mgr_avx2_t1.o \ + mb_mgr_avx2_t2.o \ mb_mgr_avx512.o \ mb_mgr_avx512_t1.o \ mb_mgr_avx512_t2.o \ diff --git a/lib/avx2_t1/mb_mgr_avx2.c b/lib/avx2_t1/mb_mgr_avx2.c index 990f7edc..12370752 100644 --- a/lib/avx2_t1/mb_mgr_avx2.c +++ b/lib/avx2_t1/mb_mgr_avx2.c @@ -25,311 +25,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *******************************************************************************/ -#include -#include -#include - -#define AVX2 - #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" -#include "include/kasumi_interface.h" -#include "include/zuc_internal.h" -#include "include/snow3g.h" -#include "include/gcm.h" -#include "include/chacha20_poly1305.h" - -#include "include/save_xmms.h" -#include "include/des.h" #include "include/cpu_feature.h" -#include "include/noaesni.h" #include "include/aesni_emu.h" #include "include/error.h" - -#include "include/arch_x86_64.h" /* self-test */ -#include "include/arch_sse_type1.h" /* poly1305 */ -#include "include/arch_avx_type1.h" -#include "include/arch_avx2_type1.h" -#include "include/arch_avx2_type2.h" - -#include "include/ooo_mgr_reset.h" - -#define SAVE_XMMS save_xmms_avx -#define RESTORE_XMMS restore_xmms_avx - -#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx -#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx -#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx - -#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx -#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx -#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx - -#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx -#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx -#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx - -#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx2 -#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx2 -#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx2 -#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx2 -#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx2 -#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx2 - -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx2 -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx - -#define AES_CTR_128 aes_cntr_128_avx -#define AES_CTR_192 aes_cntr_192_avx -#define AES_CTR_256 aes_cntr_256_avx -#define AES_CTR_128_BIT aes_cntr_bit_128_avx -#define AES_CTR_192_BIT aes_cntr_bit_192_avx -#define AES_CTR_256_BIT aes_cntr_bit_256_avx - -#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx -#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx - -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx2 -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx2 -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx2 -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx2 -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx2 -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx2 - -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx - -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 - -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx2 -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx2 - -#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx -#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx - -#define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 -#define FLUSH_JOB_SHA1 flush_job_sha1_avx2 -#define SUBMIT_JOB_SHA224 submit_job_sha224_avx2 -#define FLUSH_JOB_SHA224 flush_job_sha224_avx2 -#define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 -#define FLUSH_JOB_SHA256 flush_job_sha256_avx2 -#define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 -#define FLUSH_JOB_SHA384 flush_job_sha384_avx2 -#define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 -#define FLUSH_JOB_SHA512 flush_job_sha512_avx2 - -#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx -#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx -#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx -#define QUEUE_SIZE queue_size_avx2 - -#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX2 -#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX2 -#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX2 - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx2 -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - -#define SUBMIT_JOB_HMAC submit_job_hmac_avx2 -#define FLUSH_JOB_HMAC flush_job_hmac_avx2 -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx2 -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx2 -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx2 -#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 -#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 -#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 -#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx2 -#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 -#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 - -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx2 -#define FLUSH_JOB flush_job_avx2 -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx2 -#define QUEUE_SIZE queue_size_avx2 -#define GET_NEXT_JOB get_next_job_avx2 -#define GET_COMPLETED_JOB get_completed_job_avx2 -#define GET_NEXT_BURST get_next_burst_avx2 -#define SUBMIT_BURST submit_burst_avx2 -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2 -#define FLUSH_BURST flush_burst_avx2 -#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2 -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2 -#define SUBMIT_HASH_BURST submit_hash_burst_avx2 -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2 - -/* ====================================================================== */ - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX2 - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_avx2 -#define AES_CFB_256_ONE aes_cfb_256_one_avx2 - -#define AES128_CBC_MAC aes128_cbc_mac_x8 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx - -/* ====================================================================== */ - -uint32_t ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); - -#define ETHERNET_FCS ethernet_fcs_avx_local - -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx - -/* ====================================================================== */ - -/* - * ECB function pointers - */ -static void -(*aes_ecb_enc_128_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_128_avx; - -static void -(*aes_ecb_enc_192_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_192_avx; - -static void -(*aes_ecb_enc_256_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_enc_256_avx; - -static void -(*aes_ecb_dec_128_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_128_avx; - -static void -(*aes_ecb_dec_192_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_192_avx; - -static void -(*aes_ecb_dec_256_avx2) (const void *in, const void *keys, - void *out, uint64_t len_bytes)= - aes_ecb_dec_256_avx; - -/* ====================================================================== */ - -static void -reset_ooo_mgrs(IMB_MGR *state) -{ - /* Init AES out-of-order fields */ - ooo_mgr_aes_reset(state->aes128_ooo, 8); - ooo_mgr_aes_reset(state->aes192_ooo, 8); - ooo_mgr_aes_reset(state->aes256_ooo, 8); - - /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) - * uses same settings as AES CBC. - */ - ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); - ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); - - /* Init ZUC out-of-order fields */ - ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 8); - ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 8); - ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 8); - ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 8); - - /* Init HMAC/SHA1 out-of-order fields */ - ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); - - /* Init HMAC/SHA224 out-of-order fields */ - ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, - AVX2_NUM_SHA256_LANES); - - /* Init HMAC/SHA256 out-of-order fields */ - ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, - AVX2_NUM_SHA256_LANES); - - /* Init HMAC/SHA384 out-of-order fields */ - ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, - AVX2_NUM_SHA512_LANES); - - /* Init HMAC/SHA512 out-of-order fields */ - ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, - AVX2_NUM_SHA512_LANES); - - /* Init HMAC/MD5 out-of-order fields */ - ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); - - /* Init AES/XCBC OOO fields */ - ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); - - /* Init AES-CCM auth out-of-order fields */ - ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); - ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); - - /* Init AES-CMAC auth out-of-order fields */ - ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); - ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); - - /* Init AES CBC-S out-of-order fields */ - ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); - - /* Init SHA1 out-of-order fields */ - ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); - - /* Init SHA224 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, AVX2_NUM_SHA256_LANES); - - /* Init SHA256 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); - - /* Init SHA384 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); - - /* Init SHA512 out-of-order fields */ - ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); -} +#include "include/arch_x86_64.h" IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) @@ -341,170 +42,22 @@ init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs) } #endif - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - - - /* Check if CPU flags needed for AVX2 interface are present */ - if ((state->features & IMB_CPUFLAGS_AVX2) != IMB_CPUFLAGS_AVX2) { - imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + fallback_no_aesni(state, 1); return; } - /* Set architecture for future checks */ - state->used_arch = (uint32_t) IMB_ARCH_AVX2; - - if ((state->features & IMB_FEATURE_VAES) == IMB_FEATURE_VAES) { - aes_ecb_enc_128_avx2 = aes_ecb_enc_128_vaes_avx2; - aes_ecb_enc_192_avx2 = aes_ecb_enc_192_vaes_avx2; - aes_ecb_enc_256_avx2 = aes_ecb_enc_256_vaes_avx2; - aes_ecb_dec_128_avx2 = aes_ecb_dec_128_vaes_avx2; - aes_ecb_dec_192_avx2 = aes_ecb_dec_192_vaes_avx2; - aes_ecb_dec_256_avx2 = aes_ecb_dec_256_vaes_avx2; - } - - if (reset_mgrs) { - reset_ooo_mgrs(state); - - /* Init "in order" components */ - state->next_job = 0; - state->earliest_job = -1; - } - - /* set handlers */ - state->get_next_job = get_next_job_avx2; - state->submit_job = submit_job_avx2; - state->get_next_burst = get_next_burst_avx2; - state->submit_burst = submit_burst_avx2; - state->submit_burst_nocheck= submit_burst_nocheck_avx2; - state->flush_burst = flush_burst_avx2; - state->submit_cipher_burst = submit_cipher_burst_avx2; - state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx2; - state->submit_hash_burst = submit_hash_burst_avx2; - state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx2; - state->submit_job_nocheck = submit_job_nocheck_avx2; - state->get_completed_job = get_completed_job_avx2; - state->flush_job = flush_job_avx2; - state->queue_size = queue_size_avx2; - state->keyexp_128 = aes_keyexp_128_avx2; - state->keyexp_192 = aes_keyexp_192_avx2; - state->keyexp_256 = aes_keyexp_256_avx2; - state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx2; - state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx2; - state->xcbc_keyexp = aes_xcbc_expand_key_avx2; - state->des_key_sched = des_key_schedule; - state->sha1_one_block = sha1_one_block_avx2; - state->sha1 = sha1_avx2; - state->sha224_one_block = sha224_one_block_avx2; - state->sha224 = sha224_avx2; - state->sha256_one_block = sha256_one_block_avx2; - state->sha256 = sha256_avx2; - state->sha384_one_block = sha384_one_block_avx2; - state->sha384 = sha384_avx2; - state->sha512_one_block = sha512_one_block_avx2; - state->sha512 = sha512_avx2; - state->md5_one_block = md5_one_block_avx2; - state->aes128_cfb_one = aes_cfb_128_one_avx2; - - state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; - state->eea3_4_buffer = zuc_eea3_4_buffer_avx; - state->eea3_n_buffer = zuc_eea3_n_buffer_avx2; - state->eia3_1_buffer = zuc_eia3_1_buffer_avx2; - state->eia3_n_buffer = zuc_eia3_n_buffer_avx2; - - state->f8_1_buffer = kasumi_f8_1_buffer_avx; - state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; - state->f8_2_buffer = kasumi_f8_2_buffer_avx; - state->f8_3_buffer = kasumi_f8_3_buffer_avx; - state->f8_4_buffer = kasumi_f8_4_buffer_avx; - state->f8_n_buffer = kasumi_f8_n_buffer_avx; - state->f9_1_buffer = kasumi_f9_1_buffer_avx; - state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; - state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; - state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; - state->kasumi_key_sched_size = kasumi_key_sched_size_avx; - - state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; - state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; - state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; - state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; - state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; - state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; - state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; - state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; - state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; - state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; - state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; - - state->gcm128_enc = aes_gcm_enc_128_avx_gen4; - state->gcm192_enc = aes_gcm_enc_192_avx_gen4; - state->gcm256_enc = aes_gcm_enc_256_avx_gen4; - state->gcm128_dec = aes_gcm_dec_128_avx_gen4; - state->gcm192_dec = aes_gcm_dec_192_avx_gen4; - state->gcm256_dec = aes_gcm_dec_256_avx_gen4; - state->gcm128_init = aes_gcm_init_128_avx_gen4; - state->gcm192_init = aes_gcm_init_192_avx_gen4; - state->gcm256_init = aes_gcm_init_256_avx_gen4; - state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; - state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; - state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; - state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; - state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; - state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; - state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; - state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; - state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; - state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; - state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; - state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; - state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; - state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; - state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; - state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; - state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; - state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; - state->gcm128_pre = aes_gcm_pre_128_avx_gen4; - state->gcm192_pre = aes_gcm_pre_192_avx_gen4; - state->gcm256_pre = aes_gcm_pre_256_avx_gen4; - state->ghash = ghash_avx_gen4; - state->ghash_pre = ghash_pre_avx_gen2; + /* reset error status */ + imb_set_errno(state, 0); - state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; - state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; - state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; - state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; - state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; - state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; - state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; - state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; - state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); - state->chacha20_poly1305_init = init_chacha20_poly1305_avx; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx2; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx2; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; + if ((state->features & IMB_CPUFLAGS_AVX2_T2) == + IMB_CPUFLAGS_AVX2_T2) + init_mb_mgr_avx2_t2_internal(state, reset_mgrs); + else + init_mb_mgr_avx2_t1_internal(state, reset_mgrs); } void @@ -516,4 +69,32 @@ init_mb_mgr_avx2(IMB_MGR *state) imb_set_errno(state, IMB_ERR_SELFTEST); } -#include "mb_mgr_code.h" +IMB_JOB *submit_job_avx2(IMB_MGR *state) +{ + return IMB_SUBMIT_JOB(state); +} + +IMB_JOB *flush_job_avx2(IMB_MGR *state) +{ + return IMB_FLUSH_JOB(state); +} + +uint32_t queue_size_avx2(IMB_MGR *state) +{ + return IMB_QUEUE_SIZE(state); +} + +IMB_JOB *submit_job_nocheck_avx2(IMB_MGR *state) +{ + return IMB_SUBMIT_JOB_NOCHECK(state); +} + +IMB_JOB *get_next_job_avx2(IMB_MGR *state) +{ + return IMB_GET_NEXT_JOB(state); +} + +IMB_JOB *get_completed_job_avx2(IMB_MGR *state) +{ + return IMB_GET_COMPLETED_JOB(state); +} diff --git a/lib/avx2_t1/mb_mgr_avx2_t1.c b/lib/avx2_t1/mb_mgr_avx2_t1.c new file mode 100644 index 00000000..6467975d --- /dev/null +++ b/lib/avx2_t1/mb_mgr_avx2_t1.c @@ -0,0 +1,450 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX2 + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_avx_type1.h" +#include "include/arch_avx2_type1.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx2_t1 +#define FLUSH_JOB flush_job_avx2_t1 +#define QUEUE_SIZE queue_size_avx2_t1 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx2_t1 +#define GET_NEXT_JOB get_next_job_avx2_t1 +#define GET_COMPLETED_JOB get_completed_job_avx2_t1 +#define GET_NEXT_BURST get_next_burst_avx2_t1 +#define SUBMIT_BURST submit_burst_avx2_t1 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2_t1 +#define FLUSH_BURST flush_burst_avx2_t1 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2_t1 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2_t1 +#define SUBMIT_HASH_BURST submit_hash_burst_avx2_t1 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2_t1 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX2 + +/* AES-GCM */ +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_gcm_dec_avx2 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx2 + +/* AES-CBC */ +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx + +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx + +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx2 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx2 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx2 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx2 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx2 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx2 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx + +/* AES-CTR */ +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx2 +#define AES_CFB_256_ONE aes_cfb_256_one_avx2 + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx2 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx2 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx2 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx2 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx2 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx2 + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_avx2 +#define FLUSH_JOB_HMAC flush_job_hmac_avx2 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx2 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx2 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx2 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx2 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx2 +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx2 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 8); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX2_NUM_SHA256_LANES); + + /* Init HMAC/SHA256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX2_NUM_SHA256_LANES); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX2_NUM_SHA256_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX2 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX2) != IMB_CPUFLAGS_AVX2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX2; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_avx2; + state->keyexp_192 = aes_keyexp_192_avx2; + state->keyexp_256 = aes_keyexp_256_avx2; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx2; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx2; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx2; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_avx2; + state->sha1 = sha1_avx2; + state->sha224_one_block = sha224_one_block_avx2; + state->sha224 = sha224_avx2; + state->sha256_one_block = sha256_one_block_avx2; + state->sha256 = sha256_avx2; + state->sha384_one_block = sha384_one_block_avx2; + state->sha384 = sha384_avx2; + state->sha512_one_block = sha512_one_block_avx2; + state->sha512 = sha512_avx2; + state->md5_one_block = md5_one_block_avx2; + + state->aes128_cfb_one = aes_cfb_128_one_avx2; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx2; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx2; + state->eia3_n_buffer = zuc_eia3_n_buffer_avx2; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + + state->chacha20_poly1305_init = init_chacha20_poly1305_avx; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx2; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx2; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; + + state->gcm128_enc = aes_gcm_enc_128_avx_gen4; + state->gcm192_enc = aes_gcm_enc_192_avx_gen4; + state->gcm256_enc = aes_gcm_enc_256_avx_gen4; + state->gcm128_dec = aes_gcm_dec_128_avx_gen4; + state->gcm192_dec = aes_gcm_dec_192_avx_gen4; + state->gcm256_dec = aes_gcm_dec_256_avx_gen4; + state->gcm128_init = aes_gcm_init_128_avx_gen4; + state->gcm192_init = aes_gcm_init_192_avx_gen4; + state->gcm256_init = aes_gcm_init_256_avx_gen4; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; + state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; + state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; + state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; + state->gcm128_pre = aes_gcm_pre_128_avx_gen4; + state->gcm192_pre = aes_gcm_pre_192_avx_gen4; + state->gcm256_pre = aes_gcm_pre_256_avx_gen4; + + state->ghash = ghash_avx_gen4; + state->ghash_pre = ghash_pre_avx_gen2; + + state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; + state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; + state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; + state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; + state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; + state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; +} + +#include "mb_mgr_code.h" diff --git a/lib/avx2_t2/mb_mgr_avx2_t2.c b/lib/avx2_t2/mb_mgr_avx2_t2.c new file mode 100644 index 00000000..d9f3c605 --- /dev/null +++ b/lib/avx2_t2/mb_mgr_avx2_t2.c @@ -0,0 +1,451 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX2 + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" + +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/noaesni.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_avx_type1.h" +#include "include/arch_avx2_type1.h" +#include "include/arch_avx2_type2.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx2_t2 +#define FLUSH_JOB flush_job_avx2_t2 +#define QUEUE_SIZE queue_size_avx2_t2 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx2_t2 +#define GET_NEXT_JOB get_next_job_avx2_t2 +#define GET_COMPLETED_JOB get_completed_job_avx2_t2 +#define GET_NEXT_BURST get_next_burst_avx2_t2 +#define SUBMIT_BURST submit_burst_avx2_t2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx2_t2 +#define FLUSH_BURST flush_burst_avx2_t2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx2_t2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx2_t2 +#define SUBMIT_HASH_BURST submit_hash_burst_avx2_t2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx2_t2 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX2 + +/* AES-GCM */ +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen4 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen4 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen4 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen4 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen4 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen4 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_gcm_dec_avx2 +#define SUBMIT_JOB_AES_GCM_ENC submit_job_gcm_enc_avx2 + +/* AES-CBC */ +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx + +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx + +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_vaes_avx2 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_vaes_avx2 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_vaes_avx2 +#define AES_ECB_ENC_192 aes_ecb_enc_192_vaes_avx2 +#define AES_ECB_ENC_256 aes_ecb_enc_256_vaes_avx2 +#define AES_ECB_DEC_128 aes_ecb_dec_128_vaes_avx2 +#define AES_ECB_DEC_192 aes_ecb_dec_192_vaes_avx2 +#define AES_ECB_DEC_256 aes_ecb_dec_256_vaes_avx2 + +/* AES-CTR */ +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx2 +#define AES_CFB_256_ONE aes_cfb_256_one_avx2 + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 +#define FLUSH_JOB_SHA1 flush_job_sha1_avx2 +#define SUBMIT_JOB_SHA224 submit_job_sha224_avx2 +#define FLUSH_JOB_SHA224 flush_job_sha224_avx2 +#define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 +#define FLUSH_JOB_SHA256 flush_job_sha256_avx2 +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 +#define FLUSH_JOB_SHA384 flush_job_sha384_avx2 +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 +#define FLUSH_JOB_SHA512 flush_job_sha512_avx2 + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_avx2 +#define FLUSH_JOB_HMAC flush_job_hmac_avx2 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx2 +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx2 +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx2 +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx2 +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx2 +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx2 + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx2 +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx2 +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx2 +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx2 +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx2 +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx2 +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx2 +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx2 +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx2 +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx2 + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 8); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 8); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, + AVX2_NUM_SHA256_LANES); + + /* Init HMAC/SHA256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, + AVX2_NUM_SHA256_LANES); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX2_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX2_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, AVX2_NUM_SHA256_LANES); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX2 interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX2) != IMB_CPUFLAGS_AVX2) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX2; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_avx2; + state->keyexp_192 = aes_keyexp_192_avx2; + state->keyexp_256 = aes_keyexp_256_avx2; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx2; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx2; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx2; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_avx2; + state->sha1 = sha1_avx2; + state->sha224_one_block = sha224_one_block_avx2; + state->sha224 = sha224_avx2; + state->sha256_one_block = sha256_one_block_avx2; + state->sha256 = sha256_avx2; + state->sha384_one_block = sha384_one_block_avx2; + state->sha384 = sha384_avx2; + state->sha512_one_block = sha512_one_block_avx2; + state->sha512 = sha512_avx2; + state->md5_one_block = md5_one_block_avx2; + + state->aes128_cfb_one = aes_cfb_128_one_avx2; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx2; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx2; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx2; + state->eia3_n_buffer = zuc_eia3_n_buffer_avx2; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx2; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx2; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx2; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx2; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx2; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx2; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx2; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx2; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx2; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx2; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx2; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + + state->chacha20_poly1305_init = init_chacha20_poly1305_avx; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx2; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx2; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; + + state->gcm128_enc = aes_gcm_enc_128_avx_gen4; + state->gcm192_enc = aes_gcm_enc_192_avx_gen4; + state->gcm256_enc = aes_gcm_enc_256_avx_gen4; + state->gcm128_dec = aes_gcm_dec_128_avx_gen4; + state->gcm192_dec = aes_gcm_dec_192_avx_gen4; + state->gcm256_dec = aes_gcm_dec_256_avx_gen4; + state->gcm128_init = aes_gcm_init_128_avx_gen4; + state->gcm192_init = aes_gcm_init_192_avx_gen4; + state->gcm256_init = aes_gcm_init_256_avx_gen4; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen4; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen4; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen4; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen4; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen4; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen4; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen4; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen4; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen4; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen4; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen4; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen4; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen4; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen4; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen4; + state->gcm128_precomp = aes_gcm_precomp_128_avx_gen4; + state->gcm192_precomp = aes_gcm_precomp_192_avx_gen4; + state->gcm256_precomp = aes_gcm_precomp_256_avx_gen4; + state->gcm128_pre = aes_gcm_pre_128_avx_gen4; + state->gcm192_pre = aes_gcm_pre_192_avx_gen4; + state->gcm256_pre = aes_gcm_pre_256_avx_gen4; + + state->ghash = ghash_avx_gen4; + state->ghash_pre = ghash_pre_avx_gen2; + + state->gmac128_init = imb_aes_gmac_init_128_avx_gen4; + state->gmac192_init = imb_aes_gmac_init_192_avx_gen4; + state->gmac256_init = imb_aes_gmac_init_256_avx_gen4; + state->gmac128_update = imb_aes_gmac_update_128_avx_gen4; + state->gmac192_update = imb_aes_gmac_update_192_avx_gen4; + state->gmac256_update = imb_aes_gmac_update_256_avx_gen4; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen4; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen4; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen4; +} + +#include "mb_mgr_code.h" diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 30df925c..72b2aace 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -426,6 +426,10 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void +init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void +init_mb_mgr_avx2_t2_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void init_mb_mgr_avx512_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs); @@ -443,7 +447,9 @@ get_next_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -get_next_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +get_next_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +get_next_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -460,7 +466,9 @@ submit_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +submit_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -482,8 +490,11 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx2(IMB_MGR *state, const uint32_t n_jobs, - IMB_JOB **jobs); +submit_burst_nocheck_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -502,7 +513,9 @@ flush_burst_sse_t3(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -flush_burst_avx2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +flush_burst_avx2_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t +flush_burst_avx2_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx512_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -539,11 +552,17 @@ submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -588,11 +607,17 @@ submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -626,9 +651,13 @@ submit_hash_burst_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t submit_hash_burst_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -658,9 +687,13 @@ submit_hash_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_nocheck_avx2(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx2_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_avx512_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -694,6 +727,22 @@ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse_t3(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_t3(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_t3(IMB_MGR *state); +/* AVX2 TYPE1 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx2_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx2_t1(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx2_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2_t1(IMB_MGR *state); + +/* AVX2 TYPE2 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx2_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx2_t2(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx2_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx2_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx2_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx2_t2(IMB_MGR *state); + /* AVX512 TYPE1 manager functions */ IMB_DLL_EXPORT IMB_JOB *submit_job_avx512_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *flush_job_avx512_t1(IMB_MGR *state); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 2860171b..db7694e9 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1046,6 +1046,9 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_CPUFLAGS_AVX512_T2 (IMB_CPUFLAGS_AVX512 | IMB_FEATURE_VAES | \ IMB_FEATURE_VPCLMULQDQ | IMB_FEATURE_GFNI | \ IMB_FEATURE_AVX512_IFMA | IMB_FEATURE_SHANI) +#define IMB_CPUFLAGS_AVX2_T2 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_SHANI | \ + IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ | \ + IMB_FEATURE_GFNI) /* TOP LEVEL (IMB_MGR) Data structure fields */ diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index b43b5a0e..dfbee2f4 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -544,63 +544,117 @@ EXPORTS imb_get_mb_mgr_size @518 imb_set_pointers_mb_mgr @519 imb_get_feature_flags @520 - get_next_burst_sse @521 - get_next_burst_sse_no_aesni @522 - get_next_burst_avx @523 - get_next_burst_avx2 @524 - get_next_burst_avx512_t1 @525 - get_next_burst_avx512_t2 @526 - submit_burst_sse @527 - submit_burst_sse_no_aesni @528 - submit_burst_avx @529 - submit_burst_avx2 @530 - submit_burst_avx512_t1 @531 - submit_burst_avx512_t2 @532 - submit_burst_nocheck_sse @533 - submit_burst_nocheck_sse_no_aesni @534 - submit_burst_nocheck_avx @535 - submit_burst_nocheck_avx2 @536 - submit_burst_nocheck_avx512_t1 @537 - submit_burst_nocheck_avx512_t2 @538 - flush_burst_sse @539 - flush_burst_sse_no_aesni @540 - flush_burst_avx @541 - flush_burst_avx2 @542 - flush_burst_avx512_t1 @543 - flush_burst_avx512_t2 @544 - submit_cipher_burst_sse @545 - submit_cipher_burst_sse_no_aesni @546 - submit_cipher_burst_avx @547 - submit_cipher_burst_avx2 @548 - submit_cipher_burst_avx512_t1 @549 - submit_cipher_burst_avx512_t2 @550 - submit_cipher_burst_nocheck_sse @551 - submit_cipher_burst_nocheck_sse_no_aesni @552 - submit_cipher_burst_nocheck_avx @553 - submit_cipher_burst_nocheck_avx2 @554 - submit_cipher_burst_nocheck_avx512_t1 @555 - submit_cipher_burst_nocheck_avx512_t2 @556 - submit_hash_burst_sse @557 - submit_hash_burst_sse_no_aesni @558 - submit_hash_burst_avx @559 - submit_hash_burst_avx2 @560 - submit_hash_burst_avx512_t1 @561 - submit_hash_burst_avx512_t2 @562 - submit_hash_burst_nocheck_sse @563 - submit_hash_burst_nocheck_sse_no_aesni @564 - submit_hash_burst_nocheck_avx @565 - submit_hash_burst_nocheck_avx2 @566 - submit_hash_burst_nocheck_avx512_t1 @567 - submit_hash_burst_nocheck_avx512_t2 @568 - flush_job_avx512_t1 @569 - flush_job_avx512_t2 @570 - queue_size_avx512_t1 @571 - queue_size_avx512_t2 @572 - submit_job_avx512_t1 @573 - submit_job_avx512_t2 @574 - submit_job_nocheck_avx512_t1 @575 - submit_job_nocheck_avx512_t2 @576 - get_next_job_avx512_t1 @577 - get_next_job_avx512_t2 @578 - get_completed_job_avx512_t1 @579 - get_completed_job_avx512_t2 @580 + get_next_burst_sse_t1 @521 + get_next_burst_sse_t2 @522 + get_next_burst_sse_t3 @523 + get_next_burst_sse_no_aesni @524 + get_next_burst_avx @525 + get_next_burst_avx2_t1 @526 + get_next_burst_avx2_t2 @527 + get_next_burst_avx512_t1 @528 + get_next_burst_avx512_t2 @529 + submit_burst_sse_t1 @530 + submit_burst_sse_t2 @531 + submit_burst_sse_t3 @532 + submit_burst_sse_no_aesni @533 + submit_burst_avx @534 + submit_burst_avx2_t1 @535 + submit_burst_avx2_t2 @536 + submit_burst_avx512_t1 @537 + submit_burst_avx512_t2 @538 + submit_burst_nocheck_sse_t1 @539 + submit_burst_nocheck_sse_t2 @540 + submit_burst_nocheck_sse_t3 @541 + submit_burst_nocheck_sse_no_aesni @542 + submit_burst_nocheck_avx @543 + submit_burst_nocheck_avx2_t1 @544 + submit_burst_nocheck_avx2_t2 @545 + submit_burst_nocheck_avx512_t1 @546 + submit_burst_nocheck_avx512_t2 @547 + flush_burst_sse_t1 @548 + flush_burst_sse_t2 @549 + flush_burst_sse_t3 @550 + flush_burst_sse_no_aesni @551 + flush_burst_avx @552 + flush_burst_avx2_t1 @553 + flush_burst_avx2_t2 @554 + flush_burst_avx512_t1 @555 + flush_burst_avx512_t2 @556 + submit_cipher_burst_sse_t1 @557 + submit_cipher_burst_sse_t2 @558 + submit_cipher_burst_sse_t3 @559 + submit_cipher_burst_sse_no_aesni @560 + submit_cipher_burst_avx @561 + submit_cipher_burst_avx2_t1 @562 + submit_cipher_burst_avx2_t2 @563 + submit_cipher_burst_avx512_t1 @564 + submit_cipher_burst_avx512_t2 @565 + submit_cipher_burst_nocheck_sse_t1 @566 + submit_cipher_burst_nocheck_sse_t2 @567 + submit_cipher_burst_nocheck_sse_t3 @568 + submit_cipher_burst_nocheck_sse_no_aesni @569 + submit_cipher_burst_nocheck_avx @570 + submit_cipher_burst_nocheck_avx2_t1 @571 + submit_cipher_burst_nocheck_avx2_t2 @572 + submit_cipher_burst_nocheck_avx512_t1 @573 + submit_cipher_burst_nocheck_avx512_t2 @574 + submit_hash_burst_sse_t1 @575 + submit_hash_burst_sse_t2 @576 + submit_hash_burst_sse_t3 @577 + submit_hash_burst_sse_no_aesni @578 + submit_hash_burst_avx @579 + submit_hash_burst_avx2_t1 @580 + submit_hash_burst_avx2_t2 @581 + submit_hash_burst_avx512_t1 @582 + submit_hash_burst_avx512_t2 @583 + submit_hash_burst_nocheck_sse_t1 @584 + submit_hash_burst_nocheck_sse_t2 @585 + submit_hash_burst_nocheck_sse_t3 @586 + submit_hash_burst_nocheck_sse_no_aesni @587 + submit_hash_burst_nocheck_avx @588 + submit_hash_burst_nocheck_avx2_t1 @589 + submit_hash_burst_nocheck_avx2_t2 @590 + submit_hash_burst_nocheck_avx512_t1 @591 + submit_hash_burst_nocheck_avx512_t2 @592 + flush_job_sse_t1 @593 + flush_job_sse_t2 @594 + flush_job_sse_t3 @595 + flush_job_avx2_t1 @596 + flush_job_avx2_t2 @597 + flush_job_avx512_t1 @598 + flush_job_avx512_t2 @599 + queue_size_sse_t1 @600 + queue_size_sse_t2 @601 + queue_size_sse_t3 @602 + queue_size_avx2_t1 @603 + queue_size_avx2_t2 @604 + queue_size_avx512_t1 @605 + queue_size_avx512_t2 @606 + submit_job_sse_t1 @607 + submit_job_sse_t2 @608 + submit_job_sse_t3 @609 + submit_job_avx2_t1 @610 + submit_job_avx2_t2 @611 + submit_job_avx512_t1 @612 + submit_job_avx512_t2 @613 + submit_job_nocheck_sse_t1 @614 + submit_job_nocheck_sse_t2 @615 + submit_job_nocheck_sse_t3 @616 + submit_job_nocheck_avx2_t1 @617 + submit_job_nocheck_avx2_t2 @618 + submit_job_nocheck_avx512_t1 @619 + submit_job_nocheck_avx512_t2 @620 + get_next_job_sse_t1 @621 + get_next_job_sse_t2 @622 + get_next_job_sse_t3 @623 + get_next_job_avx2_t1 @624 + get_next_job_avx2_t2 @625 + get_next_job_avx512_t1 @626 + get_next_job_avx512_t2 @627 + get_completed_job_sse_t1 @628 + get_completed_job_sse_t2 @629 + get_completed_job_sse_t3 @630 + get_completed_job_avx2_t1 @631 + get_completed_job_avx2_t2 @632 + get_completed_job_avx512_t1 @633 + get_completed_job_avx512_t2 @634 diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 4ba3e524..1ae61583 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -449,6 +449,8 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_avx512.obj \ $(OBJ_DIR)\mb_mgr_avx.obj \ $(OBJ_DIR)\mb_mgr_avx2.obj \ + $(OBJ_DIR)\mb_mgr_avx2_t1.obj \ + $(OBJ_DIR)\mb_mgr_avx2_t2.obj \ $(OBJ_DIR)\mb_mgr_avx512.obj \ $(OBJ_DIR)\mb_mgr_avx512_t1.obj \ $(OBJ_DIR)\mb_mgr_avx512_t2.obj \ -- GitLab From c1c78939951224ac0942adc32204b081b649bb06 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 25 Aug 2022 21:01:30 +0000 Subject: [PATCH 334/369] lib: [MB_MGR] remove calls through functions pointers in hash burst API --- lib/include/mb_mgr_burst.h | 207 +++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 90 deletions(-) diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 250d77ac..3ed6a673 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -361,10 +361,7 @@ uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, - const IMB_HASH_ALG hash_alg, - void *ooo_mgr, - IMB_JOB *(*submit_fn)(void *, IMB_JOB *), - IMB_JOB *(*flush_fn)(void *)) + const IMB_HASH_ALG hash_alg) { uint32_t i, completed_jobs = 0; @@ -384,93 +381,123 @@ uint32_t submit_burst_hmac_sha_x(IMB_MGR *state, } } } - /* submit all jobs */ - for (i = 0; i < n_jobs; i++) { - IMB_JOB *job = &jobs[i]; - job = submit_fn(ooo_mgr, job); - if (job != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; + if (hash_alg == IMB_AUTH_HMAC_SHA_1) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_HMAC(state->hmac_sha_1_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } } - } - /* flush any outstanding jobs */ - if (completed_jobs != n_jobs) { - IMB_JOB *job = NULL; + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; - while ((job = flush_fn(ooo_mgr)) != NULL) { - job->status = IMB_STATUS_COMPLETED; - completed_jobs++; + while ((job = FLUSH_JOB_HMAC(state->hmac_sha_1_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } } - } + } else if (hash_alg == IMB_AUTH_HMAC_SHA_224) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; - return completed_jobs; -} + job = SUBMIT_JOB_HMAC_SHA_224(state->hmac_sha_224_ooo, + job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; -__forceinline -uint32_t submit_burst_hmac_sha_1(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_1, - (void *)state->hmac_sha_1_ooo, - (void *)SUBMIT_JOB_HMAC, - (void *)FLUSH_JOB_HMAC); -} + while ((job = + FLUSH_JOB_HMAC_SHA_224(state->hmac_sha_224_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_HMAC_SHA_256) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; -__forceinline -uint32_t submit_burst_hmac_sha_224(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_224, - (void *)state->hmac_sha_224_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_224, - (void *)FLUSH_JOB_HMAC_SHA_224); + job = SUBMIT_JOB_HMAC_SHA_256(state->hmac_sha_256_ooo, + job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; -} + while ((job = + FLUSH_JOB_HMAC_SHA_256(state->hmac_sha_256_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_HMAC_SHA_384) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; -__forceinline -uint32_t submit_burst_hmac_sha_256(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_256, - (void *)state->hmac_sha_256_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_256, - (void *)FLUSH_JOB_HMAC_SHA_256); -} + job = SUBMIT_JOB_HMAC_SHA_384(state->hmac_sha_384_ooo, + job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; -__forceinline -uint32_t submit_burst_hmac_sha_384(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_384, - (void *)state->hmac_sha_384_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_384, - (void *)FLUSH_JOB_HMAC_SHA_384); -} + while ((job = + FLUSH_JOB_HMAC_SHA_384(state->hmac_sha_384_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_HMAC_SHA_512) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; -__forceinline -uint32_t submit_burst_hmac_sha_512(IMB_MGR *state, - IMB_JOB *jobs, - const uint32_t n_jobs, - const int run_check) -{ - return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, - IMB_AUTH_HMAC_SHA_512, - (void *)state->hmac_sha_512_ooo, - (void *)SUBMIT_JOB_HMAC_SHA_512, - (void *)FLUSH_JOB_HMAC_SHA_512); + job = SUBMIT_JOB_HMAC_SHA_512(state->hmac_sha_512_ooo, + job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = + FLUSH_JOB_HMAC_SHA_512(state->hmac_sha_512_ooo)) + != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } + + return completed_jobs; } __forceinline @@ -491,20 +518,20 @@ uint32_t submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, switch (hash) { case IMB_AUTH_HMAC_SHA_1: - return submit_burst_hmac_sha_1(state, jobs, - n_jobs, run_check); + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_1); case IMB_AUTH_HMAC_SHA_224: - return submit_burst_hmac_sha_224(state, jobs, - n_jobs, run_check); + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_224); case IMB_AUTH_HMAC_SHA_256: - return submit_burst_hmac_sha_256(state, jobs, - n_jobs, run_check); + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_256); case IMB_AUTH_HMAC_SHA_384: - return submit_burst_hmac_sha_384(state, jobs, - n_jobs, run_check); + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_384); case IMB_AUTH_HMAC_SHA_512: - return submit_burst_hmac_sha_512(state, jobs, - n_jobs, run_check); + return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, + IMB_AUTH_HMAC_SHA_512); default: break; } -- GitLab From 6c32943a4f20bdc467331fae0529516e7e3d3164 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 25 Aug 2022 22:02:45 +0000 Subject: [PATCH 335/369] lib: cleanup of obsolete endbranch instructions --- lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm | 1 - lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm | 1 - .../mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm | 1 - lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm | 1 - lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm | 1 - lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm | 1 - lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm | 5 ----- lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm | 5 ----- lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm | 4 ---- lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm | 5 ----- lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm | 6 +----- lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm | 7 +------ lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm | 6 +----- lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm | 6 ------ lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm | 8 -------- lib/avx2_t1/md5_x8x2_avx2.asm | 4 +--- lib/avx2_t2/aes128_ecb_vaes_avx2.asm | 3 --- lib/avx512_t1/des_x16_avx512.asm | 8 +------- lib/avx512_t1/mb_mgr_des_avx512.asm | 14 +------------- lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm | 2 -- lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm | 3 --- lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm | 3 --- lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm | 3 --- lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm | 3 --- lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm | 9 --------- lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm | 3 +-- lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm | 1 - lib/avx512_t2/aes_docsis_dec_avx512.asm | 4 +--- lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm | 4 +--- lib/avx512_t2/aes_docsis_enc_avx512.asm | 6 +----- lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm | 6 +----- .../mb_mgr_aes128_cbcs_1_9_flush_avx512.asm | 3 +-- .../mb_mgr_aes128_cbcs_1_9_submit_avx512.asm | 2 -- ...es128_ccm_auth_submit_flush_x16_vaes_avx512.asm | 3 --- ...gr_aes128_cmac_submit_flush_x16_vaes_avx512.asm | 3 --- ...gr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm | 3 --- ...mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm | 8 +++----- ...mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm | 5 ----- lib/avx512_t2/poly_fma_avx512.asm | 6 ------ lib/avx512_t2/pon_vaes_avx512.asm | 5 ----- lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm | 1 - lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm | 1 - .../mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm | 3 --- .../mb_mgr_aes128_cmac_submit_flush_x4_sse.asm | 3 --- lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm | 1 - lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm | 1 - .../mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 3 --- .../mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm | 3 --- lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm | 9 --------- lib/sse_t1/snow3g_uea2_by4_sse.asm | 1 - 50 files changed, 15 insertions(+), 183 deletions(-) diff --git a/lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm b/lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm index 905d9215..85148570 100644 --- a/lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %define NUM_LANES 8 diff --git a/lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm b/lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm index 4f34da30..a18c5dd3 100644 --- a/lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm +++ b/lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" diff --git a/lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm b/lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm index 7f9e2627..a5d6dac3 100644 --- a/lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm +++ b/lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/memcpy.asm" diff --git a/lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm b/lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm index 3f862e9d..a4e20b97 100644 --- a/lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm +++ b/lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" diff --git a/lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm b/lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm index 28e5452b..57f10267 100644 --- a/lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm +++ b/lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %ifndef AES_XCBC_X8 diff --git a/lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm b/lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm index fe8a3e1d..525d4d24 100644 --- a/lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm +++ b/lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/memcpy.asm" diff --git a/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm index 39a12466..9583367c 100644 --- a/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_md5_flush_avx2.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" ;%define DO_DBGPRINT %include "include/dbgprint.asm" extern md5_x8x2_avx2 @@ -129,7 +128,6 @@ endstruc ; arg 1 : rcx : state MKGLOBAL(flush_job_hmac_md5_avx2,function,internal) flush_job_hmac_md5_avx2: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -161,7 +159,6 @@ flush_job_hmac_md5_avx2: %endrep copy_lane_data: - endbranch64 ; copy good lane (idx) to empty lanes mov tmp, [state + _args_data_ptr_md5 + PTR_SZ*idx] ;; tackle lower 8 lanes @@ -190,7 +187,6 @@ APPEND(upper_skip_,I): align 32 start_loop0: - endbranch64 ; Find min length vphminposuw xmm2, xmm0 vpextrw DWORD(len2), xmm2, 0 ; min value @@ -341,7 +337,6 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA return: - endbranch64 DBGPRINTL "---------- exit md5 flush -----------" vzeroupper diff --git a/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm index 456ba951..9ddc9305 100644 --- a/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_md5_submit_avx2.asm @@ -31,7 +31,6 @@ %include "include/memcpy.asm" %include "include/reg_sizes.asm" %include "include/const.inc" -%include "include/cet.inc" ;%define DO_DBGPRINT %include "include/dbgprint.asm" extern md5_x8x2_avx2 @@ -97,7 +96,6 @@ mksection .text ; arg 2 : rdx : job MKGLOBAL(submit_job_hmac_md5_avx2,function,internal) submit_job_hmac_md5_avx2: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -156,7 +154,6 @@ fast_copy: vmovdqu [lane_data + _extra_block + 0*32], ymm0 vmovdqu [lane_data + _extra_block + 1*32], ymm1 end_fast_copy: - endbranch64 mov size_offset, extra_blocks shl size_offset, 6 sub size_offset, last_len @@ -195,7 +192,6 @@ ge64_bytes: align 16 start_loop: - endbranch64 ; Find min length vmovdqa xmm0, [state + _lens_md5] vphminposuw xmm1, xmm0 @@ -351,7 +347,6 @@ clear_ret: %endif return: - endbranch64 DBGPRINTL "---------- exit md5 submit -----------" vzeroupper diff --git a/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm index f9572763..1dda74d9 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha1_flush_avx2.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" ;%define DO_DBGPRINT %include "include/dbgprint.asm" extern sha1_x8_avx2 @@ -118,7 +117,6 @@ endstruc ; arg 1 : rcx : state MKGLOBAL(flush_job_hmac_avx2,function,internal) flush_job_hmac_avx2: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 ; align stack to 32 byte boundary @@ -143,7 +141,6 @@ flush_job_hmac_avx2: %endrep copy_lane_data: - endbranch64 ; copy valid lane (idx) to empty lanes vmovdqa xmm0, [state + _lens] mov tmp, [state + _args_data_ptr + PTR_SZ*idx] @@ -302,7 +299,6 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA return: - endbranch64 vzeroupper mov rbp, [rsp + _gpr_save + 8*0] mov r12, [rsp + _gpr_save + 8*1] diff --git a/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm index 91eaaf31..d7dbea73 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha1_submit_avx2.asm @@ -33,7 +33,6 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" %include "include/const.inc" -%include "include/cet.inc" extern sha1_x8_avx2 mksection .rodata @@ -100,7 +99,6 @@ endstruc ; arg 2 : rdx : job MKGLOBAL(submit_job_hmac_avx2,function,internal) submit_job_hmac_avx2: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 ; align to 32 byte boundary @@ -154,7 +152,6 @@ fast_copy: vmovdqu [lane_data + _extra_block + 0*32], ymm0 vmovdqu [lane_data + _extra_block + 1*32], ymm1 end_fast_copy: - endbranch64 mov size_offset, extra_blocks shl size_offset, 6 sub size_offset, last_len @@ -197,7 +194,6 @@ ge64_bytes: align 16 start_loop: ; Find min length - endbranch64 vmovdqa xmm0, [state + _lens] vphminposuw xmm1, xmm0 vpextrw DWORD(len2), xmm1, 0 ; min value @@ -348,7 +344,6 @@ clear_ret: %endif return: - endbranch64 vzeroupper DBGPRINTL "---------- exit sha1 submit -----------" mov rbp, [rsp + _gpr_save + 8*0] diff --git a/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm index 2a7e76b1..7b6d3746 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha256_flush_avx2.asm @@ -29,7 +29,7 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" + extern sha256_oct_avx2 mksection .rodata @@ -118,7 +118,6 @@ endstruc ; arg 1 : state MKGLOBAL(FUNC,function,internal) FUNC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -149,7 +148,6 @@ FUNC: %endrep copy_lane_data: - endbranch64 ; copy idx to empty lanes vmovdqa xmm0, [state + _lens_sha256] mov tmp, [state + _args_data_ptr_sha256 + 8*idx] @@ -317,7 +315,6 @@ copy_full_digest: %endif clear_ret: - endbranch64 %ifdef SAFE_DATA vpxor ymm0, ymm0 @@ -360,7 +357,6 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA return: - endbranch64 vzeroupper mov rbx, [rsp + _gpr_save + 8*0] diff --git a/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm index 6fcca91e..30575b35 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha256_submit_avx2.asm @@ -31,7 +31,7 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" + extern sha256_oct_avx2 mksection .rodata @@ -102,7 +102,6 @@ endstruc ; arg 2 : rdx : job MKGLOBAL(FUNC,function,internal) FUNC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -157,7 +156,6 @@ fast_copy: vmovdqu [lane_data + _extra_block + 1*32], ymm1 end_fast_copy: - endbranch64 mov size_offset, extra_blocks shl size_offset, 6 sub size_offset, last_len @@ -202,7 +200,6 @@ ge64_bytes: align 16 start_loop: - endbranch64 ; Find min length vmovdqa xmm0, [state + _lens_sha256] vphminposuw xmm1, xmm0 @@ -376,7 +373,6 @@ copy_full_digest: %endif clear_ret: - endbranch64 %ifdef SAFE_DATA ;; Clear digest (28B/32B), outer_block (28B/32B) and extra_block (64B) of returned job @@ -407,7 +403,6 @@ clear_ret: %endif ;; SAFE_DATA return: - endbranch64 vzeroupper mov rbx, [rsp + _gpr_save + 8*0] diff --git a/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm index a54a382c..1c13d9dc 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha512_flush_avx2.asm @@ -29,7 +29,7 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" + extern sha512_x4_avx2 mksection .rodata @@ -109,7 +109,6 @@ endstruc ; arg 1 : rcx : state MKGLOBAL(FUNC,function,internal) FUNC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -132,7 +131,6 @@ FUNC: %endrep copy_lane_data: - endbranch64 ; copy good lane (idx) to empty lanes vmovdqa xmm0, [state + _lens_sha512] mov tmp, [state + _args_sha512 + _data_ptr_sha512 + PTR_SZ*idx] @@ -295,7 +293,6 @@ copy_full_digest: %endif clear_ret: - endbranch64 %ifdef SAFE_DATA vpxor ymm0, ymm0 @@ -339,7 +336,6 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA return: - endbranch64 vzeroupper mov rbx, [rsp + _gpr_save + 8*0] diff --git a/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm b/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm index 81593198..7b87b763 100644 --- a/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm +++ b/lib/avx2_t1/mb_mgr_hmac_sha512_submit_avx2.asm @@ -31,7 +31,6 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" extern sha512_x4_avx2 mksection .rodata @@ -105,7 +104,6 @@ endstruc ; arg 2 : rdx : job MKGLOBAL(FUNC,function,internal) FUNC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 @@ -159,7 +157,6 @@ fast_copy: vmovdqu [lane_data + _extra_block_sha512 + 2*32], ymm2 vmovdqu [lane_data + _extra_block_sha512 + 3*32], ymm3 end_fast_copy: - endbranch64 mov size_offset, extra_blocks shl size_offset, 7 sub size_offset, last_len @@ -203,7 +200,6 @@ ge128_bytes: align 16 start_loop: ; Find min length - endbranch64 vmovdqa xmm0, [state + _lens_sha512] vphminposuw xmm1, xmm0 vpextrw DWORD(len2), xmm1, 0 ; min value @@ -365,7 +361,6 @@ copy_full_digest: %endif clear_ret: - endbranch64 %ifdef SAFE_DATA ;; Clear digest (48B/64B), outer_block (48B/64B) and extra_block (128B) of returned job %assign J 0 @@ -398,7 +393,6 @@ clear_ret: %endif ;; SAFE_DATA return: - endbranch64 vzeroupper mov rbx, [rsp + _gpr_save + 8*0] diff --git a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm index 1e73cb51..b7f52fc3 100644 --- a/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm +++ b/lib/avx2_t1/mb_mgr_zuc_submit_flush_avx2.asm @@ -717,7 +717,6 @@ APPEND3(%%skip_eea3_copy_,I,J): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EEA3,function,internal) SUBMIT_JOB_ZUC128_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 128 ; JOB* SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -725,7 +724,6 @@ SUBMIT_JOB_ZUC128_EEA3: ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC256_EEA3,function,internal) SUBMIT_JOB_ZUC256_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 256 ; JOB* FLUSH_JOB_ZUC128_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -733,7 +731,6 @@ SUBMIT_JOB_ZUC256_EEA3: ; arg 2 : job MKGLOBAL(FLUSH_JOB_ZUC128_EEA3,function,internal) FLUSH_JOB_ZUC128_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 128 ; JOB* FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -741,7 +738,6 @@ FLUSH_JOB_ZUC128_EEA3: ; arg 2 : job MKGLOBAL(FLUSH_JOB_ZUC256_EEA3,function,internal) FLUSH_JOB_ZUC256_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 256 %macro SUBMIT_JOB_ZUC_EIA3 2 @@ -1113,7 +1109,6 @@ APPEND(%%skip_eia3_,I): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: - endbranch64 SUBMIT_JOB_ZUC_EIA3 128, 4 ret @@ -1124,7 +1119,6 @@ SUBMIT_JOB_ZUC128_EIA3: ; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: - endbranch64 cmp arg3, 8 je submit_tag_8B jb submit_tag_4B @@ -1144,7 +1138,6 @@ submit_tag_4B: ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: - endbranch64 FLUSH_JOB_ZUC_EIA3 128, 4 ret @@ -1154,7 +1147,6 @@ FLUSH_JOB_ZUC128_EIA3: ; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: - endbranch64 cmp arg2, 8 je flush_tag_8B jb flush_tag_4B diff --git a/lib/avx2_t1/md5_x8x2_avx2.asm b/lib/avx2_t1/md5_x8x2_avx2.asm index 53f99570..19b90acb 100644 --- a/lib/avx2_t1/md5_x8x2_avx2.asm +++ b/lib/avx2_t1/md5_x8x2_avx2.asm @@ -40,7 +40,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/transpose_avx2.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + mksection .rodata default rel align 64 @@ -387,7 +387,6 @@ align 32 MKGLOBAL(md5_x8x2_avx2,function,internal) md5_x8x2_avx2: - endbranch64 sub rsp, STACK_size mov DPTR1, rsp @@ -467,7 +466,6 @@ md5_x8x2_avx2: vmovdqu Y_D2,[state + 3 * MD5_DIGEST_ROW_SIZE + 32] lloop: - endbranch64 ; save old digests to stack vmovdqa [Y_AA], Y_A vmovdqa [Y_BB], Y_B diff --git a/lib/avx2_t2/aes128_ecb_vaes_avx2.asm b/lib/avx2_t2/aes128_ecb_vaes_avx2.asm index 09a38fff..e0c8a9de 100644 --- a/lib/avx2_t2/aes128_ecb_vaes_avx2.asm +++ b/lib/avx2_t2/aes128_ecb_vaes_avx2.asm @@ -45,7 +45,6 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/aes_common.asm" -%include "include/cet.inc" %ifdef LINUX %define IN rdi @@ -208,13 +207,11 @@ mksection .text align 16 MKGLOBAL(AES_ECB_ENC,function,internal) AES_ECB_ENC: - endbranch64 AES_ECB ENC ret align 16 MKGLOBAL(AES_ECB_DEC,function,internal) AES_ECB_DEC: - endbranch64 AES_ECB DEC ret diff --git a/lib/avx512_t1/des_x16_avx512.asm b/lib/avx512_t1/des_x16_avx512.asm index e7a223c1..3dedfa2a 100644 --- a/lib/avx512_t1/des_x16_avx512.asm +++ b/lib/avx512_t1/des_x16_avx512.asm @@ -53,7 +53,7 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + %ifdef LINUX %define arg1 rdi %define arg2 rsi @@ -2364,7 +2364,6 @@ mksection .text align 64 MKGLOBAL(des_x16_cbc_enc_avx512,function,internal) des_x16_cbc_enc_avx512: - endbranch64 GENERIC_DES_ENC DES ret @@ -2373,7 +2372,6 @@ des_x16_cbc_enc_avx512: align 64 MKGLOBAL(des_x16_cbc_dec_avx512,function,internal) des_x16_cbc_dec_avx512: - endbranch64 GENERIC_DES_DEC DES ret @@ -2382,7 +2380,6 @@ des_x16_cbc_dec_avx512: align 64 MKGLOBAL(des3_x16_cbc_enc_avx512,function,internal) des3_x16_cbc_enc_avx512: - endbranch64 GENERIC_DES_ENC 3DES ret @@ -2391,7 +2388,6 @@ des3_x16_cbc_enc_avx512: align 64 MKGLOBAL(des3_x16_cbc_dec_avx512,function,internal) des3_x16_cbc_dec_avx512: - endbranch64 GENERIC_DES_DEC 3DES ret @@ -2400,7 +2396,6 @@ des3_x16_cbc_dec_avx512: align 64 MKGLOBAL(docsis_des_x16_enc_avx512,function,internal) docsis_des_x16_enc_avx512: - endbranch64 GENERIC_DES_ENC DOCSIS ret @@ -2409,7 +2404,6 @@ docsis_des_x16_enc_avx512: align 64 MKGLOBAL(docsis_des_x16_dec_avx512,function,internal) docsis_des_x16_dec_avx512: - endbranch64 GENERIC_DES_DEC DOCSIS ret diff --git a/lib/avx512_t1/mb_mgr_des_avx512.asm b/lib/avx512_t1/mb_mgr_des_avx512.asm index 2e470304..9a36428f 100644 --- a/lib/avx512_t1/mb_mgr_des_avx512.asm +++ b/lib/avx512_t1/mb_mgr_des_avx512.asm @@ -49,7 +49,7 @@ ;%define DO_DBGPRINT %include "include/dbgprint.asm" %include "include/const.inc" -%include "include/cet.inc" + extern docsis_des_x16_enc_avx512 extern docsis_des_x16_dec_avx512 extern des_x16_cbc_enc_avx512 @@ -436,7 +436,6 @@ mksection .text align 64 MKGLOBAL(submit_job_des_cbc_enc_avx512,function,internal) submit_job_des_cbc_enc_avx512: - endbranch64 GENERIC_DES_SUBMIT DES, ENC ret @@ -445,7 +444,6 @@ submit_job_des_cbc_enc_avx512: align 64 MKGLOBAL(submit_job_des_cbc_dec_avx512,function,internal) submit_job_des_cbc_dec_avx512: - endbranch64 GENERIC_DES_SUBMIT DES, DEC ret @@ -454,7 +452,6 @@ submit_job_des_cbc_dec_avx512: align 64 MKGLOBAL(submit_job_docsis_des_enc_avx512,function,internal) submit_job_docsis_des_enc_avx512: - endbranch64 GENERIC_DES_SUBMIT DOCSIS, ENC ret @@ -463,7 +460,6 @@ submit_job_docsis_des_enc_avx512: align 64 MKGLOBAL(submit_job_docsis_des_dec_avx512,function,internal) submit_job_docsis_des_dec_avx512: - endbranch64 GENERIC_DES_SUBMIT DOCSIS, DEC ret @@ -472,7 +468,6 @@ submit_job_docsis_des_dec_avx512: align 64 MKGLOBAL(submit_job_3des_cbc_enc_avx512,function,internal) submit_job_3des_cbc_enc_avx512: - endbranch64 GENERIC_DES_SUBMIT 3DES, ENC ret @@ -481,7 +476,6 @@ submit_job_3des_cbc_enc_avx512: align 64 MKGLOBAL(submit_job_3des_cbc_dec_avx512,function,internal) submit_job_3des_cbc_dec_avx512: - endbranch64 GENERIC_DES_SUBMIT 3DES, DEC ret @@ -489,7 +483,6 @@ submit_job_3des_cbc_dec_avx512: align 64 MKGLOBAL(flush_job_des_cbc_enc_avx512,function,internal) flush_job_des_cbc_enc_avx512: - endbranch64 GENERIC_DES_FLUSH DES, ENC ret @@ -497,7 +490,6 @@ flush_job_des_cbc_enc_avx512: align 64 MKGLOBAL(flush_job_des_cbc_dec_avx512,function,internal) flush_job_des_cbc_dec_avx512: - endbranch64 GENERIC_DES_FLUSH DES, DEC ret @@ -505,7 +497,6 @@ flush_job_des_cbc_dec_avx512: align 64 MKGLOBAL(flush_job_docsis_des_enc_avx512,function,internal) flush_job_docsis_des_enc_avx512: - endbranch64 GENERIC_DES_FLUSH DOCSIS, ENC ret @@ -513,7 +504,6 @@ flush_job_docsis_des_enc_avx512: align 64 MKGLOBAL(flush_job_docsis_des_dec_avx512,function,internal) flush_job_docsis_des_dec_avx512: - endbranch64 GENERIC_DES_FLUSH DOCSIS, DEC ret @@ -521,7 +511,6 @@ flush_job_docsis_des_dec_avx512: align 64 MKGLOBAL(flush_job_3des_cbc_enc_avx512,function,internal) flush_job_3des_cbc_enc_avx512: - endbranch64 GENERIC_DES_FLUSH 3DES, ENC ret @@ -529,7 +518,6 @@ flush_job_3des_cbc_enc_avx512: align 64 MKGLOBAL(flush_job_3des_cbc_dec_avx512,function,internal) flush_job_3des_cbc_dec_avx512: - endbranch64 GENERIC_DES_FLUSH 3DES, DEC ret diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm index fb2ca565..5cf8ff3b 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm @@ -46,7 +46,6 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" %include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -123,7 +122,6 @@ endstruc ; arg 2 : rdx : job MKGLOBAL(submit_job_hmac_avx512,function,internal) submit_job_hmac_avx512: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -32 ; align to 32 byte boundary diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm index 3a039541..e449e94f 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_flush_avx512.asm @@ -44,7 +44,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -148,11 +147,9 @@ align 32 %ifdef SHA224 MKGLOBAL(flush_job_hmac_sha_224_avx512,function,internal) flush_job_hmac_sha_224_avx512: - endbranch64 %else MKGLOBAL(flush_job_hmac_sha_256_avx512,function,internal) flush_job_hmac_sha_256_avx512: - endbranch64 %endif mov rax, rsp sub rsp, STACK_size diff --git a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm index 0ebbb582..0660b19d 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha256_submit_avx512.asm @@ -46,7 +46,6 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" %include "include/clear_regs.asm" ;; %define DO_DBGPRINT %include "include/dbgprint.asm" @@ -120,11 +119,9 @@ endstruc %ifdef SHA224 MKGLOBAL(submit_job_hmac_sha_224_avx512,function,internal) submit_job_hmac_sha_224_avx512: - endbranch64 %else MKGLOBAL(submit_job_hmac_sha_256_avx512,function,internal) submit_job_hmac_sha_256_avx512: - endbranch64 %endif mov rax, rsp sub rsp, STACK_size diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm index b5101c80..9b825d68 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_flush_avx512.asm @@ -36,7 +36,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %include "include/clear_regs.asm" extern sha512_x8_avx512 @@ -127,7 +126,6 @@ endstruc MKGLOBAL(flush_job_hmac_sha_512_avx512,function,internal) align 64 flush_job_hmac_sha_512_avx512: - endbranch64 %else ; JOB* flush_job_hmac_sha_512_avx512(MB_MGR_HMAC_SHA_512_OOO *state) ; arg 1 : state @@ -135,7 +133,6 @@ flush_job_hmac_sha_512_avx512: MKGLOBAL(flush_job_hmac_sha_384_avx512,function,internal) align 64 flush_job_hmac_sha_384_avx512: - endbranch64 %endif mov rax, rsp sub rsp, STACK_size diff --git a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm index 1166f08c..504815cc 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha512_submit_avx512.asm @@ -38,7 +38,6 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" %include "include/clear_regs.asm" %use smartalign @@ -116,12 +115,10 @@ align 64 MKGLOBAL(submit_job_hmac_sha_512_avx512,function,internal) %define SHA_X_DIGEST_SIZE 512 submit_job_hmac_sha_512_avx512: - endbranch64 %else MKGLOBAL(submit_job_hmac_sha_384_avx512,function,internal) %define SHA_X_DIGEST_SIZE 384 submit_job_hmac_sha_384_avx512: - endbranch64 %endif mov rax, rsp diff --git a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm index 98316192..be795516 100644 --- a/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_zuc_submit_flush_avx512.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/constants.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" @@ -580,7 +579,6 @@ mksection .text ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EEA3,function,internal) SUBMIT_JOB_ZUC128_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 128 ret @@ -589,7 +587,6 @@ SUBMIT_JOB_ZUC128_EEA3: ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC256_EEA3,function,internal) SUBMIT_JOB_ZUC256_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 256 ret @@ -597,7 +594,6 @@ SUBMIT_JOB_ZUC256_EEA3: ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EEA3,function,internal) FLUSH_JOB_ZUC128_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 128 ret @@ -605,7 +601,6 @@ FLUSH_JOB_ZUC128_EEA3: ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC256_EEA3,function,internal) FLUSH_JOB_ZUC256_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 256 ret @@ -1220,7 +1215,6 @@ FLUSH_JOB_ZUC256_EEA3: ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: - endbranch64 SUBMIT_JOB_ZUC_EIA3 128, 4 ret @@ -1231,7 +1225,6 @@ SUBMIT_JOB_ZUC128_EIA3: ; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: - endbranch64 cmp arg3, 8 je submit_tag_8B jb submit_tag_4B @@ -1252,7 +1245,6 @@ submit_tag_4B: ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: - endbranch64 FLUSH_JOB_ZUC_EIA3 128, 4 ret @@ -1262,7 +1254,6 @@ FLUSH_JOB_ZUC128_EIA3: ; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: - endbranch64 cmp arg2, 8 je flush_tag_8B jb flush_tag_4B diff --git a/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm index c0a2fe35..2f5adea4 100644 --- a/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_dec_by16_vaes_avx512.asm @@ -27,7 +27,7 @@ %define CBCS %include "avx512_t2/aes_cbc_dec_by16_vaes_avx512.asm" -%include "include/cet.inc" + %define len rax mksection .text @@ -37,7 +37,6 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(aes_cbcs_1_9_dec_128_vaes_avx512,function,internal) aes_cbcs_1_9_dec_128_vaes_avx512: - endbranch64 %ifndef LINUX mov len, [rsp + 8*5] %else diff --git a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm index 5fe71208..83a02d69 100644 --- a/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_cbcs_enc_vaes_avx512.asm @@ -31,7 +31,6 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %define GPR_SAVE_AREA rsp + _gpr_save diff --git a/lib/avx512_t2/aes_docsis_dec_avx512.asm b/lib/avx512_t2/aes_docsis_dec_avx512.asm index 0afea8d3..f1dab413 100644 --- a/lib/avx512_t2/aes_docsis_dec_avx512.asm +++ b/lib/avx512_t2/aes_docsis_dec_avx512.asm @@ -31,7 +31,7 @@ %include "include/os.asm" %include "include/clear_regs.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" + ;; In System V AMD64 ABI ;; callee saves: RBX, RBP, R12-R15 ;; Windows x64 ABI @@ -1063,7 +1063,6 @@ mksection .text align 64 MKGLOBAL(aes_docsis128_dec_crc32_avx512,function,internal) aes_docsis128_dec_crc32_avx512: - endbranch64 AES_DOCSIS_DEC_CRC32 9 ret @@ -1071,7 +1070,6 @@ aes_docsis128_dec_crc32_avx512: align 64 MKGLOBAL(aes_docsis256_dec_crc32_avx512,function,internal) aes_docsis256_dec_crc32_avx512: - endbranch64 AES_DOCSIS_DEC_CRC32 13 ret diff --git a/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm index b8fd8ffe..c03684ae 100644 --- a/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm +++ b/lib/avx512_t2/aes_docsis_dec_vaes_avx512.asm @@ -32,7 +32,7 @@ %include "include/clear_regs.asm" %include "include/aes_common.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" + default rel extern ethernet_fcs_avx512_local @@ -1510,7 +1510,6 @@ mksection .text align 64 MKGLOBAL(aes_docsis128_dec_crc32_vaes_avx512,function,internal) aes_docsis128_dec_crc32_vaes_avx512: - endbranch64 AES_DOCSIS_DEC_CRC32 9 ret @@ -1518,7 +1517,6 @@ aes_docsis128_dec_crc32_vaes_avx512: align 64 MKGLOBAL(aes_docsis256_dec_crc32_vaes_avx512,function,internal) aes_docsis256_dec_crc32_vaes_avx512: - endbranch64 AES_DOCSIS_DEC_CRC32 13 ret diff --git a/lib/avx512_t2/aes_docsis_enc_avx512.asm b/lib/avx512_t2/aes_docsis_enc_avx512.asm index 6ab393db..8a6a4e09 100644 --- a/lib/avx512_t2/aes_docsis_enc_avx512.asm +++ b/lib/avx512_t2/aes_docsis_enc_avx512.asm @@ -36,7 +36,7 @@ %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" -%include "include/cet.inc" + %define APPEND(a,b) a %+ b %define CRC_LANE_STATE_TO_START 0x01 @@ -1389,7 +1389,6 @@ APPEND(%%_skip_clear_,I): align 64 MKGLOBAL(submit_job_aes_docsis128_enc_crc32_avx512,function,internal) submit_job_aes_docsis128_enc_crc32_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -1411,7 +1410,6 @@ submit_job_aes_docsis128_enc_crc32_avx512: align 64 MKGLOBAL(submit_job_aes_docsis256_enc_crc32_avx512,function,internal) submit_job_aes_docsis256_enc_crc32_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -1431,7 +1429,6 @@ submit_job_aes_docsis256_enc_crc32_avx512: align 64 MKGLOBAL(flush_job_aes_docsis128_enc_crc32_avx512,function,internal) flush_job_aes_docsis128_enc_crc32_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -1451,7 +1448,6 @@ flush_job_aes_docsis128_enc_crc32_avx512: align 64 MKGLOBAL(flush_job_aes_docsis256_enc_crc32_avx512,function,internal) flush_job_aes_docsis256_enc_crc32_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ diff --git a/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm b/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm index 1e3e8ec3..fa7adb88 100644 --- a/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm +++ b/lib/avx512_t2/aes_docsis_enc_vaes_avx512.asm @@ -35,7 +35,7 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" + %define APPEND(a,b) a %+ b %define CRC_LANE_STATE_TO_START 0x01 @@ -1987,7 +1987,6 @@ align 32 align 64 MKGLOBAL(submit_job_aes_docsis128_enc_crc32_vaes_avx512,function,internal) submit_job_aes_docsis128_enc_crc32_vaes_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -2009,7 +2008,6 @@ submit_job_aes_docsis128_enc_crc32_vaes_avx512: align 64 MKGLOBAL(submit_job_aes_docsis256_enc_crc32_vaes_avx512,function,internal) submit_job_aes_docsis256_enc_crc32_vaes_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -2029,7 +2027,6 @@ submit_job_aes_docsis256_enc_crc32_vaes_avx512: align 64 MKGLOBAL(flush_job_aes_docsis128_enc_crc32_vaes_avx512,function,internal) flush_job_aes_docsis128_enc_crc32_vaes_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ @@ -2049,7 +2046,6 @@ flush_job_aes_docsis128_enc_crc32_vaes_avx512: align 64 MKGLOBAL(flush_job_aes_docsis256_enc_crc32_vaes_avx512,function,internal) flush_job_aes_docsis256_enc_crc32_vaes_avx512: - endbranch64 FUNC_ENTRY SUBMIT_FLUSH_DOCSIS_CRC32 arg1, arg2, \ diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm index de6963ee..f7bb94f6 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_flush_avx512.asm @@ -27,7 +27,7 @@ %define CBCS %include "avx512_t2/mb_mgr_aes128_cbc_enc_flush_avx512.asm" -%include "include/cet.inc" + %define AES_CBCS_ENC_X16 aes_cbcs_1_9_enc_128_vaes_avx512 %define FLUSH_JOB_AES_CBCS_ENC flush_job_aes128_cbcs_1_9_enc_vaes_avx512 @@ -39,7 +39,6 @@ extern AES_CBCS_ENC_X16 ; arg 2 : job MKGLOBAL(FLUSH_JOB_AES_CBCS_ENC,function,internal) FLUSH_JOB_AES_CBCS_ENC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm index c34d977c..6497f60f 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cbcs_1_9_submit_avx512.asm @@ -27,7 +27,6 @@ %define CBCS %include "avx512_t2/mb_mgr_aes128_cbc_enc_submit_avx512.asm" -%include "include/cet.inc" %define AES_CBCS_ENC_X16 aes_cbcs_1_9_enc_128_vaes_avx512 %define NUM_KEYS 11 @@ -41,7 +40,6 @@ extern AES_CBCS_ENC_X16 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_CBCS_ENC,function,internal) SUBMIT_JOB_AES_CBCS_ENC: - endbranch64 mov rax, rsp sub rsp, STACK_size and rsp, -16 diff --git a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm index 52f2efa0..9e94ab39 100644 --- a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/memcpy.asm" @@ -698,14 +697,12 @@ align 64 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_CCM_AUTH,function,internal) SUBMIT_JOB_AES_CCM_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CCM_AUTH_AVX SUBMIT ; IMB_JOB * flush_job_aes128/256_ccm_auth_vaes_avx512(MB_MGR_CCM_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_AES_CCM_AUTH,function,internal) FLUSH_JOB_AES_CCM_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CCM_AUTH_AVX FLUSH mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm index abbd81ed..9a1acc8a 100644 --- a/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_cmac_submit_flush_x16_vaes_avx512.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" @@ -637,14 +636,12 @@ align 64 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_CMAC_AUTH,function,internal) SUBMIT_JOB_AES_CMAC_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_VAES_AVX512 SUBMIT ; IMB_JOB * flush_job_aes_cmac_auth_vaes_avx512(MB_MGR_CMAC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_AES_CMAC_AUTH,function,internal) FLUSH_JOB_AES_CMAC_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_VAES_AVX512 FLUSH mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm index 8703865f..ee900051 100644 --- a/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_xcbc_submit_flush_x16_vaes_avx512.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" @@ -451,14 +450,12 @@ align 64 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_XCBC,function,internal) SUBMIT_JOB_AES_XCBC: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_XCBC_VAES_AVX512 SUBMIT ; IMB_JOB * flush_job_aes_xcbc_vaes_avx512(MB_MGR_XCBC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_AES_XCBC,function,internal) FLUSH_JOB_AES_XCBC: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_XCBC_VAES_AVX512 FLUSH mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm index 7a0da197..380446c5 100644 --- a/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uea2_submit_flush_vaes_avx512.asm @@ -32,7 +32,6 @@ %include "include/os.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %include "avx512_t2/snow3g_uea2_by16_vaes_avx512.asm" %ifndef SUBMIT_JOB_SNOW3G_UEA2 @@ -380,14 +379,13 @@ mksection .text ;; arg 2 : job MKGLOBAL(SUBMIT_JOB_SNOW3G_UEA2_GEN2,function,internal) SUBMIT_JOB_SNOW3G_UEA2_GEN2: - endbranch64 SNOW3G_FUNC_START SUBMIT_FLUSH_JOB_SNOW3G_UEA2 submit, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, tmp_gp9, tmp_gp10, tmp_gp11, avx512_gen2 SNOW3G_FUNC_END ret + MKGLOBAL(SUBMIT_JOB_SNOW3G_UEA2,function,internal) SUBMIT_JOB_SNOW3G_UEA2: - endbranch64 SNOW3G_FUNC_START SUBMIT_FLUSH_JOB_SNOW3G_UEA2 submit, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, tmp_gp9, tmp_gp10, tmp_gp11, avx512_gen1 SNOW3G_FUNC_END @@ -397,16 +395,16 @@ SUBMIT_JOB_SNOW3G_UEA2: ;; arg 1 : state MKGLOBAL(FLUSH_JOB_SNOW3G_UEA2_GEN2,function,internal) FLUSH_JOB_SNOW3G_UEA2_GEN2: - endbranch64 SNOW3G_FUNC_START SUBMIT_FLUSH_JOB_SNOW3G_UEA2 flush, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, tmp_gp9, tmp_gp10, tmp_gp11, avx512_gen2 SNOW3G_FUNC_END ret + MKGLOBAL(FLUSH_JOB_SNOW3G_UEA2,function,internal) FLUSH_JOB_SNOW3G_UEA2: - endbranch64 SNOW3G_FUNC_START SUBMIT_FLUSH_JOB_SNOW3G_UEA2 flush, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, tmp_gp9, tmp_gp10, tmp_gp11, avx512_gen1 SNOW3G_FUNC_END ret + mksection stack-noexec diff --git a/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm index 9b3264c9..c10985b1 100644 --- a/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_snow3g_uia2_submit_flush_vaes_avx512.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/constants.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/clear_regs.asm" @@ -269,24 +268,20 @@ mksection .text ; arg 2 : job MKGLOBAL(SUBMIT_JOB_SNOW3G_UIA2_GEN2,function,internal) SUBMIT_JOB_SNOW3G_UIA2_GEN2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 submit, avx512_gen2 MKGLOBAL(SUBMIT_JOB_SNOW3G_UIA2,function,internal) SUBMIT_JOB_SNOW3G_UIA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 submit, avx512_gen1 ; JOB* FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_SNOW3G_UIA2_GEN2,function,internal) FLUSH_JOB_SNOW3G_UIA2_GEN2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 flush, avx512_gen2 MKGLOBAL(FLUSH_JOB_SNOW3G_UIA2,function,internal) FLUSH_JOB_SNOW3G_UIA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 flush, avx512_gen1 mksection stack-noexec diff --git a/lib/avx512_t2/poly_fma_avx512.asm b/lib/avx512_t2/poly_fma_avx512.asm index f0df908e..0eec89f3 100644 --- a/lib/avx512_t2/poly_fma_avx512.asm +++ b/lib/avx512_t2/poly_fma_avx512.asm @@ -1113,7 +1113,6 @@ mksection .text and %%T0, 0xffffffffffffff00 ; multiple of 256 bytes %%_poly1305_blocks_loop: - endbranch64 cmp %%T0, POLY1305_BLOCK_SIZE*16 jbe %%_poly1305_blocks_loop_end @@ -1211,8 +1210,6 @@ mksection .text and %%LEN, (POLY1305_BLOCK_SIZE*16 - 1) ; Get remaining lengths (LEN < 256 bytes) %%_less_than_256: - endbranch64 - cmp %%LEN, POLY1305_BLOCK_SIZE*8 jb %%_less_than_128 @@ -1378,8 +1375,6 @@ APPEND(%%_shuffle_blocks_, i): %endrep %%_end_shuffle: - endbranch64 - ; zmm13-zmm15 contain the 8 blocks of message plus the previous accumulator ; zmm22-24 contain the 3x44-bit limbs of the powers of R ; zmm25-26 contain the 3x44-bit limbs of the powers of R' (5*4*R) @@ -1812,7 +1807,6 @@ poly1305_aead_complete_fma_avx512: align 32 MKGLOBAL(poly1305_mac_fma_avx512,function,internal) poly1305_mac_fma_avx512: - endbranch64 FUNC_ENTRY %ifndef LINUX diff --git a/lib/avx512_t2/pon_vaes_avx512.asm b/lib/avx512_t2/pon_vaes_avx512.asm index 91babf85..1ab0de6c 100644 --- a/lib/avx512_t2/pon_vaes_avx512.asm +++ b/lib/avx512_t2/pon_vaes_avx512.asm @@ -31,7 +31,6 @@ %include "include/os.asm" %include "include/memcpy.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" extern aes_cntr_pon_enc_128_vaes_avx512 extern aes_cntr_pon_dec_128_vaes_avx512 @@ -505,7 +504,6 @@ endstruc align 64 MKGLOBAL(submit_job_pon_enc_vaes_avx512,function,internal) submit_job_pon_enc_vaes_avx512: - endbranch64 AES128_CTR_PON_ENC CTR ret @@ -513,7 +511,6 @@ submit_job_pon_enc_vaes_avx512: align 64 MKGLOBAL(submit_job_pon_dec_vaes_avx512,function,internal) submit_job_pon_dec_vaes_avx512: - endbranch64 AES128_CTR_PON_DEC CTR ret @@ -521,7 +518,6 @@ submit_job_pon_dec_vaes_avx512: align 64 MKGLOBAL(submit_job_pon_enc_no_ctr_vaes_avx512,function,internal) submit_job_pon_enc_no_ctr_vaes_avx512: - endbranch64 AES128_CTR_PON_ENC NO_CTR ret @@ -529,7 +525,6 @@ submit_job_pon_enc_no_ctr_vaes_avx512: align 64 MKGLOBAL(submit_job_pon_dec_no_ctr_vaes_avx512,function,internal) submit_job_pon_dec_no_ctr_vaes_avx512: - endbranch64 AES128_CTR_PON_DEC NO_CTR ret diff --git a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm index 60b96f39..54ec68b4 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_flush_sse.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %define NUM_LANES 4 diff --git a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm index 79bc51c6..396fb202 100644 --- a/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cbcs_1_9_submit_sse.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" diff --git a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm index 024e045a..bfff5c2b 100644 --- a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/const.inc" %include "include/memcpy.asm" @@ -588,14 +587,12 @@ align 64 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_CCM_AUTH,function,internal) SUBMIT_JOB_AES_CCM_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CCM_AUTH_SSE SUBMIT ; IMB_JOB * flush_job_aes_ccm_auth_sse(MB_MGR_CCM_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_AES_CCM_AUTH,function,internal) FLUSH_JOB_AES_CCM_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CCM_AUTH_SSE FLUSH mksection stack-noexec diff --git a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm index 8aecdc2d..358c67c1 100644 --- a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm @@ -32,7 +32,6 @@ %include "include/reg_sizes.asm" %include "include/memcpy.asm" %include "include/const.inc" -%include "include/cet.inc" ;%define DO_DBGPRINT %include "include/dbgprint.asm" @@ -552,14 +551,12 @@ align 64 ; arg 2 : job MKGLOBAL(SUBMIT_JOB_AES_CMAC_AUTH,function,internal) SUBMIT_JOB_AES_CMAC_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE SUBMIT ; IMB_JOB * flush_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_AES_CMAC_AUTH,function,internal) FLUSH_JOB_AES_CMAC_AUTH: - endbranch64 GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE FLUSH mksection stack-noexec diff --git a/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm index 12359383..8d765e99 100644 --- a/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_xcbc_flush_x4_sse.asm @@ -28,7 +28,6 @@ %include "include/os.asm" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %ifndef AES_XCBC_X4 diff --git a/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm index c6409d06..73afe3e5 100644 --- a/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_xcbc_submit_x4_sse.asm @@ -29,7 +29,6 @@ %include "include/const.inc" %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" -%include "include/cet.inc" %include "include/reg_sizes.asm" %include "include/memcpy.asm" %ifndef AES_XCBC_X4 diff --git a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index a55a9fe9..ffac453d 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %include "sse_t1/snow3g_uea2_by4_sse.asm" %define SUBMIT_JOB_SNOW3G_UEA2 submit_job_snow3g_uea2_sse @@ -409,7 +408,6 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; MKGLOBAL(SUBMIT_JOB_SNOW3G_UEA2,function,internal) SUBMIT_JOB_SNOW3G_UEA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UEA2_SSE submit, tmp_gp1, tmp_gp2, tmp_gp3, \ tmp_gp4, tmp_gp5, tmp_gp6, tmp_gp7, \ tmp_gp8, xmm0, xmm1, xmm2, xmm3, xmm4, \ @@ -420,7 +418,6 @@ SUBMIT_JOB_SNOW3G_UEA2: MKGLOBAL(FLUSH_JOB_SNOW3G_UEA2,function,internal) FLUSH_JOB_SNOW3G_UEA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UEA2_SSE flush, tmp_gp1, tmp_gp2, tmp_gp3, tmp_gp4,\ tmp_gp5, tmp_gp6, tmp_gp7, tmp_gp8, xmm0, \ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, \ diff --git a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm index cc69e15a..e501a8dd 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm @@ -29,7 +29,6 @@ %include "include/imb_job.asm" %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" -%include "include/cet.inc" %include "include/clear_regs.asm" %include "sse_t1/snow3g_uea2_by4_sse.asm" @@ -283,7 +282,6 @@ APPEND(skip_lane_copy_,i): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_SNOW3G_UIA2,function,internal) SUBMIT_JOB_SNOW3G_UIA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 submit, tmp_gp0, tmp_gp1, \ tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ @@ -295,7 +293,6 @@ SUBMIT_JOB_SNOW3G_UIA2: ; arg 1 : state MKGLOBAL(FLUSH_JOB_SNOW3G_UIA2,function,internal) FLUSH_JOB_SNOW3G_UIA2: - endbranch64 SUBMIT_FLUSH_JOB_SNOW3G_UIA2 flush, tmp_gp0, tmp_gp1, \ tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ diff --git a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm index d348de2e..efc6f9a7 100644 --- a/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm +++ b/lib/sse_t1/mb_mgr_zuc_submit_flush_sse.asm @@ -30,7 +30,6 @@ %include "include/mb_mgr_datastruct.asm" %include "include/reg_sizes.asm" %include "include/const.inc" -%include "include/cet.inc" %include "include/clear_regs.asm" %ifndef SUBMIT_JOB_ZUC128_EEA3 @@ -683,7 +682,6 @@ APPEND3(%%skip_eea3_copy_,I,J): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EEA3,function,internal) SUBMIT_JOB_ZUC128_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 128 ; JOB* SUBMIT_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state, IMB_JOB *job) @@ -691,21 +689,18 @@ SUBMIT_JOB_ZUC128_EEA3: ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC256_EEA3,function,internal) SUBMIT_JOB_ZUC256_EEA3: - endbranch64 SUBMIT_JOB_ZUC_EEA3 256 ; JOB* FLUSH_JOB_ZUC128_EEA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EEA3,function,internal) FLUSH_JOB_ZUC128_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 128 ; JOB* FLUSH_JOB_ZUC256_EEA3(MB_MGR_ZUC_OOO *state) ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC256_EEA3,function,internal) FLUSH_JOB_ZUC256_EEA3: - endbranch64 FLUSH_JOB_ZUC_EEA3 256 %macro SUBMIT_JOB_ZUC_EIA3 2 @@ -1065,7 +1060,6 @@ APPEND(%%skip_eia3_,I): ; arg 2 : job MKGLOBAL(SUBMIT_JOB_ZUC128_EIA3,function,internal) SUBMIT_JOB_ZUC128_EIA3: - endbranch64 SUBMIT_JOB_ZUC_EIA3 128, 4 ret @@ -1076,7 +1070,6 @@ SUBMIT_JOB_ZUC128_EIA3: ; arg 3 : tag size (4, 8 or 16 bytes) MKGLOBAL(SUBMIT_JOB_ZUC256_EIA3,function,internal) SUBMIT_JOB_ZUC256_EIA3: - endbranch64 cmp arg3, 8 je submit_tag_8B jb submit_tag_4B @@ -1097,7 +1090,6 @@ submit_tag_4B: ; arg 1 : state MKGLOBAL(FLUSH_JOB_ZUC128_EIA3,function,internal) FLUSH_JOB_ZUC128_EIA3: - endbranch64 FLUSH_JOB_ZUC_EIA3 128, 4 ret @@ -1107,7 +1099,6 @@ FLUSH_JOB_ZUC128_EIA3: ; arg 2 : tag size (4, 8 or 16 bytes) MKGLOBAL(FLUSH_JOB_ZUC256_EIA3,function,internal) FLUSH_JOB_ZUC256_EIA3: - endbranch64 cmp arg2, 8 je flush_tag_8B jb flush_tag_4B diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index 9fcf8b14..7e37d652 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -30,7 +30,6 @@ %include "include/memcpy.asm" %include "include/imb_job.asm" %include "include/clear_regs.asm" -%include "include/cet.inc" %include "include/mb_mgr_datastruct.asm" %include "include/memcpy.asm" %include "include/transpose_sse.asm" -- GitLab From 28c5f788237afb9886ee7d05f607d3edff7f611f Mon Sep 17 00:00:00 2001 From: Kamila Lipinska Date: Wed, 17 Aug 2022 12:23:34 +0100 Subject: [PATCH 336/369] sse_t1: [SNOW3G-UEA2]: Add code for SAFE_LOOKUP option disabled --- ...mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm | 4 +- ...mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm | 58 ++-- lib/sse_t1/snow3g_uea2_by4_sse.asm | 275 ++++++++++++++---- 3 files changed, 251 insertions(+), 86 deletions(-) diff --git a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm index ffac453d..d06c403f 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uea2_submit_flush_x4_sse.asm @@ -255,7 +255,7 @@ mksection .text ;; Create mask with lanes in use pxor %%TMP_XMM_2, %%TMP_XMM_2 pxor %%TMP_XMM_3, %%TMP_XMM_3 - pcmpeqq %%TMP_XMM_2, [state + _snow3g_job_in_lane] + pcmpeqq %%TMP_XMM_2, [state + _snow3g_job_in_lane] pcmpeqq %%TMP_XMM_3, [state + _snow3g_job_in_lane + 16] pshufd %%TMP_XMM_2, %%TMP_XMM_2, 0x88 ;; lane order: 1,0,1,0 pshufd %%TMP_XMM_3, %%TMP_XMM_3, 0x88 ;; lane order: 3,2,3,2 @@ -292,7 +292,7 @@ mksection .text movdqa %%TMP_XMM_4, [state+_snow3g_args_byte_length] psubd %%TMP_XMM_4, %%TMP_XMM_0 movdqa [state+_snow3g_args_byte_length], %%TMP_XMM_4 - + ;; Do cipher / clock operation for all lanes and given common length SNOW3G_ENC_DEC state, %%TGP0, %%TGP1, %%TGP2, %%TGP3, %%TGP4, %%TGP5, \ %%TMP_XMM_0, %%TMP_XMM_1, %%TMP_XMM_2, %%TMP_XMM_3, \ diff --git a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm index e501a8dd..4490a748 100644 --- a/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_snow3g_uia2_submit_flush_x4_sse.asm @@ -59,9 +59,11 @@ mksection .text %define tmp_gp1 rbp %define tmp_gp2 r9 %define tmp_gp3 r10 -%define tmp_gp4 r13 %define init_lanes r11 %define tmp_state r12 +%define tmp_gp4 r13 +%define tmp_gp5 r14 +%define tmp_gp6 r15 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Get lane nr from ptr to the list of unused lanes. @@ -91,29 +93,31 @@ mksection .text mov [%%JOB_LANES + %%LANE_NR*8], %%JOB %endmacro -%macro SUBMIT_FLUSH_JOB_SNOW3G_UIA2 22 +%macro SUBMIT_FLUSH_JOB_SNOW3G_UIA2 24 %define %%SUBMIT_FLUSH %1 ;; [in] submit/flush selector %define %%UNUSED_LANES %2 ;; [clobbered] GP register %define %%LANE %3 ;; [clobbered] GP register %define %%TGP0 %4 ;; [clobbered] GP register %define %%TGP1 %5 ;; [clobbered] GP register %define %%TGP2 %6 ;; [clobbered] GP register -%define %%TMP_XMM_0 %7 ;; [clobbered] xmm register -%define %%TMP_XMM_1 %8 ;; [clobbered] xmm register -%define %%TMP_XMM_2 %9 ;; [clobbered] xmm register -%define %%TMP_XMM_3 %10 ;; [clobbered] xmm register -%define %%TMP_XMM_4 %11 ;; [clobbered] xmm register -%define %%TMP_XMM_5 %12 ;; [clobbered] xmm register -%define %%TMP_XMM_6 %13 ;; [clobbered] xmm register -%define %%TMP_XMM_7 %14 ;; [clobbered] xmm register -%define %%TMP_XMM_8 %15 ;; [clobbered] xmm register -%define %%TMP_XMM_9 %16 ;; [clobbered] xmm register -%define %%TMP_XMM_10 %17 ;; [clobbered] xmm register -%define %%TMP_XMM_11 %18 ;; [clobbered] xmm register -%define %%TMP_XMM_12 %19 ;; [clobbered] xmm register -%define %%TMP_XMM_13 %20 ;; [clobbered] xmm register -%define %%TMP_XMM_14 %21 ;; [clobbered] xmm register -%define %%TMP_XMM_15 %22 ;; [clobbered] xmm register +%define %%TGP3 %7 ;; [clobbered] GP register +%define %%TGP4 %8 ;; [clobbered] GP register +%define %%TMP_XMM_0 %9 ;; [clobbered] xmm register +%define %%TMP_XMM_1 %10 ;; [clobbered] xmm register +%define %%TMP_XMM_2 %11 ;; [clobbered] xmm register +%define %%TMP_XMM_3 %12 ;; [clobbered] xmm register +%define %%TMP_XMM_4 %13 ;; [clobbered] xmm register +%define %%TMP_XMM_5 %14 ;; [clobbered] xmm register +%define %%TMP_XMM_6 %15 ;; [clobbered] xmm register +%define %%TMP_XMM_7 %16 ;; [clobbered] xmm register +%define %%TMP_XMM_8 %17 ;; [clobbered] xmm register +%define %%TMP_XMM_9 %18 ;; [clobbered] xmm register +%define %%TMP_XMM_10 %19 ;; [clobbered] xmm register +%define %%TMP_XMM_11 %20 ;; [clobbered] xmm register +%define %%TMP_XMM_12 %21 ;; [clobbered] xmm register +%define %%TMP_XMM_13 %22 ;; [clobbered] xmm register +%define %%TMP_XMM_14 %23 ;; [clobbered] xmm register +%define %%TMP_XMM_15 %24 ;; [clobbered] xmm register SNOW3G_FUNC_START xor job_rax, job_rax ;; assume NULL return job @@ -254,7 +258,7 @@ APPEND(skip_lane_copy_,i): SNOW3G_AUTH_INIT_5_BY_4 {state + _snow3g_args_keys}, \ {state + _snow3g_args_IV}, \ - %%TGP0, %%TGP1, %%TGP2, \ + %%TGP0, %%TGP1, %%TGP2, %%TGP3, %%TGP4, \ %%TMP_XMM_0, %%TMP_XMM_1, %%TMP_XMM_2, \ %%TMP_XMM_3, %%TMP_XMM_4, %%TMP_XMM_5, \ %%TMP_XMM_6, %%TMP_XMM_7, %%TMP_XMM_8, \ @@ -283,10 +287,10 @@ APPEND(skip_lane_copy_,i): MKGLOBAL(SUBMIT_JOB_SNOW3G_UIA2,function,internal) SUBMIT_JOB_SNOW3G_UIA2: SUBMIT_FLUSH_JOB_SNOW3G_UIA2 submit, tmp_gp0, tmp_gp1, \ - tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ - xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, \ - xmm13, xmm14, xmm15 + tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, \ + tmp_gp6, xmm0, xmm1, xmm2, xmm3, xmm4, \ + xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, \ + xmm11, xmm12, xmm13, xmm14, xmm15 ret ; JOB* FLUSH_JOB_SNOW3G_UIA2(MB_MGR_SNOW3G_OOO *state) @@ -294,10 +298,10 @@ SUBMIT_JOB_SNOW3G_UIA2: MKGLOBAL(FLUSH_JOB_SNOW3G_UIA2,function,internal) FLUSH_JOB_SNOW3G_UIA2: SUBMIT_FLUSH_JOB_SNOW3G_UIA2 flush, tmp_gp0, tmp_gp1, \ - tmp_gp2, tmp_gp3, tmp_gp4, xmm0, \ - xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, \ - xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, \ - xmm13, xmm14, xmm15 + tmp_gp2, tmp_gp3, tmp_gp4, tmp_gp5, \ + tmp_gp6, xmm0, xmm1, xmm2, xmm3, xmm4, \ + xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, \ + xmm11, xmm12, xmm13, xmm14, xmm15 ret mksection stack-noexec diff --git a/lib/sse_t1/snow3g_uea2_by4_sse.asm b/lib/sse_t1/snow3g_uea2_by4_sse.asm index 7e37d652..5b3a3561 100644 --- a/lib/sse_t1/snow3g_uea2_by4_sse.asm +++ b/lib/sse_t1/snow3g_uea2_by4_sse.asm @@ -34,6 +34,10 @@ %include "include/memcpy.asm" %include "include/transpose_sse.asm" +extern snow3g_table_A_mul +extern snow3g_table_A_div +extern snow3g_table_S2 + align 64 const_fixup_mask: times 2 dq 0x7272727272727272 @@ -169,7 +173,7 @@ mksection .text struc STACK _keystream: resb (4 * 16) -_gpr_save: resq 8 +_gpr_save: resq 10 _rsp_save: resq 1 endstruc @@ -187,11 +191,14 @@ endstruc mov [rsp + _gpr_save + 8 * 3], rsi mov [rsp + _gpr_save + 8 * 4], rdi mov [rsp + _gpr_save + 8 * 5], r13 + mov [rsp + _gpr_save + 8 * 6], r14 + mov [rsp + _gpr_save + 8 * 7], r15 + %ifdef LINUX - mov [rsp + _gpr_save + 8 * 6], r9 + mov [rsp + _gpr_save + 8 * 8], r9 %else - mov [rsp + _gpr_save + 8 * 6], rcx - mov [rsp + _gpr_save + 8 * 7], rdx + mov [rsp + _gpr_save + 8 * 8], rcx + mov [rsp + _gpr_save + 8 * 9], rdx %endif mov [rsp + _rsp_save], rax ;; original SP %endmacro @@ -206,11 +213,13 @@ endstruc mov rsi, [rsp + _gpr_save + 8 * 3] mov rdi, [rsp + _gpr_save + 8 * 4] mov r13, [rsp + _gpr_save + 8 * 5] + mov r14, [rsp + _gpr_save + 8 * 6] + mov r15, [rsp + _gpr_save + 8 * 7] %ifdef LINUX - mov r9, [rsp + _gpr_save + 8 * 6] + mov r9, [rsp + _gpr_save + 8 * 8] %else - mov rcx, [rsp + _gpr_save + 8 * 6] - mov rdx, [rsp + _gpr_save + 8 * 7] + mov rcx, [rsp + _gpr_save + 8 * 8] + mov rdx, [rsp + _gpr_save + 8 * 9] %endif mov rsp, [rsp + _rsp_save] ;; original SP %endmacro @@ -363,6 +372,24 @@ endstruc por %%OUT_SUBSTITUTE_VAL, %%TMP2 %endmacro +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Search SNOW3G S2 box value per byte from FSM2 indicated in args. +;; Fill single dword in output depending on given lane nr +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%macro S2_BOX_BYTE_SEARCH 7 +%define %%OUT %1 ;; [out] xmm register (1 dword filled) +%define %%FSM_R2 %2 ;; [in] ptr to FSM2 values per 4 lanes +%define %%TABLE_PTR %3 ;; [in] address of table for search +%define %%LANE %4 ;; [in] lane nr +%define %%BYTE_NR %5 ;; [in] byte number for search (from FSM2) +%define %%BYTE_OFFSET %6 ;; [in] byte offset for output +%define %%TMP_64_1 %7 ;; [clobbered] temp gpr + + movzx %%TMP_64_1, byte[%%FSM_R2 + %%LANE*4 + %%BYTE_NR] + pinsrd %%OUT, [%%TABLE_PTR + %%TMP_64_1*8 + %%BYTE_OFFSET], %%LANE + +%endmacro + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; SNOW3G S2 box calculation for 4 32-bit values passed in 1st input parameter. ;; Clobbers all 15 input xmm registers @@ -416,41 +443,76 @@ endstruc ;; 32-bit values each. ;; Values under FSM_R1-FSM_R3 are updated as a result of this macro. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro SNOW3G_FSM_CLOCK 20 +%macro SNOW3G_FSM_CLOCK 11-20 %define %%FSM_R1 %1 ;; [in] address of 4 FSM values R1 %define %%FSM_R2 %2 ;; [in] address of 4 FSM values R2 %define %%FSM_R3 %3 ;; [in] address of 4 FSM values R3 -%define %%TMP1 %4 ;; [clobbered] temp xmm register -%define %%TMP2 %5 ;; [clobbered] temp xmm register -%define %%TMP3 %6 ;; [clobbered] temp xmm register -%define %%TMP4 %7 ;; [clobbered] temp xmm register -%define %%TMP5 %8 ;; [clobbered] temp xmm register -%define %%TMP6 %9 ;; [clobbered] temp xmm register -%define %%TMP7 %10 ;; [clobbered] temp xmm register -%define %%TMP8 %11 ;; [clobbered] temp xmm register -%define %%TMP9 %12 ;; [clobbered] temp xmm register -%define %%TMP10 %13 ;; [clobbered] temp xmm register -%define %%TMP11 %14 ;; [clobbered] temp xmm register -%define %%TMP12 %15 ;; [clobbered] temp xmm register -%define %%TMP13 %16 ;; [clobbered] temp xmm register -%define %%TMP14 %17 ;; [clobbered] temp xmm register -%define %%TMP15 %18 ;; [clobbered] temp xmm register -%define %%TMP16 %19 ;; [clobbered] temp xmm register +%define %%TMP1 %4 ;; [clobbered] temp xmm register +%define %%TMP2 %5 ;; [clobbered] temp xmm register +%define %%TMP3 %6 ;; [clobbered] temp xmm register +%define %%TMP4 %7 ;; [clobbered] temp xmm register +%define %%TMP5 %8 ;; [clobbered] temp xmm register +%ifdef SAFE_LOOKUP +%define %%TMP6 %9 ;; [clobbered] temp xmm register +%define %%TMP7 %10 ;; [clobbered] temp xmm register +%define %%TMP8 %11 ;; [clobbered] temp xmm register +%define %%TMP9 %12 ;; [clobbered] temp xmm register +%define %%TMP10 %13 ;; [clobbered] temp xmm register +%define %%TMP11 %14 ;; [clobbered] temp xmm register +%define %%TMP12 %15 ;; [clobbered] temp xmm register +%define %%TMP13 %16 ;; [clobbered] temp xmm register +%define %%TMP14 %17 ;; [clobbered] temp xmm register +%define %%TMP15 %18 ;; [clobbered] temp xmm register +%define %%TMP16 %19 ;; [clobbered] temp xmm register %define %%LFSR_5 %20 ;; [in] address of 4 LFSR 5 values +%else +%define %%TMP_64 %9 ;; [clobbered] temp gp register +%define %%TMP_64_1 %10 ;; [clobbered] temp gp register +%define %%LFSR_5 %11 ;; [in] address of 4 LFSR 5 values - movdqa %%TMP1, [ %%FSM_R2 ] +%endif ;; SAFE_LOOKUP ;; FSM_3 = S2_box(FSM_2) - S2_BOX_SSE %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ +%ifdef SAFE_LOOKUP + movdqa %%TMP15, [ %%FSM_R2 ] + S2_BOX_SSE %%TMP15, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, \ - %%TMP11, %%TMP12, %%TMP13, %%TMP14, %%TMP15 + %%TMP11, %%TMP12, %%TMP13, %%TMP14, %%TMP1 +%else + lea %%TMP_64, [rel snow3g_table_S2] + ;; w0= S2[(x >> 24) & 0xff]; + ;; w1= S2[(x >> 16) & 0xff]; + ;; w2= S2[(x >> 8) & 0xff]; + ;; w3= S2[x & 0xff]; + S2_BOX_BYTE_SEARCH %%TMP1, %%FSM_R2, %%TMP_64, 0, 3, 0, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP2, %%FSM_R2, %%TMP_64, 0, 2, 1, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP3, %%FSM_R2, %%TMP_64, 0, 1, 2, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP4, %%FSM_R2, %%TMP_64, 0, 0, 3, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP1, %%FSM_R2, %%TMP_64, 1, 3, 0, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP2, %%FSM_R2, %%TMP_64, 1, 2, 1, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP3, %%FSM_R2, %%TMP_64, 1, 1, 2, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP4, %%FSM_R2, %%TMP_64, 1, 0, 3, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP1, %%FSM_R2, %%TMP_64, 2, 3, 0, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP2, %%FSM_R2, %%TMP_64, 2, 2, 1, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP3, %%FSM_R2, %%TMP_64, 2, 1, 2, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP4, %%FSM_R2, %%TMP_64, 2, 0, 3, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP1, %%FSM_R2, %%TMP_64, 3, 3, 0, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP2, %%FSM_R2, %%TMP_64, 3, 2, 1, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP3, %%FSM_R2, %%TMP_64, 3, 1, 2, %%TMP_64_1 + S2_BOX_BYTE_SEARCH %%TMP4, %%FSM_R2, %%TMP_64, 3, 0, 3, %%TMP_64_1 + + pxor %%TMP4, %%TMP3 + pxor %%TMP1, %%TMP2 + pxor %%TMP1, %%TMP4 +%endif ;; SAFE_LOOKUP + ;; R = (FSM_R3 ^ LFSR_5) + FSM_R2 - movdqa %%TMP16, [%%FSM_R3] - pxor %%TMP16, [%%LFSR_5] - paddd %%TMP16, [%%FSM_R2] + movdqa %%TMP5, [%%FSM_R3] + pxor %%TMP5, [%%LFSR_5] + paddd %%TMP5, [%%FSM_R2] ;; FSM_3 = S2_box(FSM_2) - movdqa [%%FSM_R3], %%TMP15 + movdqa [%%FSM_R3], %%TMP1 ;; FSM_R2 = S1_box(FSM_R1) movdqa %%TMP3, [%%FSM_R1] @@ -463,7 +525,7 @@ endstruc movdqa [%%FSM_R2], %%TMP3 ;; FSM_1 = R - movdqa [%%FSM_R1], %%TMP16 + movdqa [%%FSM_R1], %%TMP5 %endmacro @@ -475,6 +537,7 @@ endstruc ;; Result of mul_alpha and div_alpha operations are precalculated and expected ;; under %%OP_TABLE address. This function searches those tables. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%ifdef SAFE_LOOKUP %macro ALPHA_OP 7 %define %%LFSR_X %1 ;; [in/clobbered] xmm with LFSR value %define %%OP_TABLE %2 ;; [in] address of mulalpha/divalpha val table @@ -521,6 +584,47 @@ endstruc pxor %%TMP1, %%TMP2 %endmacro +%else ;; SAFE_LOOKUP + +%macro ALPHA_OP_NOT_SAFE 5 +%define %%LFSR_PTR %1 ;; [in] r64 with address of LFSR register + ;; for mulalpha pass LFSR + ;; for divalpha pass LSFR 11 +%define %%OP_TABLE %2 ;; [in] address of mulalpha/divalpha val table +%define %%TMP1 %3 ;; [out] temporary xmm register +%define %%TMP_64 %4 ;; [clobbered] temporary gp register +%define %%TMP_64_1 %5 ;; [clobbered] temporary gp register + lea %%TMP_64, [rel %%OP_TABLE] + +%ifidn %%OP_TABLE, snow3g_table_A_div + movzx %%TMP_64_1, byte[%%LFSR_PTR] +%else + movzx %%TMP_64_1, byte[%%LFSR_PTR+3] +%endif + movd %%TMP1, [%%TMP_64 + %%TMP_64_1*4] + +%ifidn %%OP_TABLE, snow3g_table_A_div + movzx %%TMP_64_1, byte[%%LFSR_PTR+4] +%else + movzx %%TMP_64_1, byte[%%LFSR_PTR+7] +%endif + pinsrd %%TMP1, [%%TMP_64 + %%TMP_64_1*4], 1 +%ifidn %%OP_TABLE, snow3g_table_A_div + movzx %%TMP_64_1, byte[%%LFSR_PTR+8] +%else + movzx %%TMP_64_1, byte[%%LFSR_PTR+11] +%endif + pinsrd %%TMP1, [%%TMP_64 + %%TMP_64_1*4], 2 +%ifidn %%OP_TABLE, snow3g_table_A_div + movzx %%TMP_64_1, byte[%%LFSR_PTR+12] +%else + movzx %%TMP_64_1, byte[%%LFSR_PTR+15] +%endif + pinsrd %%TMP1, [%%TMP_64 + %%TMP_64_1*4], 3 +%endmacro + +%endif ;; SAFE_LOOKUP + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Perform SNOW3G LFSR shift operation. ;; This operation is common for initialization mode and keystream mode, the only @@ -582,8 +686,8 @@ endstruc ;; In initialization mode F is stored on stack. ;; In keystream mode keystream is stored on stack. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro SNOW3G_KEY_GEN_SSE 18 -%define %%STATE %1 ;; [in] ptr to LFSR/FSM struct +%macro SNOW3G_KEY_GEN_SSE 12-18 +%define %%STATE %1 ;; [in] ptr to LFSR/FSM struct %define %%TMP1 %2 ;; [clobbered] temporary xmm register %define %%TMP2 %3 ;; [clobbered] temporary xmm register %define %%TMP3 %4 ;; [clobbered] temporary xmm register @@ -591,6 +695,7 @@ endstruc %define %%TMP5 %6 ;; [clobbered] temporary xmm register %define %%TMP6 %7 ;; [clobbered] temporary xmm register %define %%TMP7 %8 ;; [clobbered] temporary xmm register +%ifdef SAFE_LOOKUP %define %%TMP8 %9 ;; [clobbered] temporary xmm register %define %%TMP9 %10 ;; [clobbered] temporary xmm register %define %%TMP10 %11 ;; [clobbered] temporary xmm register @@ -598,9 +703,16 @@ endstruc %define %%TMP12 %13 ;; [clobbered] temporary xmm register %define %%TMP13 %14 ;; [clobbered] temporary xmm register %define %%TMP14 %15 ;; [clobbered] temporary xmm register -%define %%TMP15 %16 ;; [out] generated keystream -%define %%TMP16 %17 ;; [out] generated keystream +%define %%TMP15 %16 ;; [clobbered] temporary xmm register +%define %%TMP16 %17 ;; [clobbered] temporary xmm register %define %%DWORD_ITER %18 ;; [in] gp reg with offset for stack storing keystream +%else ;;SAFE_LOOKUP +%define %%TMP15 %9 ;; [clobbered] temporary xmm register +%define %%TMP_64_1 %10 ;; [clobbered] temp gpr +%define %%TMP_64_2 %11 ;; [clobbered] temp gpr +%define %%DWORD_ITER %12 ;; [in] gp reg with offset for stack storing keystream +%endif ;;SAFE_LOOKUP + ;; Calculate F = (LFSR_S[15] + FSM_R1) ^ FSM_R2; movdqa %%TMP1, [ %%STATE + _snow3g_args_LFSR_15 ] @@ -619,6 +731,7 @@ endstruc movdqa [rsp + _keystream + %%DWORD_ITER], %%TMP2 ;; FSM Clock +%ifdef SAFE_LOOKUP SNOW3G_FSM_CLOCK {%%STATE + _snow3g_args_FSM_1}, \ {%%STATE + _snow3g_args_FSM_2}, \ {%%STATE + _snow3g_args_FSM_3}, %%TMP1, %%TMP2, \ @@ -626,20 +739,31 @@ endstruc %%TMP8, %%TMP9, %%TMP10, %%TMP11, %%TMP12, \ %%TMP13, %%TMP14, %%TMP15, %%TMP16, \ %%STATE + _snow3g_args_LFSR_5 - - ;; LFSR clock: mul alpha movdqa %%TMP15, [ %%STATE + _snow3g_args_LFSR_0 ] movdqa %%TMP2, [rel ms_byte_mask] pshufb %%TMP15, %%TMP2 ALPHA_OP %%TMP15, mul_alpha, %%TMP2, %%TMP3, %%TMP4, \ %%TMP5, %%TMP6 + ; LFSR clock: div alpha - ;; LFSR clock: div alpha movdqa %%TMP15, [%%STATE + _snow3g_args_LFSR_11 ] movdqa %%TMP7, [rel ls_byte_mask] pshufb %%TMP15, %%TMP7 ALPHA_OP %%TMP15, div_alpha, %%TMP7, %%TMP3, %%TMP4, \ %%TMP5, %%TMP6 +%else + SNOW3G_FSM_CLOCK {%%STATE + _snow3g_args_FSM_1}, \ + {%%STATE + _snow3g_args_FSM_2}, \ + {%%STATE + _snow3g_args_FSM_3}, %%TMP1, %%TMP2, \ + %%TMP3, %%TMP4, %%TMP5, %%TMP_64_1, %%TMP_64_2, \ + %%STATE + _snow3g_args_LFSR_5 + ALPHA_OP_NOT_SAFE {%%STATE + _snow3g_args_LFSR_0}, snow3g_table_A_mul, \ + %%TMP2, %%TMP_64_1, %%TMP_64_2 + + + ALPHA_OP_NOT_SAFE {%%STATE + _snow3g_args_LFSR_11}, snow3g_table_A_div, \ + %%TMP7, %%TMP_64_1, %%TMP_64_2 +%endif ;; SAFE_LOOKUP movdqa %%TMP15, [%%STATE + _snow3g_args_LFSR_2 ] pxor %%TMP15, %%TMP2 @@ -879,9 +1003,14 @@ endstruc xor %%TMP1_64, %%TMP1_64 %%next_dqw_round: +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP2_64, %%IN, %%TMP1_64 +%endif inc %%TMP1_64 cmp %%TMP1_64, 4 @@ -904,9 +1033,14 @@ endstruc %%next_dw: xor %%TMP1_64, %%TMP1_64 +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP2_64, %%IN, %%TMP1_64 +%endif SNOW3G_OUTPUT 4, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 SNOW3G_OUTPUT 4, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%TMP2_64 @@ -919,9 +1053,14 @@ endstruc %%no_full_dws_to_write_out: ;; Process last dw/bytes: xor %%TMP1_64, %%TMP1_64 +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP2_64, %%IN, %%TMP1_64 +%endif SNOW3G_OUTPUT 3, %%STATE, 0, %%IN, %%OUT, %%TMP1_64, %%LENGTH SNOW3G_OUTPUT 3, %%STATE, 1, %%IN, %%OUT, %%TMP1_64, %%LENGTH @@ -934,29 +1073,31 @@ endstruc ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Generate 5 double words of key stream for SNOW3G authentication ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro SNOW3G_AUTH_INIT_5_BY_4 22 +%macro SNOW3G_AUTH_INIT_5_BY_4 24 %define %%KEY %1 ;; [in] array of pointers to 4 keys %define %%IV %2 ;; [in] array of pointers to 4 IV's %define %%DST_PTR %3 ;; [in] destination buffer to put 5DW of keystream for each lane %define %%TMP1_64 %4 ;; [clobbered] r64 gp reg temp %define %%TMP2_64 %5 ;; [clobbered] r64 gp reg temp -%define %%TMP1 %6 ;; [clobbered] temporary xmm register -%define %%TMP2 %7 ;; [clobbered] temporary xmm register -%define %%TMP3 %8 ;; [clobbered] temporary xmm register -%define %%TMP4 %9 ;; [clobbered] temporary xmm register -%define %%TMP5 %10 ;; [clobbered] temporary xmm register -%define %%TMP6 %11 ;; [clobbered] temporary xmm register -%define %%TMP7 %12 ;; [clobbered] temporary xmm register -%define %%TMP8 %13 ;; [clobbered] temporary xmm register -%define %%TMP9 %14 ;; [clobbered] temporary xmm register -%define %%TMP10 %15 ;; [clobbered] temporary xmm register -%define %%TMP11 %16 ;; [clobbered] temporary xmm register -%define %%TMP12 %17 ;; [clobbered] temporary xmm register -%define %%TMP13 %18 ;; [clobbered] temporary xmm register -%define %%TMP14 %19 ;; [clobbered] temporary xmm register -%define %%TMP15 %20 ;; [clobbered] temporary xmm register -%define %%TMP16 %21 ;; [clobbered] temporary xmm register -%define %%STATE %22 ;; [in] ptr to LFSR/FSM struct +%define %%TMP3_64 %6 ;; [clobbered] r64 gp reg temp +%define %%TMP4_64 %7 ;; [clobbered] r64 gp reg temp +%define %%TMP1 %8 ;; [clobbered] temporary xmm register +%define %%TMP2 %9 ;; [clobbered] temporary xmm register +%define %%TMP3 %10 ;; [clobbered] temporary xmm register +%define %%TMP4 %11 ;; [clobbered] temporary xmm register +%define %%TMP5 %12 ;; [clobbered] temporary xmm register +%define %%TMP6 %13 ;; [clobbered] temporary xmm register +%define %%TMP7 %14 ;; [clobbered] temporary xmm register +%define %%TMP8 %15 ;; [clobbered] temporary xmm register +%define %%TMP9 %16 ;; [clobbered] temporary xmm register +%define %%TMP10 %17 ;; [clobbered] temporary xmm register +%define %%TMP11 %18 ;; [clobbered] temporary xmm register +%define %%TMP12 %19 ;; [clobbered] temporary xmm register +%define %%TMP13 %20 ;; [clobbered] temporary xmm register +%define %%TMP14 %21 ;; [clobbered] temporary xmm register +%define %%TMP15 %22 ;; [clobbered] temporary xmm register +%define %%TMP16 %23 ;; [clobbered] temporary xmm register +%define %%STATE %24 ;; [in] ptr to LFSR/FSM struct %define KEYGEN_STAGE _snow3g_args_LD_ST_MASK %define INIT1_DONE _snow3g_args_LD_ST_MASK+16 @@ -975,9 +1116,14 @@ endstruc xor %%TMP2_64, %%TMP2_64 %%next_auth_round: +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP3_64, %%TMP4_64, %%TMP2_64 +%endif dec %%TMP1_64 jnz %%next_auth_round @@ -985,9 +1131,14 @@ endstruc movdqa %%TMP1, [rel all_fs] movdqa [state + INIT1_DONE], %%TMP1 +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP3_64, %%TMP4_64, %%TMP2_64 +%endif ;; Put all lanes in KEYGEN state movdqa %%TMP1, [rel all_fs] @@ -997,9 +1148,14 @@ endstruc xor %%TMP1_64, %%TMP1_64 %%next_auth_round2: +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP1_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP3_64, %%TMP4_64, %%TMP1_64 +%endif inc %%TMP1_64 cmp %%TMP1_64, 4 jb %%next_auth_round2 @@ -1013,9 +1169,14 @@ endstruc movdqu [%%DST_PTR + 3*32], %%TMP4 ;; Generate final dw of keystream for each lane +%ifdef SAFE_LOOKUP SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ %%TMP6, %%TMP7, %%TMP8, %%TMP9, %%TMP10, %%TMP11, \ %%TMP12, %%TMP13, %%TMP14, %%TMP15, %%TMP16, %%TMP2_64 +%else + SNOW3G_KEY_GEN_SSE %%STATE, %%TMP1, %%TMP2, %%TMP3, %%TMP4, %%TMP5, \ + %%TMP6, %%TMP7, %%TMP15, %%TMP3_64, %%TMP4_64, %%TMP2_64 +%endif ;; Store final dw of keystream for each lane mov DWORD(%%TMP1_64), [rsp + _keystream + 0*4] -- GitLab From ff23ee913e990609752543c027b7d9c2163df9a2 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 5 Sep 2022 16:51:33 +0100 Subject: [PATCH 337/369] test: add new line after info messages --- test/direct_api_param_test.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/direct_api_param_test.c b/test/direct_api_param_test.c index 0a32fbce..ad0f7ff9 100644 --- a/test/direct_api_param_test.c +++ b/test/direct_api_param_test.c @@ -5248,14 +5248,15 @@ direct_api_param_test(struct IMB_MGR *mb_mgr) void *handler; #endif #endif - printf("Extended Invalid Direct API arguments test:"); + printf("Extended Invalid Direct API arguments test:\n"); test_suite_start(&ts, "INVALID-ARGS"); #ifndef DEBUG handler = signal(SIGSEGV, seg_handler); #endif if ((mb_mgr->features & IMB_FEATURE_SAFE_PARAM) == 0) { - printf("SAFE_PARAM feature disabled, skipping remaining tests"); + printf("SAFE_PARAM feature disabled, " + "skipping remaining tests\n"); goto dir_api_exit; } errors += test_IMB_AES_KEYEXP_128(mb_mgr); -- GitLab From 8492a7b7b0396ecb102b93fb77530efbbd12638a Mon Sep 17 00:00:00 2001 From: De Lara Guarch Date: Tue, 6 Sep 2022 09:30:37 +0100 Subject: [PATCH 338/369] lib: [HMAC-SHA1] preserve XMM6-15 for Windows --- lib/avx/sha1_x4_avx.asm | 36 +++++++++++++++- lib/avx2_t1/sha1_x8_avx2.asm | 36 +++++++++++++++- .../mb_mgr_hmac_sha1_flush_avx512.asm | 2 +- .../mb_mgr_hmac_sha1_submit_avx512.asm | 2 +- lib/avx512_t1/sha1_x16_avx512.asm | 41 ++++++++++++++++++- lib/sse_t1/sha1_x4_sse.asm | 38 +++++++++++++++-- lib/sse_t2/sha1_ni_x1_sse.asm | 8 ++++ lib/sse_t2/sha1_ni_x2_sse.asm | 21 ++++++++++ 8 files changed, 176 insertions(+), 8 deletions(-) diff --git a/lib/avx/sha1_x4_avx.asm b/lib/avx/sha1_x4_avx.asm index 79e68f3e..b67ef360 100644 --- a/lib/avx/sha1_x4_avx.asm +++ b/lib/avx/sha1_x4_avx.asm @@ -230,7 +230,7 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; FRAMESZ must be an odd multiple of 8 -%define FRAMESZ 16*16 + 8 +%define FRAMESZ 16*16 + 16*10 + 8 %define VMOVPS vmovdqu @@ -355,6 +355,19 @@ sha1_mult_avx: sub rsp, FRAMESZ +%ifndef LINUX + vmovdqa [rsp + 16*16 + 0*16], xmm6 + vmovdqa [rsp + 16*16 + 1*16], xmm7 + vmovdqa [rsp + 16*16 + 2*16], xmm8 + vmovdqa [rsp + 16*16 + 3*16], xmm9 + vmovdqa [rsp + 16*16 + 4*16], xmm10 + vmovdqa [rsp + 16*16 + 5*16], xmm11 + vmovdqa [rsp + 16*16 + 6*16], xmm12 + vmovdqa [rsp + 16*16 + 7*16], xmm13 + vmovdqa [rsp + 16*16 + 8*16], xmm14 + vmovdqa [rsp + 16*16 + 9*16], xmm15 +%endif + ;; Initialize digests vmovdqa A, [arg1 + 0*SHA1_DIGEST_ROW_SIZE] vmovdqa B, [arg1 + 1*SHA1_DIGEST_ROW_SIZE] @@ -481,6 +494,27 @@ lloop: %endrep %endif +%ifndef LINUX + vmovdqa xmm6, [rsp + 16*16 + 0*16] + vmovdqa xmm7, [rsp + 16*16 + 1*16] + vmovdqa xmm8, [rsp + 16*16 + 2*16] + vmovdqa xmm9, [rsp + 16*16 + 3*16] + vmovdqa xmm10, [rsp + 16*16 + 4*16] + vmovdqa xmm11, [rsp + 16*16 + 5*16] + vmovdqa xmm12, [rsp + 16*16 + 6*16] + vmovdqa xmm13, [rsp + 16*16 + 7*16] + vmovdqa xmm14, [rsp + 16*16 + 8*16] + vmovdqa xmm15, [rsp + 16*16 + 9*16] + +%ifdef SAFE_DATA + ; xmm0 already 0 +%assign i 0 +%rep 10 + vmovdqa [rsp + 16*16 + i*16], xmm0 +%assign i (i+1) +%endrep +%endif +%endif add rsp, FRAMESZ ret diff --git a/lib/avx2_t1/sha1_x8_avx2.asm b/lib/avx2_t1/sha1_x8_avx2.asm index 04b288dd..5b0de3e5 100644 --- a/lib/avx2_t1/sha1_x8_avx2.asm +++ b/lib/avx2_t1/sha1_x8_avx2.asm @@ -146,7 +146,7 @@ mksection .text ;; Assume stack aligned to 32 bytes before call ;; Therefore FRAMESIZE mod 32 must be 32-8 = 24 -%define FRAMESZ 32*16 + 24 +%define FRAMESZ 32*16 + 16*10 + 24 %define VMOVPS vmovups @@ -368,6 +368,19 @@ MKGLOBAL(sha1_x8_avx2,function,internal) sha1_x8_avx2: sub rsp, FRAMESZ +%ifndef LINUX + vmovdqa [rsp + 32*16 + 0*16], xmm6 + vmovdqa [rsp + 32*16 + 1*16], xmm7 + vmovdqa [rsp + 32*16 + 2*16], xmm8 + vmovdqa [rsp + 32*16 + 3*16], xmm9 + vmovdqa [rsp + 32*16 + 4*16], xmm10 + vmovdqa [rsp + 32*16 + 5*16], xmm11 + vmovdqa [rsp + 32*16 + 6*16], xmm12 + vmovdqa [rsp + 32*16 + 7*16], xmm13 + vmovdqa [rsp + 32*16 + 8*16], xmm14 + vmovdqa [rsp + 32*16 + 9*16], xmm15 +%endif + ;; Initialize digests vmovdqu A, [state + 0*SHA1_DIGEST_ROW_SIZE] vmovdqu B, [state + 1*SHA1_DIGEST_ROW_SIZE] @@ -516,6 +529,27 @@ lloop: %endrep %endif +%ifndef LINUX + vmovdqa xmm6, [rsp + 32*16 + 0*16] + vmovdqa xmm7, [rsp + 32*16 + 1*16] + vmovdqa xmm8, [rsp + 32*16 + 2*16] + vmovdqa xmm9, [rsp + 32*16 + 3*16] + vmovdqa xmm10, [rsp + 32*16 + 4*16] + vmovdqa xmm11, [rsp + 32*16 + 5*16] + vmovdqa xmm12, [rsp + 32*16 + 6*16] + vmovdqa xmm13, [rsp + 32*16 + 7*16] + vmovdqa xmm14, [rsp + 32*16 + 8*16] + vmovdqa xmm15, [rsp + 32*16 + 9*16] + +%ifdef SAFE_DATA + ; xmm0 already 0 +%assign i 0 +%rep 10 + vmovdqa [rsp + 32*16 + i*16], xmm0 +%assign i (i+1) +%endrep +%endif +%endif add rsp, FRAMESZ ret diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm index f374e064..3cc4be23 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_flush_avx512.asm @@ -351,7 +351,7 @@ APPEND(skip_clear_,I): %endif ;; SAFE_DATA %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif diff --git a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm index 5cf8ff3b..947994bb 100644 --- a/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm +++ b/lib/avx512_t1/mb_mgr_hmac_sha1_submit_avx512.asm @@ -382,7 +382,7 @@ clear_ret: return: %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif diff --git a/lib/avx512_t1/sha1_x16_avx512.asm b/lib/avx512_t1/sha1_x16_avx512.asm index 501be9b8..afa52c58 100644 --- a/lib/avx512_t1/sha1_x16_avx512.asm +++ b/lib/avx512_t1/sha1_x16_avx512.asm @@ -290,12 +290,29 @@ mksection .text mov rsp, [rsp + GP_OFFSET + 4*8] ;; rsp pointer %endmacro +;; FRAMESZ must be an odd multiple of 8 +%define FRAMESZ 16*10 + 8 + align 64 ; void sha1_mult_x16_avx3(void **input_data, UINT128 *digest, UINT32 size) ; arg 1 : pointer to SHA1 args structure ; arg 2 : size (in blocks) ;; assumed to be >= 1 MKGLOBAL(sha1_x16_avx512,function,internal) sha1_x16_avx512: +%ifndef LINUX + sub rsp, FRAMESZ + + vmovdqa [rsp + 0*16], xmm6 + vmovdqa [rsp + 1*16], xmm7 + vmovdqa [rsp + 2*16], xmm8 + vmovdqa [rsp + 3*16], xmm9 + vmovdqa [rsp + 4*16], xmm10 + vmovdqa [rsp + 5*16], xmm11 + vmovdqa [rsp + 6*16], xmm12 + vmovdqa [rsp + 7*16], xmm13 + vmovdqa [rsp + 8*16], xmm14 + vmovdqa [rsp + 9*16], xmm15 +%endif ;; Initialize digests vmovdqu32 A, [state + 0*SHA1_DIGEST_ROW_SIZE] vmovdqu32 B, [state + 1*SHA1_DIGEST_ROW_SIZE] @@ -496,11 +513,33 @@ lastLoop: mov [state + _data_ptr_sha1 + 15*PTR_SZ], inp7 %ifdef SAFE_DATA - clear_all_zmms_asm + clear_scratch_zmms_asm %else vzeroupper %endif ;; SAFE_DATA +%ifndef LINUX + vmovdqa xmm6, [rsp + 0*16] + vmovdqa xmm7, [rsp + 1*16] + vmovdqa xmm8, [rsp + 2*16] + vmovdqa xmm9, [rsp + 3*16] + vmovdqa xmm10, [rsp + 4*16] + vmovdqa xmm11, [rsp + 5*16] + vmovdqa xmm12, [rsp + 6*16] + vmovdqa xmm13, [rsp + 7*16] + vmovdqa xmm14, [rsp + 8*16] + vmovdqa xmm15, [rsp + 9*16] + +%ifdef SAFE_DATA + ; xmm0 already 0 +%assign i 0 +%rep 10 + vmovdqa [rsp + i*16], xmm0 +%assign i (i+1) +%endrep +%endif ;SAFE_DATA + add rsp, FRAMESZ +%endif ; !LINUX ret ; void call_sha1_x16_avx512_from_c(SHA1_ARGS *args, UINT32 size_in_blocks); diff --git a/lib/sse_t1/sha1_x4_sse.asm b/lib/sse_t1/sha1_x4_sse.asm index b648c5ff..7994bc1b 100644 --- a/lib/sse_t1/sha1_x4_sse.asm +++ b/lib/sse_t1/sha1_x4_sse.asm @@ -228,7 +228,7 @@ mksection .text ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -%define FRAMESZ 16*16 + 8 +%define FRAMESZ 16*16 + 16*10 + 8 %define MOVPS movdqu @@ -353,6 +353,18 @@ sha1_mult_sse: sub rsp, FRAMESZ +%ifndef LINUX + movdqa [rsp + 16*16 + 0*16], xmm6 + movdqa [rsp + 16*16 + 1*16], xmm7 + movdqa [rsp + 16*16 + 2*16], xmm8 + movdqa [rsp + 16*16 + 3*16], xmm9 + movdqa [rsp + 16*16 + 4*16], xmm10 + movdqa [rsp + 16*16 + 5*16], xmm11 + movdqa [rsp + 16*16 + 6*16], xmm12 + movdqa [rsp + 16*16 + 7*16], xmm13 + movdqa [rsp + 16*16 + 8*16], xmm14 + movdqa [rsp + 16*16 + 9*16], xmm15 +%endif ;; Initialize digests movdqa A, [arg1 + 0*SHA1_DIGEST_ROW_SIZE] movdqa B, [arg1 + 1*SHA1_DIGEST_ROW_SIZE] @@ -472,15 +484,35 @@ lloop: ;; Clear stack frame (16*16 bytes) %ifdef SAFE_DATA - pxor xmm0, xmm0 + clear_all_xmms_sse_asm %assign i 0 %rep 16 movdqa [rsp + i*16], xmm0 %assign i (i+1) %endrep - clear_all_xmms_sse_asm %endif +%ifndef LINUX + movdqa xmm6, [rsp + 16*16 + 0*16] + movdqa xmm7, [rsp + 16*16 + 1*16] + movdqa xmm8, [rsp + 16*16 + 2*16] + movdqa xmm9, [rsp + 16*16 + 3*16] + movdqa xmm10, [rsp + 16*16 + 4*16] + movdqa xmm11, [rsp + 16*16 + 5*16] + movdqa xmm12, [rsp + 16*16 + 6*16] + movdqa xmm13, [rsp + 16*16 + 7*16] + movdqa xmm14, [rsp + 16*16 + 8*16] + movdqa xmm15, [rsp + 16*16 + 9*16] + +%ifdef SAFE_DATA + ; xmm0 already 0 +%assign i 0 +%rep 10 + movdqa [rsp + 16*16 + i*16], xmm0 +%assign i (i+1) +%endrep +%endif +%endif add rsp, FRAMESZ ret diff --git a/lib/sse_t2/sha1_ni_x1_sse.asm b/lib/sse_t2/sha1_ni_x1_sse.asm index 9fdb2e73..8022746b 100644 --- a/lib/sse_t2/sha1_ni_x1_sse.asm +++ b/lib/sse_t2/sha1_ni_x1_sse.asm @@ -66,6 +66,7 @@ struc frame .E_SAVE reso 1 .ABCD_SAVEb reso 1 .E_SAVEb reso 1 +.XMM_SAVE reso 3 .align resq 1 endstruc @@ -103,6 +104,10 @@ align 32 sha1_ni_x1: sub rsp, frame_size + movdqa [rsp + frame.XMM_SAVE], xmm6 + movdqa [rsp + frame.XMM_SAVE + 16], xmm14 + movdqa [rsp + frame.XMM_SAVE + 16*2], xmm15 + shl NUM_BLKS, 6 ; convert to bytes jz done_hash @@ -322,6 +327,9 @@ done_hash: movdqa [rsp + 3*16], MSG0 %endif + movdqa xmm6, [rsp + frame.XMM_SAVE] + movdqa xmm14, [rsp + frame.XMM_SAVE + 16] + movdqa xmm15, [rsp + frame.XMM_SAVE + 16*2] add rsp, frame_size ret diff --git a/lib/sse_t2/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm index 8d1c30e7..27a61bbc 100644 --- a/lib/sse_t2/sha1_ni_x2_sse.asm +++ b/lib/sse_t2/sha1_ni_x2_sse.asm @@ -65,6 +65,7 @@ struc frame .E_SAVE reso 1 .ABCD_SAVEb reso 1 .E_SAVEb reso 1 +.XMM_SAVE reso 10 .align resq 1 endstruc @@ -174,6 +175,16 @@ align 32 sha1_ni: sub rsp, frame_size + movdqa [rsp + frame.XMM_SAVE], xmm6 + movdqa [rsp + frame.XMM_SAVE + 16], xmm7 + movdqa [rsp + frame.XMM_SAVE + 16*2], xmm8 + movdqa [rsp + frame.XMM_SAVE + 16*3], xmm9 + movdqa [rsp + frame.XMM_SAVE + 16*4], xmm10 + movdqa [rsp + frame.XMM_SAVE + 16*5], xmm11 + movdqa [rsp + frame.XMM_SAVE + 16*6], xmm12 + movdqa [rsp + frame.XMM_SAVE + 16*7], xmm13 + movdqa [rsp + frame.XMM_SAVE + 16*8], xmm14 + movdqa [rsp + frame.XMM_SAVE + 16*9], xmm15 DBGPRINTL "enter sha1-ni-x2" shl NUM_BLKS, 6 ; convert to bytes @@ -548,6 +559,16 @@ done_hash: clear_all_xmms_sse_asm %endif + movdqa xmm6, [rsp + frame.XMM_SAVE] + movdqa xmm7, [rsp + frame.XMM_SAVE + 16], + movdqa xmm8, [rsp + frame.XMM_SAVE + 16*2] + movdqa xmm9, [rsp + frame.XMM_SAVE + 16*3] + movdqa xmm10, [rsp + frame.XMM_SAVE + 16*4] + movdqa xmm11, [rsp + frame.XMM_SAVE + 16*5] + movdqa xmm12, [rsp + frame.XMM_SAVE + 16*6] + movdqa xmm13, [rsp + frame.XMM_SAVE + 16*7] + movdqa xmm14, [rsp + frame.XMM_SAVE + 16*8] + movdqa xmm15, [rsp + frame.XMM_SAVE + 16*9] add rsp, frame_size ret -- GitLab From 4ed48c29fffafae7fd2517151d850391a1cd4f38 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Wed, 7 Sep 2022 15:21:31 +0000 Subject: [PATCH 339/369] lib: [SHA256-NI] update data pointers after digest --- .../mb_mgr_hmac_sha256_flush_ni_sse.asm | 2 +- lib/sse_t2/sha256_ni_x1_sse.asm | 41 +++++++++---------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm index a566b12c..07b284b8 100644 --- a/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm +++ b/lib/sse_t2/mb_mgr_hmac_sha256_flush_ni_sse.asm @@ -148,7 +148,7 @@ copy_lane_data: je len_is_0 ; set length lane to 0 - mov dword [state + _lens_sha256 + idx*2], 0 + mov dword [state + _lens_sha256], 0 ; "state" and "args" are the same address, arg1 ; len is arg2 diff --git a/lib/sse_t2/sha256_ni_x1_sse.asm b/lib/sse_t2/sha256_ni_x1_sse.asm index e1bef2a7..b6a02647 100644 --- a/lib/sse_t2/sha256_ni_x1_sse.asm +++ b/lib/sse_t2/sha256_ni_x1_sse.asm @@ -65,12 +65,9 @@ endstruc %define args arg1 %define NUM_BLKS arg2 %define lane arg3 -%define tmp arg4 %define INP r10 -%define SHA256CONSTANTS r11 - ;; MSG MUST be xmm0 (implicit argument) %define MSG xmm0 %define STATE0 xmm1 @@ -194,7 +191,6 @@ sha256_ni_x1: palignr STATE0, STATE1, 8 ; ABEF pblendw STATE1, MSGTMP4, 0xF0 ; CDGH - lea SHA256CONSTANTS,[rel K256] movdqa SHUF_MASK, [rel PSHUFFLE_BYTE_FLIP_MASK] .loop0: @@ -206,7 +202,7 @@ sha256_ni_x1: movdqu MSG, [INP + 0*16] pshufb MSG, SHUF_MASK movdqa MSGTMP0, MSG - paddd MSG, [SHA256CONSTANTS + 0*16] + paddd MSG, [rel K256 + 0*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument pshufd MSG, MSG, 0x0E sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument @@ -215,7 +211,7 @@ sha256_ni_x1: movdqu MSG, [INP + 1*16] pshufb MSG, SHUF_MASK movdqa MSGTMP1, MSG - paddd MSG, [SHA256CONSTANTS + 1*16] + paddd MSG, [rel K256 + 1*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument pshufd MSG, MSG, 0x0E sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument @@ -225,7 +221,7 @@ sha256_ni_x1: movdqu MSG, [INP + 2*16] pshufb MSG, SHUF_MASK movdqa MSGTMP2, MSG - paddd MSG, [SHA256CONSTANTS + 2*16] + paddd MSG, [rel K256 + 2*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument pshufd MSG, MSG, 0x0E sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument @@ -235,7 +231,7 @@ sha256_ni_x1: movdqu MSG, [INP + 3*16] pshufb MSG, SHUF_MASK movdqa MSGTMP3, MSG - paddd MSG, [SHA256CONSTANTS + 3*16] + paddd MSG, [rel K256 + 3*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP3 palignr MSGTMP, MSGTMP2, 4 @@ -247,7 +243,7 @@ sha256_ni_x1: ;; Rounds 16-19 movdqa MSG, MSGTMP0 - paddd MSG, [SHA256CONSTANTS + 4*16] + paddd MSG, [rel K256 + 4*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP0 palignr MSGTMP, MSGTMP3, 4 @@ -259,7 +255,7 @@ sha256_ni_x1: ;; Rounds 20-23 movdqa MSG, MSGTMP1 - paddd MSG, [SHA256CONSTANTS + 5*16] + paddd MSG, [rel K256 + 5*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP1 palignr MSGTMP, MSGTMP0, 4 @@ -271,7 +267,7 @@ sha256_ni_x1: ;; Rounds 24-27 movdqa MSG, MSGTMP2 - paddd MSG, [SHA256CONSTANTS + 6*16] + paddd MSG, [rel K256 + 6*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP2 palignr MSGTMP, MSGTMP1, 4 @@ -283,7 +279,7 @@ sha256_ni_x1: ;; Rounds 28-31 movdqa MSG, MSGTMP3 - paddd MSG, [SHA256CONSTANTS + 7*16] + paddd MSG, [rel K256 + 7*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP3 palignr MSGTMP, MSGTMP2, 4 @@ -295,7 +291,7 @@ sha256_ni_x1: ;; Rounds 32-35 movdqa MSG, MSGTMP0 - paddd MSG, [SHA256CONSTANTS + 8*16] + paddd MSG, [rel K256 + 8*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP0 palignr MSGTMP, MSGTMP3, 4 @@ -307,7 +303,7 @@ sha256_ni_x1: ;; Rounds 36-39 movdqa MSG, MSGTMP1 - paddd MSG, [SHA256CONSTANTS + 9*16] + paddd MSG, [rel K256 + 9*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP1 palignr MSGTMP, MSGTMP0, 4 @@ -319,7 +315,7 @@ sha256_ni_x1: ;; Rounds 40-43 movdqa MSG, MSGTMP2 - paddd MSG, [SHA256CONSTANTS + 10*16] + paddd MSG, [rel K256 + 10*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP2 palignr MSGTMP, MSGTMP1, 4 @@ -331,7 +327,7 @@ sha256_ni_x1: ;; Rounds 44-47 movdqa MSG, MSGTMP3 - paddd MSG, [SHA256CONSTANTS + 11*16] + paddd MSG, [rel K256 + 11*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP3 palignr MSGTMP, MSGTMP2, 4 @@ -343,7 +339,7 @@ sha256_ni_x1: ;; Rounds 48-51 movdqa MSG, MSGTMP0 - paddd MSG, [SHA256CONSTANTS + 12*16] + paddd MSG, [rel K256 + 12*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP0 palignr MSGTMP, MSGTMP3, 4 @@ -355,7 +351,7 @@ sha256_ni_x1: ;; Rounds 52-55 movdqa MSG, MSGTMP1 - paddd MSG, [SHA256CONSTANTS + 13*16] + paddd MSG, [rel K256 + 13*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP1 palignr MSGTMP, MSGTMP0, 4 @@ -366,7 +362,7 @@ sha256_ni_x1: ;; Rounds 56-59 movdqa MSG, MSGTMP2 - paddd MSG, [SHA256CONSTANTS + 14*16] + paddd MSG, [rel K256 + 14*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument movdqa MSGTMP, MSGTMP2 palignr MSGTMP, MSGTMP1, 4 @@ -377,7 +373,7 @@ sha256_ni_x1: ;; Rounds 60-63 movdqa MSG, MSGTMP3 - paddd MSG, [SHA256CONSTANTS + 15*16] + paddd MSG, [rel K256 + 15*16] sha256rnds2 STATE1, STATE0, MSG ; MSG is implicit argument pshufd MSG, MSG, 0x0E sha256rnds2 STATE0, STATE1, MSG ; MSG is implicit argument @@ -389,8 +385,6 @@ sha256_ni_x1: cmp INP, NUM_BLKS jne .loop0 - ;; update data pointers - mov [args + _data_ptr_sha256 + lane*PTR_SZ], INP ; Reorder for writeback pshufd STATE0, STATE0, 0x1B ; FEBA @@ -404,6 +398,9 @@ sha256_ni_x1: movdqu [args + lane + 1*16], STATE1 shr lane, 5 + ;; update data pointers + mov [args + _data_ptr_sha256 + lane*PTR_SZ], INP + done_hash: ;; Clear stack frame (4*16 bytes) -- GitLab From 1a53aba52ba4648ce71d6e823a1b234bf5381dda Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Mon, 12 Sep 2022 14:17:32 +0000 Subject: [PATCH 340/369] lib: add flag to enable/disable use of GFNI ISA Use --gfni-off to disable the use of GFNI instructions, so implementations using GFNI instructions won't be used even if GFNI is supported. --- lib/intel-ipsec-mb.h | 5 +++++ lib/x86_64/alloc.c | 2 ++ lib/x86_64/cpu_feature.c | 3 +++ perf/ipsec_perf.c | 6 ++++++ test/ipsec_xvalid.c | 2 ++ test/main.c | 2 ++ test/utils.c | 4 ++++ 7 files changed, 24 insertions(+) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index db7694e9..7b81fb4c 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -999,6 +999,7 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_FLAG_SHANI_OFF (1ULL << 0) /**< disable use of SHANI extension */ #define IMB_FLAG_AESNI_OFF (1ULL << 1) /**< disable use of AESNI extension */ +#define IMB_FLAG_GFNI_OFF (1ULL << 2) /**< disable use of GFNI extension */ /** * Multi-buffer manager detected features @@ -1305,6 +1306,8 @@ IMB_DLL_EXPORT const char *imb_get_strerror(int errnum); * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) of + * Galois Field extensions. * * @return Pointer to allocated memory for IMB_MGR structure * @retval NULL on allocation error @@ -1341,6 +1344,8 @@ IMB_DLL_EXPORT size_t imb_get_mb_mgr_size(void); * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) + * of Galois Field extensions. * * @param [in] reset_mgr if 0, IMB_MGR structure is not cleared, else it is. * diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 592810c4..191d4ee8 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -165,6 +165,7 @@ static void set_ooo_mgr_road_block(IMB_MGR *mgr) * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions. * * @param reset_mgr if 0, IMB_MGR structure is not cleared, else it is. * @@ -277,6 +278,7 @@ free_mem(void *ptr) * IMB_FLAG_SHANI_OFF - disable use (and detection) of SHA extensions, * currently SHANI is only available for SSE * IMB_FLAG_AESNI_OFF - disable use (and detection) of AES extensions. + * IMB_FLAG_GFNI_OFF - disable use (and detection) of Galois Field extensions. * * @return Pointer to allocated memory for MB_MGR structure * @retval NULL on allocation error diff --git a/lib/x86_64/cpu_feature.c b/lib/x86_64/cpu_feature.c index 185a9dd0..5248853b 100644 --- a/lib/x86_64/cpu_feature.c +++ b/lib/x86_64/cpu_feature.c @@ -250,6 +250,9 @@ uint64_t cpu_feature_adjust(const uint64_t flags, uint64_t features) if (flags & IMB_FLAG_AESNI_OFF) features &= ~IMB_FEATURE_AESNI; + if (flags & IMB_FLAG_GFNI_OFF) + features &= ~IMB_FEATURE_GFNI; + return features; } diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 7c27f24f..847d7a92 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -3260,6 +3260,8 @@ static void usage(void) "-o val: Use for the SHA size increment, default is 24\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n" + "--gfni-on: use Galois Field extensions, default: auto-detect\n" + "--gfni-off: don't use Galois Field extensions\n" "--force-job-api: use JOB API" " (direct API used for GCM/GHASH/CHACHA20_POLY1305 API by default)\n" "--gcm-sgl-api: use direct SGL API for GCM perf tests" @@ -3729,6 +3731,10 @@ int main(int argc, char *argv[]) flags &= (~IMB_FLAG_SHANI_OFF); } else if (strcmp(argv[i], "--shani-off") == 0) { flags |= IMB_FLAG_SHANI_OFF; + } else if (strcmp(argv[i], "--gfni-on") == 0) { + flags &= (~IMB_FLAG_GFNI_OFF); + } else if (strcmp(argv[i], "--gfni-off") == 0) { + flags |= IMB_FLAG_GFNI_OFF; } else if (strcmp(argv[i], "--force-job-api") == 0) { use_job_api = 1; } else if (strcmp(argv[i], "--gcm-sgl-api") == 0) { diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index c05d07fa..ef870c28 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -2707,6 +2707,8 @@ static void usage(const char *app_name) "--aesni-emu: Do AESNI_EMU (disabled by default)\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n" + "--gfni-on: use Galois Field extensions, default: auto-detect\n" + "--gfni-off: don't use Galois Field extensions\n" "--cipher-iv-size: size of cipher IV.\n" "--auth-iv-size: size of authentication IV.\n" "--tag-size: size of authentication tag\n" diff --git a/test/main.c b/test/main.c index 5ada502a..8ff63103 100644 --- a/test/main.c +++ b/test/main.c @@ -257,6 +257,8 @@ usage(const char *name) "--auto-detect: auto detects current architecture " "to run the tests\n Note: Auto detection " "option now run by default and will be removed in the future\n" + "--gfni-on: use Galois Field extensions, default: auto-detect\n" + "--gfni-off: don't use Galois Field extensions\n" "--shani-on: use SHA extensions, default: auto-detect\n" "--shani-off: don't use SHA extensions\n", name); } diff --git a/test/utils.c b/test/utils.c index 4c48d26c..95e6a383 100644 --- a/test/utils.c +++ b/test/utils.c @@ -189,6 +189,10 @@ update_flags_and_archs(const char *arg, *flags &= (~IMB_FLAG_SHANI_OFF); else if (strcmp(arg, "--shani-off") == 0) *flags |= IMB_FLAG_SHANI_OFF; + else if (strcmp(arg, "--gfni-on") == 0) + *flags &= (~IMB_FLAG_GFNI_OFF); + else if (strcmp(arg, "--gfni-off") == 0) + *flags |= IMB_FLAG_GFNI_OFF; else match = 0; return match; -- GitLab From 6864c83b9a4b3b0aa041ddd356c0c1989b61c0c7 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 12 Sep 2022 13:25:20 +0000 Subject: [PATCH 341/369] avx: [MB_MGR] change current avx code to avx type-1 --- lib/Makefile | 19 +- lib/avx2_t1/mb_mgr_avx2_t1.c | 1 - lib/{avx => avx_t1}/README | 0 .../aes128_cbc_dec_by8_avx.asm | 0 lib/{avx => avx_t1}/aes128_cbc_enc_x8_avx.asm | 0 lib/{avx => avx_t1}/aes128_cbc_mac_x8_avx.asm | 2 +- .../aes128_cbcs_1_9_dec_by8_avx.asm | 2 +- .../aes128_cbcs_1_9_enc_x8_avx.asm | 2 +- lib/{avx => avx_t1}/aes128_cntr_by8_avx.asm | 0 .../aes128_cntr_ccm_by8_avx.asm | 2 +- lib/{avx => avx_t1}/aes128_ecb_by8_avx.asm | 0 lib/{avx => avx_t1}/aes128_gcm_by8_avx.asm | 2 +- .../aes128_xcbc_mac_x8_avx.asm | 2 +- .../aes192_cbc_dec_by8_avx.asm | 0 lib/{avx => avx_t1}/aes192_cbc_enc_x8_avx.asm | 0 lib/{avx => avx_t1}/aes192_cntr_by8_avx.asm | 0 lib/{avx => avx_t1}/aes192_ecb_by8_avx.asm | 2 +- lib/{avx => avx_t1}/aes192_gcm_by8_avx.asm | 2 +- .../aes256_cbc_dec_by8_avx.asm | 0 lib/{avx => avx_t1}/aes256_cbc_enc_x8_avx.asm | 0 lib/{avx => avx_t1}/aes256_cbc_mac_x8_avx.asm | 2 +- lib/{avx => avx_t1}/aes256_cntr_by8_avx.asm | 0 .../aes256_cntr_ccm_by8_avx.asm | 2 +- lib/{avx => avx_t1}/aes256_ecb_by8_avx.asm | 2 +- lib/{avx => avx_t1}/aes256_gcm_by8_avx.asm | 2 +- lib/{avx => avx_t1}/aes_cfb_avx.asm | 0 lib/{avx => avx_t1}/chacha20_avx.asm | 0 lib/{avx => avx_t1}/crc16_x25_avx.asm | 0 lib/{avx => avx_t1}/crc32_by8_avx.asm | 0 lib/{avx => avx_t1}/crc32_fp_avx.asm | 0 lib/{avx => avx_t1}/crc32_iuup_avx.asm | 0 lib/{avx => avx_t1}/crc32_lte_avx.asm | 0 lib/{avx => avx_t1}/crc32_refl_by8_avx.asm | 0 lib/{avx => avx_t1}/crc32_sctp_avx.asm | 0 lib/{avx => avx_t1}/crc32_wimax_avx.asm | 0 lib/{avx => avx_t1}/ethernet_fcs_avx.asm | 0 lib/{avx => avx_t1}/gcm_avx_gen2.asm | 0 lib/{avx => avx_t1}/kasumi_avx.c | 0 .../mb_mgr_aes128_cbc_enc_flush_avx.asm | 0 .../mb_mgr_aes128_cbc_enc_submit_avx.asm | 0 .../mb_mgr_aes128_cbcs_1_9_flush_avx.asm | 0 .../mb_mgr_aes128_cbcs_1_9_submit_avx.asm | 0 ...gr_aes128_ccm_auth_submit_flush_x8_avx.asm | 0 ...mb_mgr_aes128_cmac_submit_flush_x8_avx.asm | 0 .../mb_mgr_aes128_xcbc_flush_x8_avx.asm | 0 .../mb_mgr_aes128_xcbc_submit_x8_avx.asm | 0 .../mb_mgr_aes192_cbc_enc_flush_avx.asm | 2 +- .../mb_mgr_aes192_cbc_enc_submit_avx.asm | 2 +- .../mb_mgr_aes256_cbc_enc_flush_avx.asm | 2 +- .../mb_mgr_aes256_cbc_enc_submit_avx.asm | 2 +- ...gr_aes256_ccm_auth_submit_flush_x8_avx.asm | 2 +- ...mb_mgr_aes256_cmac_submit_flush_x8_avx.asm | 2 +- lib/avx_t1/mb_mgr_avx.c | 96 ++++++ .../mb_mgr_avx.c => avx_t1/mb_mgr_avx_t1.c} | 313 ++++++++---------- .../mb_mgr_hmac_md5_flush_avx.asm | 0 .../mb_mgr_hmac_md5_submit_avx.asm | 0 .../mb_mgr_hmac_sha1_flush_avx.asm | 0 .../mb_mgr_hmac_sha1_submit_avx.asm | 0 .../mb_mgr_hmac_sha224_flush_avx.asm | 2 +- .../mb_mgr_hmac_sha224_submit_avx.asm | 2 +- .../mb_mgr_hmac_sha256_flush_avx.asm | 0 .../mb_mgr_hmac_sha256_submit_avx.asm | 0 .../mb_mgr_hmac_sha384_flush_avx.asm | 2 +- .../mb_mgr_hmac_sha384_submit_avx.asm | 2 +- .../mb_mgr_hmac_sha512_flush_avx.asm | 0 .../mb_mgr_hmac_sha512_submit_avx.asm | 0 .../mb_mgr_zuc_submit_flush_avx.asm | 0 lib/{avx => avx_t1}/md5_x4x2_avx.asm | 0 lib/{avx => avx_t1}/memcpy_avx.asm | 0 lib/{avx => avx_t1}/pon_by8_avx.asm | 0 lib/{avx => avx_t1}/sha1_one_block_avx.asm | 0 lib/{avx => avx_t1}/sha1_x4_avx.asm | 0 lib/{avx => avx_t1}/sha224_one_block_avx.asm | 2 +- lib/{avx => avx_t1}/sha256_mult_avx.asm | 0 lib/{avx => avx_t1}/sha256_one_block_avx.asm | 0 lib/{avx => avx_t1}/sha384_one_block_avx.asm | 2 +- lib/{avx => avx_t1}/sha512_one_block_avx.asm | 0 lib/{avx => avx_t1}/sha512_x2_avx.asm | 0 lib/{avx => avx_t1}/sha_avx.c | 0 lib/{avx => avx_t1}/sha_mb_avx.c | 0 lib/{avx => avx_t1}/snow3g_avx.c | 0 lib/{avx => avx_t1}/snow3g_uia2_by4_avx.asm | 0 lib/{avx => avx_t1}/snow_v_avx.asm | 0 lib/{avx => avx_t1}/zuc_top_avx.c | 0 lib/{avx => avx_t1}/zuc_x4_avx.asm | 0 lib/include/ipsec_ooo_mgr.h | 83 ++--- lib/intel-ipsec-mb.h | 2 + lib/libIPSec_MB.def | 96 +++--- lib/win_x64.mak | 15 +- 89 files changed, 381 insertions(+), 292 deletions(-) rename lib/{avx => avx_t1}/README (100%) rename lib/{avx => avx_t1}/aes128_cbc_dec_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes128_cbc_enc_x8_avx.asm (100%) rename lib/{avx => avx_t1}/aes128_cbc_mac_x8_avx.asm (97%) rename lib/{avx => avx_t1}/aes128_cbcs_1_9_dec_by8_avx.asm (97%) rename lib/{avx => avx_t1}/aes128_cbcs_1_9_enc_x8_avx.asm (98%) rename lib/{avx => avx_t1}/aes128_cntr_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes128_cntr_ccm_by8_avx.asm (97%) rename lib/{avx => avx_t1}/aes128_ecb_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes128_gcm_by8_avx.asm (98%) rename lib/{avx => avx_t1}/aes128_xcbc_mac_x8_avx.asm (97%) rename lib/{avx => avx_t1}/aes192_cbc_dec_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes192_cbc_enc_x8_avx.asm (100%) rename lib/{avx => avx_t1}/aes192_cntr_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes192_ecb_by8_avx.asm (97%) rename lib/{avx => avx_t1}/aes192_gcm_by8_avx.asm (98%) rename lib/{avx => avx_t1}/aes256_cbc_dec_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes256_cbc_enc_x8_avx.asm (100%) rename lib/{avx => avx_t1}/aes256_cbc_mac_x8_avx.asm (97%) rename lib/{avx => avx_t1}/aes256_cntr_by8_avx.asm (100%) rename lib/{avx => avx_t1}/aes256_cntr_ccm_by8_avx.asm (97%) rename lib/{avx => avx_t1}/aes256_ecb_by8_avx.asm (97%) rename lib/{avx => avx_t1}/aes256_gcm_by8_avx.asm (98%) rename lib/{avx => avx_t1}/aes_cfb_avx.asm (100%) rename lib/{avx => avx_t1}/chacha20_avx.asm (100%) rename lib/{avx => avx_t1}/crc16_x25_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_by8_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_fp_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_iuup_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_lte_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_refl_by8_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_sctp_avx.asm (100%) rename lib/{avx => avx_t1}/crc32_wimax_avx.asm (100%) rename lib/{avx => avx_t1}/ethernet_fcs_avx.asm (100%) rename lib/{avx => avx_t1}/gcm_avx_gen2.asm (100%) rename lib/{avx => avx_t1}/kasumi_avx.c (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_cbc_enc_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_cbc_enc_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_cbcs_1_9_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_cbcs_1_9_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_xcbc_flush_x8_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes128_xcbc_submit_x8_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_aes192_cbc_enc_flush_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_aes192_cbc_enc_submit_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_aes256_cbc_enc_flush_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_aes256_cbc_enc_submit_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm (96%) create mode 100644 lib/avx_t1/mb_mgr_avx.c rename lib/{avx/mb_mgr_avx.c => avx_t1/mb_mgr_avx_t1.c} (86%) rename lib/{avx => avx_t1}/mb_mgr_hmac_md5_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_md5_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha1_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha1_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha224_flush_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha224_submit_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha256_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha256_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha384_flush_avx.asm (97%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha384_submit_avx.asm (96%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha512_flush_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_hmac_sha512_submit_avx.asm (100%) rename lib/{avx => avx_t1}/mb_mgr_zuc_submit_flush_avx.asm (100%) rename lib/{avx => avx_t1}/md5_x4x2_avx.asm (100%) rename lib/{avx => avx_t1}/memcpy_avx.asm (100%) rename lib/{avx => avx_t1}/pon_by8_avx.asm (100%) rename lib/{avx => avx_t1}/sha1_one_block_avx.asm (100%) rename lib/{avx => avx_t1}/sha1_x4_avx.asm (100%) rename lib/{avx => avx_t1}/sha224_one_block_avx.asm (97%) rename lib/{avx => avx_t1}/sha256_mult_avx.asm (100%) rename lib/{avx => avx_t1}/sha256_one_block_avx.asm (100%) rename lib/{avx => avx_t1}/sha384_one_block_avx.asm (97%) rename lib/{avx => avx_t1}/sha512_one_block_avx.asm (100%) rename lib/{avx => avx_t1}/sha512_x2_avx.asm (100%) rename lib/{avx => avx_t1}/sha_avx.c (100%) rename lib/{avx => avx_t1}/sha_mb_avx.c (100%) rename lib/{avx => avx_t1}/snow3g_avx.c (100%) rename lib/{avx => avx_t1}/snow3g_uia2_by4_avx.asm (100%) rename lib/{avx => avx_t1}/snow_v_avx.asm (100%) rename lib/{avx => avx_t1}/zuc_top_avx.c (100%) rename lib/{avx => avx_t1}/zuc_x4_avx.asm (100%) diff --git a/lib/Makefile b/lib/Makefile index 838d6a1c..bdd94bba 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -247,6 +247,7 @@ SAFE_OPTIONS_MSG2="All safe options enabled by default." # c_lib_objs := \ mb_mgr_avx.o \ + mb_mgr_avx_t1.o \ mb_mgr_avx2.o \ mb_mgr_avx2_t1.o \ mb_mgr_avx2_t2.o \ @@ -848,10 +849,24 @@ ifeq ($(CC_HAS_CET),1) mv $@.tmp $@ endif -$(OBJ_DIR)/%.o:avx/%.c +$(OBJ_DIR)/%.o:avx_t1/%.c $(CC) -MMD $(OPT_AVX) -c $(CFLAGS) $< -o $@ -$(OBJ_DIR)/%.o:avx/%.asm +$(OBJ_DIR)/%.o:avx_t1/%.asm +ifeq ($(USE_YASM),y) + $(YASM) $(YASM_FLAGS) $< -o $@ +else + $(NASM) -MD $(@:.o=.d) -MT $@ -o $@ $(NASM_FLAGS) $< +endif +ifeq ($(CC_HAS_CET),1) + $(LD) $(CET_LDFLAGS) -o $@.tmp $@ + mv $@.tmp $@ +endif + +$(OBJ_DIR)/%.o:avx_t2/%.c + $(CC) -MMD $(OPT_AVX) -c $(CFLAGS) $< -o $@ + +$(OBJ_DIR)/%.o:avx_t2/%.asm ifeq ($(USE_YASM),y) $(YASM) $(YASM_FLAGS) $< -o $@ else diff --git a/lib/avx2_t1/mb_mgr_avx2_t1.c b/lib/avx2_t1/mb_mgr_avx2_t1.c index 6467975d..b4575cb0 100644 --- a/lib/avx2_t1/mb_mgr_avx2_t1.c +++ b/lib/avx2_t1/mb_mgr_avx2_t1.c @@ -310,7 +310,6 @@ init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs) state->earliest_job = -1; } - /* set handlers */ /* set handlers */ state->get_next_job = GET_NEXT_JOB; state->submit_job = SUBMIT_JOB; diff --git a/lib/avx/README b/lib/avx_t1/README similarity index 100% rename from lib/avx/README rename to lib/avx_t1/README diff --git a/lib/avx/aes128_cbc_dec_by8_avx.asm b/lib/avx_t1/aes128_cbc_dec_by8_avx.asm similarity index 100% rename from lib/avx/aes128_cbc_dec_by8_avx.asm rename to lib/avx_t1/aes128_cbc_dec_by8_avx.asm diff --git a/lib/avx/aes128_cbc_enc_x8_avx.asm b/lib/avx_t1/aes128_cbc_enc_x8_avx.asm similarity index 100% rename from lib/avx/aes128_cbc_enc_x8_avx.asm rename to lib/avx_t1/aes128_cbc_enc_x8_avx.asm diff --git a/lib/avx/aes128_cbc_mac_x8_avx.asm b/lib/avx_t1/aes128_cbc_mac_x8_avx.asm similarity index 97% rename from lib/avx/aes128_cbc_mac_x8_avx.asm rename to lib/avx_t1/aes128_cbc_mac_x8_avx.asm index ea6ffb8e..870da05e 100644 --- a/lib/avx/aes128_cbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes128_cbc_mac_x8_avx.asm @@ -34,4 +34,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "avx/aes128_cbc_enc_x8_avx.asm" +%include "avx_t1/aes128_cbc_enc_x8_avx.asm" diff --git a/lib/avx/aes128_cbcs_1_9_dec_by8_avx.asm b/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm similarity index 97% rename from lib/avx/aes128_cbcs_1_9_dec_by8_avx.asm rename to lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm index d9e6a3c1..b0982c2f 100644 --- a/lib/avx/aes128_cbcs_1_9_dec_by8_avx.asm +++ b/lib/avx_t1/aes128_cbcs_1_9_dec_by8_avx.asm @@ -33,4 +33,4 @@ %define CBCS %endif -%include "avx/aes128_cbc_dec_by8_avx.asm" +%include "avx_t1/aes128_cbc_dec_by8_avx.asm" diff --git a/lib/avx/aes128_cbcs_1_9_enc_x8_avx.asm b/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm similarity index 98% rename from lib/avx/aes128_cbcs_1_9_enc_x8_avx.asm rename to lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm index a2d891b9..d3051f2f 100644 --- a/lib/avx/aes128_cbcs_1_9_enc_x8_avx.asm +++ b/lib/avx_t1/aes128_cbcs_1_9_enc_x8_avx.asm @@ -49,4 +49,4 @@ %define ARG_KEYS _aesarg_keys %define ARG_IV _aesarg_IV -%include "avx/aes128_cbc_enc_x8_avx.asm" +%include "avx_t1/aes128_cbc_enc_x8_avx.asm" diff --git a/lib/avx/aes128_cntr_by8_avx.asm b/lib/avx_t1/aes128_cntr_by8_avx.asm similarity index 100% rename from lib/avx/aes128_cntr_by8_avx.asm rename to lib/avx_t1/aes128_cntr_by8_avx.asm diff --git a/lib/avx/aes128_cntr_ccm_by8_avx.asm b/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm similarity index 97% rename from lib/avx/aes128_cntr_ccm_by8_avx.asm rename to lib/avx_t1/aes128_cntr_ccm_by8_avx.asm index 485f5923..37bb556c 100644 --- a/lib/avx/aes128_cntr_ccm_by8_avx.asm +++ b/lib/avx_t1/aes128_cntr_ccm_by8_avx.asm @@ -29,4 +29,4 @@ %ifndef AES_CNTR_CCM_128 %define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx %endif -%include "avx/aes128_cntr_by8_avx.asm" +%include "avx_t1/aes128_cntr_by8_avx.asm" diff --git a/lib/avx/aes128_ecb_by8_avx.asm b/lib/avx_t1/aes128_ecb_by8_avx.asm similarity index 100% rename from lib/avx/aes128_ecb_by8_avx.asm rename to lib/avx_t1/aes128_ecb_by8_avx.asm diff --git a/lib/avx/aes128_gcm_by8_avx.asm b/lib/avx_t1/aes128_gcm_by8_avx.asm similarity index 98% rename from lib/avx/aes128_gcm_by8_avx.asm rename to lib/avx_t1/aes128_gcm_by8_avx.asm index 2aa61e78..dba11298 100644 --- a/lib/avx/aes128_gcm_by8_avx.asm +++ b/lib/avx_t1/aes128_gcm_by8_avx.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM128_MODE 1 -%include "avx/gcm_avx_gen2.asm" +%include "avx_t1/gcm_avx_gen2.asm" diff --git a/lib/avx/aes128_xcbc_mac_x8_avx.asm b/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm similarity index 97% rename from lib/avx/aes128_xcbc_mac_x8_avx.asm rename to lib/avx_t1/aes128_xcbc_mac_x8_avx.asm index facf70ae..9d9ec414 100644 --- a/lib/avx/aes128_xcbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes128_xcbc_mac_x8_avx.asm @@ -34,4 +34,4 @@ %define ARG_KEYS _aesxcbcarg_keys %define ARG_IV _aesxcbcarg_ICV -%include "avx/aes128_cbc_enc_x8_avx.asm" +%include "avx_t1/aes128_cbc_enc_x8_avx.asm" diff --git a/lib/avx/aes192_cbc_dec_by8_avx.asm b/lib/avx_t1/aes192_cbc_dec_by8_avx.asm similarity index 100% rename from lib/avx/aes192_cbc_dec_by8_avx.asm rename to lib/avx_t1/aes192_cbc_dec_by8_avx.asm diff --git a/lib/avx/aes192_cbc_enc_x8_avx.asm b/lib/avx_t1/aes192_cbc_enc_x8_avx.asm similarity index 100% rename from lib/avx/aes192_cbc_enc_x8_avx.asm rename to lib/avx_t1/aes192_cbc_enc_x8_avx.asm diff --git a/lib/avx/aes192_cntr_by8_avx.asm b/lib/avx_t1/aes192_cntr_by8_avx.asm similarity index 100% rename from lib/avx/aes192_cntr_by8_avx.asm rename to lib/avx_t1/aes192_cntr_by8_avx.asm diff --git a/lib/avx/aes192_ecb_by8_avx.asm b/lib/avx_t1/aes192_ecb_by8_avx.asm similarity index 97% rename from lib/avx/aes192_ecb_by8_avx.asm rename to lib/avx_t1/aes192_ecb_by8_avx.asm index 873538fe..b7402bad 100644 --- a/lib/avx/aes192_ecb_by8_avx.asm +++ b/lib/avx_t1/aes192_ecb_by8_avx.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 12 %include "include/os.asm" -%include "avx/aes128_ecb_by8_avx.asm" +%include "avx_t1/aes128_ecb_by8_avx.asm" diff --git a/lib/avx/aes192_gcm_by8_avx.asm b/lib/avx_t1/aes192_gcm_by8_avx.asm similarity index 98% rename from lib/avx/aes192_gcm_by8_avx.asm rename to lib/avx_t1/aes192_gcm_by8_avx.asm index 8592e82b..e072d850 100644 --- a/lib/avx/aes192_gcm_by8_avx.asm +++ b/lib/avx_t1/aes192_gcm_by8_avx.asm @@ -28,4 +28,4 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM192_MODE 1 -%include "avx/gcm_avx_gen2.asm" +%include "avx_t1/gcm_avx_gen2.asm" diff --git a/lib/avx/aes256_cbc_dec_by8_avx.asm b/lib/avx_t1/aes256_cbc_dec_by8_avx.asm similarity index 100% rename from lib/avx/aes256_cbc_dec_by8_avx.asm rename to lib/avx_t1/aes256_cbc_dec_by8_avx.asm diff --git a/lib/avx/aes256_cbc_enc_x8_avx.asm b/lib/avx_t1/aes256_cbc_enc_x8_avx.asm similarity index 100% rename from lib/avx/aes256_cbc_enc_x8_avx.asm rename to lib/avx_t1/aes256_cbc_enc_x8_avx.asm diff --git a/lib/avx/aes256_cbc_mac_x8_avx.asm b/lib/avx_t1/aes256_cbc_mac_x8_avx.asm similarity index 97% rename from lib/avx/aes256_cbc_mac_x8_avx.asm rename to lib/avx_t1/aes256_cbc_mac_x8_avx.asm index 45674e34..842eea9d 100644 --- a/lib/avx/aes256_cbc_mac_x8_avx.asm +++ b/lib/avx_t1/aes256_cbc_mac_x8_avx.asm @@ -28,4 +28,4 @@ ;;; Routine to compute CBC-MAC. It is based on 256 bit CBC AES encrypt code. %define CBC_MAC 1 -%include "avx/aes256_cbc_enc_x8_avx.asm" +%include "avx_t1/aes256_cbc_enc_x8_avx.asm" diff --git a/lib/avx/aes256_cntr_by8_avx.asm b/lib/avx_t1/aes256_cntr_by8_avx.asm similarity index 100% rename from lib/avx/aes256_cntr_by8_avx.asm rename to lib/avx_t1/aes256_cntr_by8_avx.asm diff --git a/lib/avx/aes256_cntr_ccm_by8_avx.asm b/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm similarity index 97% rename from lib/avx/aes256_cntr_ccm_by8_avx.asm rename to lib/avx_t1/aes256_cntr_ccm_by8_avx.asm index 9279757a..d51bab55 100644 --- a/lib/avx/aes256_cntr_ccm_by8_avx.asm +++ b/lib/avx_t1/aes256_cntr_ccm_by8_avx.asm @@ -29,4 +29,4 @@ %ifndef AES_CNTR_CCM_256 %define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx %endif -%include "avx/aes256_cntr_by8_avx.asm" +%include "avx_t1/aes256_cntr_by8_avx.asm" diff --git a/lib/avx/aes256_ecb_by8_avx.asm b/lib/avx_t1/aes256_ecb_by8_avx.asm similarity index 97% rename from lib/avx/aes256_ecb_by8_avx.asm rename to lib/avx_t1/aes256_ecb_by8_avx.asm index 44423419..514ffa89 100644 --- a/lib/avx/aes256_ecb_by8_avx.asm +++ b/lib/avx_t1/aes256_ecb_by8_avx.asm @@ -30,4 +30,4 @@ %define AES_ECB_NROUNDS 14 %include "include/os.asm" -%include "avx/aes128_ecb_by8_avx.asm" +%include "avx_t1/aes128_ecb_by8_avx.asm" diff --git a/lib/avx/aes256_gcm_by8_avx.asm b/lib/avx_t1/aes256_gcm_by8_avx.asm similarity index 98% rename from lib/avx/aes256_gcm_by8_avx.asm rename to lib/avx_t1/aes256_gcm_by8_avx.asm index 3584c9b4..9d5146e9 100644 --- a/lib/avx/aes256_gcm_by8_avx.asm +++ b/lib/avx_t1/aes256_gcm_by8_avx.asm @@ -27,4 +27,4 @@ ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; %define GCM256_MODE 1 -%include "avx/gcm_avx_gen2.asm" +%include "avx_t1/gcm_avx_gen2.asm" diff --git a/lib/avx/aes_cfb_avx.asm b/lib/avx_t1/aes_cfb_avx.asm similarity index 100% rename from lib/avx/aes_cfb_avx.asm rename to lib/avx_t1/aes_cfb_avx.asm diff --git a/lib/avx/chacha20_avx.asm b/lib/avx_t1/chacha20_avx.asm similarity index 100% rename from lib/avx/chacha20_avx.asm rename to lib/avx_t1/chacha20_avx.asm diff --git a/lib/avx/crc16_x25_avx.asm b/lib/avx_t1/crc16_x25_avx.asm similarity index 100% rename from lib/avx/crc16_x25_avx.asm rename to lib/avx_t1/crc16_x25_avx.asm diff --git a/lib/avx/crc32_by8_avx.asm b/lib/avx_t1/crc32_by8_avx.asm similarity index 100% rename from lib/avx/crc32_by8_avx.asm rename to lib/avx_t1/crc32_by8_avx.asm diff --git a/lib/avx/crc32_fp_avx.asm b/lib/avx_t1/crc32_fp_avx.asm similarity index 100% rename from lib/avx/crc32_fp_avx.asm rename to lib/avx_t1/crc32_fp_avx.asm diff --git a/lib/avx/crc32_iuup_avx.asm b/lib/avx_t1/crc32_iuup_avx.asm similarity index 100% rename from lib/avx/crc32_iuup_avx.asm rename to lib/avx_t1/crc32_iuup_avx.asm diff --git a/lib/avx/crc32_lte_avx.asm b/lib/avx_t1/crc32_lte_avx.asm similarity index 100% rename from lib/avx/crc32_lte_avx.asm rename to lib/avx_t1/crc32_lte_avx.asm diff --git a/lib/avx/crc32_refl_by8_avx.asm b/lib/avx_t1/crc32_refl_by8_avx.asm similarity index 100% rename from lib/avx/crc32_refl_by8_avx.asm rename to lib/avx_t1/crc32_refl_by8_avx.asm diff --git a/lib/avx/crc32_sctp_avx.asm b/lib/avx_t1/crc32_sctp_avx.asm similarity index 100% rename from lib/avx/crc32_sctp_avx.asm rename to lib/avx_t1/crc32_sctp_avx.asm diff --git a/lib/avx/crc32_wimax_avx.asm b/lib/avx_t1/crc32_wimax_avx.asm similarity index 100% rename from lib/avx/crc32_wimax_avx.asm rename to lib/avx_t1/crc32_wimax_avx.asm diff --git a/lib/avx/ethernet_fcs_avx.asm b/lib/avx_t1/ethernet_fcs_avx.asm similarity index 100% rename from lib/avx/ethernet_fcs_avx.asm rename to lib/avx_t1/ethernet_fcs_avx.asm diff --git a/lib/avx/gcm_avx_gen2.asm b/lib/avx_t1/gcm_avx_gen2.asm similarity index 100% rename from lib/avx/gcm_avx_gen2.asm rename to lib/avx_t1/gcm_avx_gen2.asm diff --git a/lib/avx/kasumi_avx.c b/lib/avx_t1/kasumi_avx.c similarity index 100% rename from lib/avx/kasumi_avx.c rename to lib/avx_t1/kasumi_avx.c diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_cbc_enc_flush_avx.asm rename to lib/avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm diff --git a/lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_cbc_enc_submit_avx.asm rename to lib/avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm diff --git a/lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_cbcs_1_9_flush_avx.asm rename to lib/avx_t1/mb_mgr_aes128_cbcs_1_9_flush_avx.asm diff --git a/lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm b/lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_cbcs_1_9_submit_avx.asm rename to lib/avx_t1/mb_mgr_aes128_cbcs_1_9_submit_avx.asm diff --git a/lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm diff --git a/lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm diff --git a/lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_xcbc_flush_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes128_xcbc_flush_x8_avx.asm diff --git a/lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm similarity index 100% rename from lib/avx/mb_mgr_aes128_xcbc_submit_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes128_xcbc_submit_x8_avx.asm diff --git a/lib/avx/mb_mgr_aes192_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes192_cbc_enc_flush_avx.asm rename to lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm index 70d092f1..13797d30 100644 --- a/lib/avx/mb_mgr_aes192_cbc_enc_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes192_cbc_enc_flush_avx.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X8 aes_cbc_enc_192_x8 %define FLUSH_JOB_AES_ENC flush_job_aes192_enc_avx -%include "avx/mb_mgr_aes128_cbc_enc_flush_avx.asm" +%include "avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm" diff --git a/lib/avx/mb_mgr_aes192_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes192_cbc_enc_submit_avx.asm rename to lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm index 9089b116..1dd9ce6c 100644 --- a/lib/avx/mb_mgr_aes192_cbc_enc_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes192_cbc_enc_submit_avx.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X8 aes_cbc_enc_192_x8 %define SUBMIT_JOB_AES_ENC submit_job_aes192_enc_avx -%include "avx/mb_mgr_aes128_cbc_enc_submit_avx.asm" +%include "avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm" diff --git a/lib/avx/mb_mgr_aes256_cbc_enc_flush_avx.asm b/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes256_cbc_enc_flush_avx.asm rename to lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm index 68c108ba..75e85042 100644 --- a/lib/avx/mb_mgr_aes256_cbc_enc_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cbc_enc_flush_avx.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X8 aes_cbc_enc_256_x8 %define FLUSH_JOB_AES_ENC flush_job_aes256_enc_avx -%include "avx/mb_mgr_aes128_cbc_enc_flush_avx.asm" +%include "avx_t1/mb_mgr_aes128_cbc_enc_flush_avx.asm" diff --git a/lib/avx/mb_mgr_aes256_cbc_enc_submit_avx.asm b/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes256_cbc_enc_submit_avx.asm rename to lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm index c6a26a8c..8aa20d9d 100644 --- a/lib/avx/mb_mgr_aes256_cbc_enc_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cbc_enc_submit_avx.asm @@ -27,4 +27,4 @@ %define AES_CBC_ENC_X8 aes_cbc_enc_256_x8 %define SUBMIT_JOB_AES_ENC submit_job_aes256_enc_avx -%include "avx/mb_mgr_aes128_cbc_enc_submit_avx.asm" +%include "avx_t1/mb_mgr_aes128_cbc_enc_submit_avx.asm" diff --git a/lib/avx/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm index e3063862..186f319f 100644 --- a/lib/avx/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_ccm_auth_submit_flush_x8_avx.asm @@ -32,4 +32,4 @@ %define FLUSH_JOB_AES_CCM_AUTH flush_job_aes256_ccm_auth_avx %endif -%include "avx/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm" +%include "avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm" diff --git a/lib/avx/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm similarity index 96% rename from lib/avx/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm rename to lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm index 29f826ef..62aa60ef 100644 --- a/lib/avx/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes256_cmac_submit_flush_x8_avx.asm @@ -29,4 +29,4 @@ %define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes256_cmac_auth_avx %define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes256_cmac_auth_avx -%include "avx/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm" +%include "avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm" diff --git a/lib/avx_t1/mb_mgr_avx.c b/lib/avx_t1/mb_mgr_avx.c new file mode 100644 index 00000000..5d2b6055 --- /dev/null +++ b/lib/avx_t1/mb_mgr_avx.c @@ -0,0 +1,96 @@ +/******************************************************************************* + Copyright (c) 2012-2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/cpu_feature.h" +#include "include/aesni_emu.h" +#include "include/error.h" +#include "include/arch_x86_64.h" + +IMB_DLL_LOCAL void +init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) +{ +#ifdef SAFE_PARAM + if (state == NULL) { + imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); + return; + } +#endif + + if (!(state->features & IMB_FEATURE_AESNI)) { + fallback_no_aesni(state, 1); + return; + } + + /* reset error status */ + imb_set_errno(state, 0); + + state->features = cpu_feature_adjust(state->flags, + cpu_feature_detect()); + + init_mb_mgr_avx_t1_internal(state, reset_mgrs); +} + +void +init_mb_mgr_avx(IMB_MGR *state) +{ + init_mb_mgr_avx_t1_internal(state, 1); + + if (!self_test(state)) + imb_set_errno(state, IMB_ERR_SELFTEST); +} + +IMB_JOB *submit_job_avx(IMB_MGR *state) +{ + return IMB_SUBMIT_JOB(state); +} + +IMB_JOB *flush_job_avx(IMB_MGR *state) +{ + return IMB_FLUSH_JOB(state); +} + +uint32_t queue_size_avx(IMB_MGR *state) +{ + return IMB_QUEUE_SIZE(state); +} + +IMB_JOB *submit_job_nocheck_avx(IMB_MGR *state) +{ + return IMB_SUBMIT_JOB_NOCHECK(state); +} + +IMB_JOB *get_next_job_avx(IMB_MGR *state) +{ + return IMB_GET_NEXT_JOB(state); +} + +IMB_JOB *get_completed_job_avx(IMB_MGR *state) +{ + return IMB_GET_COMPLETED_JOB(state); +} diff --git a/lib/avx/mb_mgr_avx.c b/lib/avx_t1/mb_mgr_avx_t1.c similarity index 86% rename from lib/avx/mb_mgr_avx.c rename to lib/avx_t1/mb_mgr_avx_t1.c index a9372bdc..cb64f28b 100644 --- a/lib/avx/mb_mgr_avx.c +++ b/lib/avx_t1/mb_mgr_avx_t1.c @@ -1,5 +1,5 @@ /******************************************************************************* - Copyright (c) 2012-2022, Intel Corporation + Copyright (c) 2022, Intel Corporation Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -50,15 +50,70 @@ #define SAVE_XMMS save_xmms_avx #define RESTORE_XMMS restore_xmms_avx +/* JOB API */ +#define SUBMIT_JOB submit_job_avx_t1 +#define FLUSH_JOB flush_job_avx_t1 +#define QUEUE_SIZE queue_size_avx_t1 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx_t1 +#define GET_NEXT_JOB get_next_job_avx_t1 +#define GET_COMPLETED_JOB get_completed_job_avx_t1 +#define GET_NEXT_BURST get_next_burst_avx_t1 +#define SUBMIT_BURST submit_burst_avx_t1 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx_t1 +#define FLUSH_BURST flush_burst_avx_t1 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx_t1 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx_t1 +#define SUBMIT_HASH_BURST submit_hash_burst_avx_t1 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx_t1 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX + +/* AES-GCM */ +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx + +/* AES-CBC */ #define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx #define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx #define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx + #define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx #define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx #define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx + #define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx #define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx #define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ #define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx #define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx #define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx @@ -66,19 +121,14 @@ #define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx #define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx -#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx -#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx -#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx -#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx -#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx -#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx -#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx -#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx - -#define AES_CBC_DEC_128 aes_cbc_dec_128_avx -#define AES_CBC_DEC_192 aes_cbc_dec_192_avx -#define AES_CBC_DEC_256 aes_cbc_dec_256_avx +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx +/* AES-CTR */ #define AES_CTR_128 aes_cntr_128_avx #define AES_CTR_192 aes_cntr_192_avx #define AES_CTR_256 aes_cntr_256_avx @@ -86,34 +136,38 @@ #define AES_CTR_192_BIT aes_cntr_bit_192_avx #define AES_CTR_256_BIT aes_cntr_bit_256_avx +/* AES-CCM */ #define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx #define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx -#define AES_ECB_ENC_128 aes_ecb_enc_128_avx -#define AES_ECB_ENC_192 aes_ecb_enc_192_avx -#define AES_ECB_ENC_256 aes_ecb_enc_256_avx -#define AES_ECB_DEC_128 aes_ecb_dec_128_avx -#define AES_ECB_DEC_192 aes_ecb_dec_192_avx -#define AES_ECB_DEC_256 aes_ecb_dec_256_avx +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx -#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx -#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx -#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx -#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx -#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 -#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 -#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 -#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 -#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 -#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx -#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx -#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx +#define AES_CFB_256_ONE aes_cfb_256_one_avx + +/* AES-XCBC */ #define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx #define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx #define FLUSH_JOB_SHA1 flush_job_sha1_avx #define SUBMIT_JOB_SHA224 submit_job_sha224_avx @@ -125,25 +179,7 @@ #define SUBMIT_JOB_SHA512 submit_job_sha512_avx #define FLUSH_JOB_SHA512 flush_job_sha512_avx -#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx -#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx -#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx -#define QUEUE_SIZE queue_size_avx - -#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX -#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX -#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX -#define FLUSH_JOB_CIPHER_DEC FLUSH_JOB_CIPHER_DEC_AVX - -#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx - -#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx -#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx -#define POLY1305_MAC poly1305_mac_scalar - -#define SUBMIT_JOB_SNOW_V snow_v_avx -#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx - +/* HMAC-SHA1/224/256/384/512 */ #define SUBMIT_JOB_HMAC submit_job_hmac_avx #define FLUSH_JOB_HMAC flush_job_hmac_avx #define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx @@ -157,65 +193,29 @@ #define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx #define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx -/* ====================================================================== */ - -#define SUBMIT_JOB submit_job_avx -#define FLUSH_JOB flush_job_avx -#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx -#define GET_NEXT_JOB get_next_job_avx -#define GET_COMPLETED_JOB get_completed_job_avx -#define GET_NEXT_BURST get_next_burst_avx -#define SUBMIT_BURST submit_burst_avx -#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx -#define FLUSH_BURST flush_burst_avx -#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx -#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx -#define SUBMIT_HASH_BURST submit_hash_burst_avx -#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx - -/* ====================================================================== */ - - -#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX -#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX - -/* ====================================================================== */ - -#define AES_CFB_128_ONE aes_cfb_128_one_avx -#define AES_CFB_256_ONE aes_cfb_256_one_avx - - -#define AES128_CBC_MAC aes128_cbc_mac_x8 - -#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx -#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx - -#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx -#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx - -#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx -#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx - -#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx -#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx +#define POLY1305_MAC poly1305_mac_scalar -/* ====================================================================== */ +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx -uint32_t -ethernet_fcs_avx_local(const void *msg, const uint64_t len, - const void *tag_ouput); +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx +/* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local -/* ====================================================================== */ - -#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx -#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx -#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx -#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx - -/* ====================================================================== */ - static void reset_ooo_mgrs(IMB_MGR *state) { /* Init AES out-of-order fields */ @@ -290,26 +290,8 @@ static void reset_ooo_mgrs(IMB_MGR *state) } IMB_DLL_LOCAL void -init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) +init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs) { -#ifdef SAFE_PARAM - if (state == NULL) { - imb_set_errno(NULL, IMB_ERR_NULL_MBMGR); - return; - } -#endif - - /* reset error status */ - imb_set_errno(state, 0); - - state->features = cpu_feature_adjust(state->flags, - cpu_feature_detect()); - - if (!(state->features & IMB_FEATURE_AESNI)) { - fallback_no_aesni(state, reset_mgrs); - return; - } - /* Check if CPU flags needed for AVX interface are present */ if ((state->features & IMB_CPUFLAGS_AVX) != IMB_CPUFLAGS_AVX) { imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); @@ -327,28 +309,32 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->earliest_job = -1; } - /* set AVX handlers */ - state->get_next_job = get_next_job_avx; - state->submit_job = submit_job_avx; - state->get_next_burst = get_next_burst_avx; - state->submit_burst = submit_burst_avx; - state->submit_burst_nocheck= submit_burst_nocheck_avx; - state->flush_burst = flush_burst_avx; - state->submit_cipher_burst = submit_cipher_burst_avx; - state->submit_cipher_burst_nocheck = submit_cipher_burst_nocheck_avx; - state->submit_hash_burst = submit_hash_burst_avx; - state->submit_hash_burst_nocheck = submit_hash_burst_nocheck_avx; - state->submit_job_nocheck = submit_job_nocheck_avx; - state->get_completed_job = get_completed_job_avx; - state->flush_job = flush_job_avx; - state->queue_size = queue_size_avx; + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + state->keyexp_128 = aes_keyexp_128_avx; state->keyexp_192 = aes_keyexp_192_avx; state->keyexp_256 = aes_keyexp_256_avx; + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx; state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx; + state->xcbc_keyexp = aes_xcbc_expand_key_avx; state->des_key_sched = des_key_schedule; + state->sha1_one_block = sha1_one_block_avx; state->sha1 = sha1_avx; state->sha224_one_block = sha224_one_block_avx; @@ -360,6 +346,7 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->sha512_one_block = sha512_one_block_avx; state->sha512 = sha512_avx; state->md5_one_block = md5_one_block_avx; + state->aes128_cfb_one = aes_cfb_128_one_avx; state->eea3_1_buffer = zuc_eea3_1_buffer_avx; @@ -392,6 +379,27 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->snow3g_init_key_sched = snow3g_init_key_sched_avx; state->snow3g_key_sched_size = snow3g_key_sched_size_avx; + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + + state->chacha20_poly1305_init = init_chacha20_poly1305_avx; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; + state->gcm128_enc = aes_gcm_enc_128_avx_gen2; state->gcm192_enc = aes_gcm_enc_192_avx_gen2; state->gcm256_enc = aes_gcm_enc_256_avx_gen2; @@ -422,6 +430,7 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->gcm128_pre = aes_gcm_pre_128_avx_gen2; state->gcm192_pre = aes_gcm_pre_192_avx_gen2; state->gcm256_pre = aes_gcm_pre_256_avx_gen2; + state->ghash = ghash_avx_gen2; state->ghash_pre = ghash_pre_avx_gen2; @@ -434,34 +443,6 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2; state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2; state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2; - state->hec_32 = hec_32_avx; - state->hec_64 = hec_64_avx; - state->crc32_ethernet_fcs = ethernet_fcs_avx; - state->crc16_x25 = crc16_x25_avx; - state->crc32_sctp = crc32_sctp_avx; - state->crc24_lte_a = crc24_lte_a_avx; - state->crc24_lte_b = crc24_lte_b_avx; - state->crc16_fp_data = crc16_fp_data_avx; - state->crc11_fp_header = crc11_fp_header_avx; - state->crc7_fp_header = crc7_fp_header_avx; - state->crc10_iuup_data = crc10_iuup_data_avx; - state->crc6_iuup_header = crc6_iuup_header_avx; - state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; - state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; - - state->chacha20_poly1305_init = init_chacha20_poly1305_avx; - state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx; - state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; - state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; -} - -void -init_mb_mgr_avx(IMB_MGR *state) -{ - init_mb_mgr_avx_internal(state, 1); - - if (!self_test(state)) - imb_set_errno(state, IMB_ERR_SELFTEST); } #include "mb_mgr_code.h" diff --git a/lib/avx/mb_mgr_hmac_md5_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_md5_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_md5_flush_avx.asm diff --git a/lib/avx/mb_mgr_hmac_md5_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_md5_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_md5_submit_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha1_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha1_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha1_flush_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha1_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha1_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha1_submit_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha224_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm similarity index 96% rename from lib/avx/mb_mgr_hmac_sha224_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm index 2f130535..1efdcea1 100644 --- a/lib/avx/mb_mgr_hmac_sha224_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha224_flush_avx.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_224_avx %define SHA224 -%include "avx/mb_mgr_hmac_sha256_flush_avx.asm" +%include "avx_t1/mb_mgr_hmac_sha256_flush_avx.asm" diff --git a/lib/avx/mb_mgr_hmac_sha224_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm similarity index 96% rename from lib/avx/mb_mgr_hmac_sha224_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm index 2895957a..d26a0ab6 100644 --- a/lib/avx/mb_mgr_hmac_sha224_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha224_submit_avx.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_224_avx %define SHA224 -%include "avx/mb_mgr_hmac_sha256_submit_avx.asm" +%include "avx_t1/mb_mgr_hmac_sha256_submit_avx.asm" diff --git a/lib/avx/mb_mgr_hmac_sha256_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha256_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha256_flush_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha256_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha256_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha256_submit_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha384_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm similarity index 97% rename from lib/avx/mb_mgr_hmac_sha384_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm index cdf2e836..9ee3c124 100644 --- a/lib/avx/mb_mgr_hmac_sha384_flush_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha384_flush_avx.asm @@ -28,4 +28,4 @@ %define FUNC flush_job_hmac_sha_384_avx %define SHA_X_DIGEST_SIZE 384 -%include "avx/mb_mgr_hmac_sha512_flush_avx.asm" +%include "avx_t1/mb_mgr_hmac_sha512_flush_avx.asm" diff --git a/lib/avx/mb_mgr_hmac_sha384_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm similarity index 96% rename from lib/avx/mb_mgr_hmac_sha384_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm index fa8dc5b1..70a90ba8 100644 --- a/lib/avx/mb_mgr_hmac_sha384_submit_avx.asm +++ b/lib/avx_t1/mb_mgr_hmac_sha384_submit_avx.asm @@ -28,4 +28,4 @@ %define FUNC submit_job_hmac_sha_384_avx %define SHA_X_DIGEST_SIZE 384 -%include "avx/mb_mgr_hmac_sha512_submit_avx.asm" +%include "avx_t1/mb_mgr_hmac_sha512_submit_avx.asm" diff --git a/lib/avx/mb_mgr_hmac_sha512_flush_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha512_flush_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha512_flush_avx.asm diff --git a/lib/avx/mb_mgr_hmac_sha512_submit_avx.asm b/lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm similarity index 100% rename from lib/avx/mb_mgr_hmac_sha512_submit_avx.asm rename to lib/avx_t1/mb_mgr_hmac_sha512_submit_avx.asm diff --git a/lib/avx/mb_mgr_zuc_submit_flush_avx.asm b/lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm similarity index 100% rename from lib/avx/mb_mgr_zuc_submit_flush_avx.asm rename to lib/avx_t1/mb_mgr_zuc_submit_flush_avx.asm diff --git a/lib/avx/md5_x4x2_avx.asm b/lib/avx_t1/md5_x4x2_avx.asm similarity index 100% rename from lib/avx/md5_x4x2_avx.asm rename to lib/avx_t1/md5_x4x2_avx.asm diff --git a/lib/avx/memcpy_avx.asm b/lib/avx_t1/memcpy_avx.asm similarity index 100% rename from lib/avx/memcpy_avx.asm rename to lib/avx_t1/memcpy_avx.asm diff --git a/lib/avx/pon_by8_avx.asm b/lib/avx_t1/pon_by8_avx.asm similarity index 100% rename from lib/avx/pon_by8_avx.asm rename to lib/avx_t1/pon_by8_avx.asm diff --git a/lib/avx/sha1_one_block_avx.asm b/lib/avx_t1/sha1_one_block_avx.asm similarity index 100% rename from lib/avx/sha1_one_block_avx.asm rename to lib/avx_t1/sha1_one_block_avx.asm diff --git a/lib/avx/sha1_x4_avx.asm b/lib/avx_t1/sha1_x4_avx.asm similarity index 100% rename from lib/avx/sha1_x4_avx.asm rename to lib/avx_t1/sha1_x4_avx.asm diff --git a/lib/avx/sha224_one_block_avx.asm b/lib/avx_t1/sha224_one_block_avx.asm similarity index 97% rename from lib/avx/sha224_one_block_avx.asm rename to lib/avx_t1/sha224_one_block_avx.asm index 19213a70..f80b5579 100644 --- a/lib/avx/sha224_one_block_avx.asm +++ b/lib/avx_t1/sha224_one_block_avx.asm @@ -30,4 +30,4 @@ %define FUNC sha224_block_avx -%include "avx/sha256_one_block_avx.asm" +%include "avx_t1/sha256_one_block_avx.asm" diff --git a/lib/avx/sha256_mult_avx.asm b/lib/avx_t1/sha256_mult_avx.asm similarity index 100% rename from lib/avx/sha256_mult_avx.asm rename to lib/avx_t1/sha256_mult_avx.asm diff --git a/lib/avx/sha256_one_block_avx.asm b/lib/avx_t1/sha256_one_block_avx.asm similarity index 100% rename from lib/avx/sha256_one_block_avx.asm rename to lib/avx_t1/sha256_one_block_avx.asm diff --git a/lib/avx/sha384_one_block_avx.asm b/lib/avx_t1/sha384_one_block_avx.asm similarity index 97% rename from lib/avx/sha384_one_block_avx.asm rename to lib/avx_t1/sha384_one_block_avx.asm index b0230308..fc9bf735 100644 --- a/lib/avx/sha384_one_block_avx.asm +++ b/lib/avx_t1/sha384_one_block_avx.asm @@ -30,4 +30,4 @@ %define FUNC sha384_block_avx -%include "avx/sha512_one_block_avx.asm" +%include "avx_t1/sha512_one_block_avx.asm" diff --git a/lib/avx/sha512_one_block_avx.asm b/lib/avx_t1/sha512_one_block_avx.asm similarity index 100% rename from lib/avx/sha512_one_block_avx.asm rename to lib/avx_t1/sha512_one_block_avx.asm diff --git a/lib/avx/sha512_x2_avx.asm b/lib/avx_t1/sha512_x2_avx.asm similarity index 100% rename from lib/avx/sha512_x2_avx.asm rename to lib/avx_t1/sha512_x2_avx.asm diff --git a/lib/avx/sha_avx.c b/lib/avx_t1/sha_avx.c similarity index 100% rename from lib/avx/sha_avx.c rename to lib/avx_t1/sha_avx.c diff --git a/lib/avx/sha_mb_avx.c b/lib/avx_t1/sha_mb_avx.c similarity index 100% rename from lib/avx/sha_mb_avx.c rename to lib/avx_t1/sha_mb_avx.c diff --git a/lib/avx/snow3g_avx.c b/lib/avx_t1/snow3g_avx.c similarity index 100% rename from lib/avx/snow3g_avx.c rename to lib/avx_t1/snow3g_avx.c diff --git a/lib/avx/snow3g_uia2_by4_avx.asm b/lib/avx_t1/snow3g_uia2_by4_avx.asm similarity index 100% rename from lib/avx/snow3g_uia2_by4_avx.asm rename to lib/avx_t1/snow3g_uia2_by4_avx.asm diff --git a/lib/avx/snow_v_avx.asm b/lib/avx_t1/snow_v_avx.asm similarity index 100% rename from lib/avx/snow_v_avx.asm rename to lib/avx_t1/snow_v_avx.asm diff --git a/lib/avx/zuc_top_avx.c b/lib/avx_t1/zuc_top_avx.c similarity index 100% rename from lib/avx/zuc_top_avx.c rename to lib/avx_t1/zuc_top_avx.c diff --git a/lib/avx/zuc_x4_avx.asm b/lib/avx_t1/zuc_x4_avx.asm similarity index 100% rename from lib/avx/zuc_x4_avx.asm rename to lib/avx_t1/zuc_x4_avx.asm diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index 72b2aace..cafcfe18 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -424,6 +424,8 @@ init_mb_mgr_sse_t3_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void +init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs); @@ -436,8 +438,6 @@ init_mb_mgr_avx512_t1_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx512_t2_internal(IMB_MGR *state, const int reset_mgrs); -IMB_DLL_EXPORT uint32_t -get_next_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_sse_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -445,7 +445,7 @@ get_next_burst_sse_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -get_next_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +get_next_burst_avx_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -455,8 +455,6 @@ get_next_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_sse(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_sse_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -464,7 +462,7 @@ submit_burst_sse_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_avx(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +submit_burst_avx_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -474,9 +472,6 @@ submit_burst_avx512_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); -IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_sse(IMB_MGR *state, const uint32_t n_jobs, - IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_nocheck_sse_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -487,8 +482,8 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -submit_burst_nocheck_avx(IMB_MGR *state, const uint32_t n_jobs, - IMB_JOB **jobs); +submit_burst_nocheck_avx_t1(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -502,8 +497,6 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx512_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); -IMB_DLL_EXPORT uint32_t -flush_burst_sse(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_sse_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -511,7 +504,7 @@ flush_burst_sse_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_sse_t3(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t -flush_burst_avx(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +flush_burst_avx_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx2_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -521,12 +514,6 @@ flush_burst_avx512_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx512_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); -IMB_DLL_EXPORT uint32_t -submit_cipher_burst_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_sse_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -546,11 +533,11 @@ submit_cipher_burst_sse_t3(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_avx_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -577,12 +564,6 @@ submit_cipher_burst_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_sse_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -601,11 +582,11 @@ submit_cipher_burst_nocheck_sse_t3(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t -submit_cipher_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_CIPHER_MODE cipher, - const IMB_CIPHER_DIRECTION dir, - const IMB_KEY_SIZE_BYTES key_size); +submit_cipher_burst_nocheck_avx_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -630,10 +611,7 @@ submit_cipher_burst_nocheck_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_MODE cipher, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); -IMB_DLL_EXPORT uint32_t -submit_hash_burst_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); + IMB_DLL_EXPORT uint32_t submit_hash_burst_sse_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -647,9 +625,9 @@ submit_hash_burst_sse_t3(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_avx_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t submit_hash_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -666,10 +644,7 @@ IMB_DLL_EXPORT uint32_t submit_hash_burst_avx512_t2(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); -IMB_DLL_EXPORT uint32_t -submit_hash_burst_nocheck_sse(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); + IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_sse_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -683,9 +658,9 @@ submit_hash_burst_nocheck_sse_t3(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t -submit_hash_burst_nocheck_avx(IMB_MGR *state, IMB_JOB *jobs, - const uint32_t n_jobs, - const IMB_HASH_ALG hash); +submit_hash_burst_nocheck_avx_t1(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, @@ -727,6 +702,14 @@ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_sse_t3(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_sse_t3(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_sse_t3(IMB_MGR *state); +/* AVX TYPE1 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx_t1(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx_t1(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx_t1(IMB_MGR *state); + /* AVX2 TYPE1 manager functions */ IMB_DLL_EXPORT IMB_JOB *submit_job_avx2_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *flush_job_avx2_t1(IMB_MGR *state); diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 7b81fb4c..c55b33b8 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1050,6 +1050,8 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_CPUFLAGS_AVX2_T2 (IMB_CPUFLAGS_AVX2 | IMB_FEATURE_SHANI | \ IMB_FEATURE_VAES | IMB_FEATURE_VPCLMULQDQ | \ IMB_FEATURE_GFNI) +#define IMB_CPUFLAGS_AVX_T2 (IMB_CPUFLAGS_AVX | IMB_FEATURE_SHANI | \ + IMB_FEATURE_GFNI) /* TOP LEVEL (IMB_MGR) Data structure fields */ diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index dfbee2f4..d84127e2 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -548,7 +548,7 @@ EXPORTS get_next_burst_sse_t2 @522 get_next_burst_sse_t3 @523 get_next_burst_sse_no_aesni @524 - get_next_burst_avx @525 + get_next_burst_avx_t1 @525 get_next_burst_avx2_t1 @526 get_next_burst_avx2_t2 @527 get_next_burst_avx512_t1 @528 @@ -557,7 +557,7 @@ EXPORTS submit_burst_sse_t2 @531 submit_burst_sse_t3 @532 submit_burst_sse_no_aesni @533 - submit_burst_avx @534 + submit_burst_avx_t1 @534 submit_burst_avx2_t1 @535 submit_burst_avx2_t2 @536 submit_burst_avx512_t1 @537 @@ -575,7 +575,7 @@ EXPORTS flush_burst_sse_t2 @549 flush_burst_sse_t3 @550 flush_burst_sse_no_aesni @551 - flush_burst_avx @552 + flush_burst_avx_t1 @552 flush_burst_avx2_t1 @553 flush_burst_avx2_t2 @554 flush_burst_avx512_t1 @555 @@ -584,7 +584,7 @@ EXPORTS submit_cipher_burst_sse_t2 @558 submit_cipher_burst_sse_t3 @559 submit_cipher_burst_sse_no_aesni @560 - submit_cipher_burst_avx @561 + submit_cipher_burst_avx_t1 @561 submit_cipher_burst_avx2_t1 @562 submit_cipher_burst_avx2_t2 @563 submit_cipher_burst_avx512_t1 @564 @@ -602,7 +602,7 @@ EXPORTS submit_hash_burst_sse_t2 @576 submit_hash_burst_sse_t3 @577 submit_hash_burst_sse_no_aesni @578 - submit_hash_burst_avx @579 + submit_hash_burst_avx_t1 @579 submit_hash_burst_avx2_t1 @580 submit_hash_burst_avx2_t2 @581 submit_hash_burst_avx512_t1 @582 @@ -611,7 +611,7 @@ EXPORTS submit_hash_burst_nocheck_sse_t2 @585 submit_hash_burst_nocheck_sse_t3 @586 submit_hash_burst_nocheck_sse_no_aesni @587 - submit_hash_burst_nocheck_avx @588 + submit_hash_burst_nocheck_avx_t1 @588 submit_hash_burst_nocheck_avx2_t1 @589 submit_hash_burst_nocheck_avx2_t2 @590 submit_hash_burst_nocheck_avx512_t1 @591 @@ -619,42 +619,48 @@ EXPORTS flush_job_sse_t1 @593 flush_job_sse_t2 @594 flush_job_sse_t3 @595 - flush_job_avx2_t1 @596 - flush_job_avx2_t2 @597 - flush_job_avx512_t1 @598 - flush_job_avx512_t2 @599 - queue_size_sse_t1 @600 - queue_size_sse_t2 @601 - queue_size_sse_t3 @602 - queue_size_avx2_t1 @603 - queue_size_avx2_t2 @604 - queue_size_avx512_t1 @605 - queue_size_avx512_t2 @606 - submit_job_sse_t1 @607 - submit_job_sse_t2 @608 - submit_job_sse_t3 @609 - submit_job_avx2_t1 @610 - submit_job_avx2_t2 @611 - submit_job_avx512_t1 @612 - submit_job_avx512_t2 @613 - submit_job_nocheck_sse_t1 @614 - submit_job_nocheck_sse_t2 @615 - submit_job_nocheck_sse_t3 @616 - submit_job_nocheck_avx2_t1 @617 - submit_job_nocheck_avx2_t2 @618 - submit_job_nocheck_avx512_t1 @619 - submit_job_nocheck_avx512_t2 @620 - get_next_job_sse_t1 @621 - get_next_job_sse_t2 @622 - get_next_job_sse_t3 @623 - get_next_job_avx2_t1 @624 - get_next_job_avx2_t2 @625 - get_next_job_avx512_t1 @626 - get_next_job_avx512_t2 @627 - get_completed_job_sse_t1 @628 - get_completed_job_sse_t2 @629 - get_completed_job_sse_t3 @630 - get_completed_job_avx2_t1 @631 - get_completed_job_avx2_t2 @632 - get_completed_job_avx512_t1 @633 - get_completed_job_avx512_t2 @634 + flush_job_avx_t1 @596 + flush_job_avx2_t1 @597 + flush_job_avx2_t2 @598 + flush_job_avx512_t1 @599 + flush_job_avx512_t2 @601 + queue_size_sse_t1 @602 + queue_size_sse_t2 @603 + queue_size_sse_t3 @604 + queue_size_avx_t1 @605 + queue_size_avx2_t1 @606 + queue_size_avx2_t2 @607 + queue_size_avx512_t1 @608 + queue_size_avx512_t2 @609 + submit_job_sse_t1 @610 + submit_job_sse_t2 @611 + submit_job_sse_t3 @612 + submit_job_avx_t1 @613 + submit_job_avx2_t1 @614 + submit_job_avx2_t2 @615 + submit_job_avx512_t1 @616 + submit_job_avx512_t2 @617 + submit_job_nocheck_sse_t1 @618 + submit_job_nocheck_sse_t2 @619 + submit_job_nocheck_sse_t3 @620 + submit_job_nocheck_avx_t1 @621 + submit_job_nocheck_avx2_t1 @622 + submit_job_nocheck_avx2_t2 @623 + submit_job_nocheck_avx512_t1 @624 + submit_job_nocheck_avx512_t2 @625 + get_next_job_sse_t1 @626 + get_next_job_sse_t2 @627 + get_next_job_sse_t3 @628 + get_next_job_avx_t1 @629 + get_next_job_avx2_t1 @630 + get_next_job_avx2_t2 @631 + get_next_job_avx512_t1 @632 + get_next_job_avx512_t2 @633 + get_completed_job_sse_t1 @634 + get_completed_job_sse_t2 @635 + get_completed_job_sse_t3 @636 + get_completed_job_avx_t1 @637 + get_completed_job_avx2_t1 @638 + get_completed_job_avx2_t2 @639 + get_completed_job_avx512_t1 @640 + get_completed_job_avx512_t2 @641 diff --git a/lib/win_x64.mak b/lib/win_x64.mak index 1ae61583..c269dd19 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -125,8 +125,7 @@ LINK_TOOL = link LINKFLAGS = $(DLFLAGS) /nologo /machine:X64 AS = nasm -AFLAGS = $(DAFLAGS) -Werror -fwin64 -Xvc -DWIN_ABI -Iinclude/ \ - -I./ -Iavx/ -Iavx2/ -Iavx512/ -Isse/ +AFLAGS = $(DAFLAGS) -Werror -fwin64 -Xvc -DWIN_ABI -I. # dependency !ifndef DEPTOOL @@ -448,6 +447,7 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_avx512.obj \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_avx512.obj \ $(OBJ_DIR)\mb_mgr_avx.obj \ + $(OBJ_DIR)\mb_mgr_avx_t1.obj \ $(OBJ_DIR)\mb_mgr_avx2.obj \ $(OBJ_DIR)\mb_mgr_avx2_t1.obj \ $(OBJ_DIR)\mb_mgr_avx2_t2.obj \ @@ -632,11 +632,18 @@ $(DEPALL): $(all_objs) {sse_t3\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< -{avx\}.c{$(OBJ_DIR)}.obj: +{avx_t1\}.c{$(OBJ_DIR)}.obj: $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep -{avx\}.asm{$(OBJ_DIR)}.obj: +{avx_t1\}.asm{$(OBJ_DIR)}.obj: + $(AS) -MD $@.dep -o $@ $(AFLAGS) $< + +{avx_t2\}.c{$(OBJ_DIR)}.obj: + $(CC) /arch:AVX /Fo$@ /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +{avx_t2\}.asm{$(OBJ_DIR)}.obj: $(AS) -MD $@.dep -o $@ $(AFLAGS) $< {avx2_t1\}.c{$(OBJ_DIR)}.obj: -- GitLab From ceb7f3f49f317cacc6abb3c6b7ae8abb6e231153 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 12 Sep 2022 15:31:16 +0000 Subject: [PATCH 342/369] avx: [MB_MGR] add avx type-2 code path --- lib/Makefile | 1 + lib/avx_t1/mb_mgr_avx.c | 8 +- lib/avx_t2/mb_mgr_avx_t2.c | 448 ++++++++++++++++++++++++++++++++++++ lib/include/ipsec_ooo_mgr.h | 39 ++++ lib/libIPSec_MB.def | 243 ++++++++++--------- lib/win_x64.mak | 1 + 6 files changed, 623 insertions(+), 117 deletions(-) create mode 100644 lib/avx_t2/mb_mgr_avx_t2.c diff --git a/lib/Makefile b/lib/Makefile index bdd94bba..c592114d 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -248,6 +248,7 @@ SAFE_OPTIONS_MSG2="All safe options enabled by default." c_lib_objs := \ mb_mgr_avx.o \ mb_mgr_avx_t1.o \ + mb_mgr_avx_t2.o \ mb_mgr_avx2.o \ mb_mgr_avx2_t1.o \ mb_mgr_avx2_t2.o \ diff --git a/lib/avx_t1/mb_mgr_avx.c b/lib/avx_t1/mb_mgr_avx.c index 5d2b6055..21438ed2 100644 --- a/lib/avx_t1/mb_mgr_avx.c +++ b/lib/avx_t1/mb_mgr_avx.c @@ -53,13 +53,17 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs) state->features = cpu_feature_adjust(state->flags, cpu_feature_detect()); - init_mb_mgr_avx_t1_internal(state, reset_mgrs); + if ((state->features & IMB_CPUFLAGS_AVX_T2) == + IMB_CPUFLAGS_AVX_T2) + init_mb_mgr_avx_t2_internal(state, reset_mgrs); + else + init_mb_mgr_avx_t1_internal(state, reset_mgrs); } void init_mb_mgr_avx(IMB_MGR *state) { - init_mb_mgr_avx_t1_internal(state, 1); + init_mb_mgr_avx_internal(state, 1); if (!self_test(state)) imb_set_errno(state, IMB_ERR_SELFTEST); diff --git a/lib/avx_t2/mb_mgr_avx_t2.c b/lib/avx_t2/mb_mgr_avx_t2.c new file mode 100644 index 00000000..37a3c810 --- /dev/null +++ b/lib/avx_t2/mb_mgr_avx_t2.c @@ -0,0 +1,448 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include + +#define AVX + +#include "intel-ipsec-mb.h" +#include "include/ipsec_ooo_mgr.h" +#include "include/kasumi_interface.h" +#include "include/zuc_internal.h" +#include "include/snow3g.h" +#include "include/gcm.h" +#include "include/chacha20_poly1305.h" +#include "include/save_xmms.h" +#include "include/des.h" +#include "include/cpu_feature.h" +#include "include/aesni_emu.h" +#include "include/error.h" + +#include "include/arch_sse_type2.h" /* shani */ +#include "include/arch_avx_type1.h" + +#include "include/ooo_mgr_reset.h" + +#define SAVE_XMMS save_xmms_avx +#define RESTORE_XMMS restore_xmms_avx + +/* JOB API */ +#define SUBMIT_JOB submit_job_avx_t2 +#define FLUSH_JOB flush_job_avx_t2 +#define QUEUE_SIZE queue_size_avx_t2 +#define SUBMIT_JOB_NOCHECK submit_job_nocheck_avx_t2 +#define GET_NEXT_JOB get_next_job_avx_t2 +#define GET_COMPLETED_JOB get_completed_job_avx_t2 +#define GET_NEXT_BURST get_next_burst_avx_t2 +#define SUBMIT_BURST submit_burst_avx_t2 +#define SUBMIT_BURST_NOCHECK submit_burst_nocheck_avx_t2 +#define FLUSH_BURST flush_burst_avx_t2 +#define SUBMIT_CIPHER_BURST submit_cipher_burst_avx_t2 +#define SUBMIT_CIPHER_BURST_NOCHECK submit_cipher_burst_nocheck_avx_t2 +#define SUBMIT_HASH_BURST submit_hash_burst_avx_t2 +#define SUBMIT_HASH_BURST_NOCHECK submit_hash_burst_nocheck_avx_t2 + +/* Hash */ +#define SUBMIT_JOB_HASH SUBMIT_JOB_HASH_AVX_T2 +#define FLUSH_JOB_HASH FLUSH_JOB_HASH_AVX_T2 + +/* Cipher encrypt / decrypt */ +#define SUBMIT_JOB_CIPHER_ENC SUBMIT_JOB_CIPHER_ENC_AVX_T2 +#define FLUSH_JOB_CIPHER_ENC FLUSH_JOB_CIPHER_ENC_AVX_T2 +#define SUBMIT_JOB_CIPHER_DEC SUBMIT_JOB_CIPHER_DEC_AVX_T2 + +/* AES-GCM */ +#define AES_GCM_DEC_IV_128 aes_gcm_dec_var_iv_128_avx_gen2 +#define AES_GCM_ENC_IV_128 aes_gcm_enc_var_iv_128_avx_gen2 +#define AES_GCM_DEC_IV_192 aes_gcm_dec_var_iv_192_avx_gen2 +#define AES_GCM_ENC_IV_192 aes_gcm_enc_var_iv_192_avx_gen2 +#define AES_GCM_DEC_IV_256 aes_gcm_dec_var_iv_256_avx_gen2 +#define AES_GCM_ENC_IV_256 aes_gcm_enc_var_iv_256_avx_gen2 + +#define SUBMIT_JOB_AES_GCM_DEC submit_job_aes_gcm_dec_avx +#define SUBMIT_JOB_AES_GCM_ENC submit_job_aes_gcm_enc_avx + +/* AES-CBC */ +#define SUBMIT_JOB_AES_CBC_128_ENC submit_job_aes128_enc_avx +#define SUBMIT_JOB_AES_CBC_128_DEC submit_job_aes128_dec_avx +#define FLUSH_JOB_AES_CBC_128_ENC flush_job_aes128_enc_avx + +#define SUBMIT_JOB_AES_CBC_192_ENC submit_job_aes192_enc_avx +#define SUBMIT_JOB_AES_CBC_192_DEC submit_job_aes192_dec_avx +#define FLUSH_JOB_AES_CBC_192_ENC flush_job_aes192_enc_avx + +#define SUBMIT_JOB_AES_CBC_256_ENC submit_job_aes256_enc_avx +#define SUBMIT_JOB_AES_CBC_256_DEC submit_job_aes256_dec_avx +#define FLUSH_JOB_AES_CBC_256_ENC flush_job_aes256_enc_avx + +#define AES_CBC_DEC_128 aes_cbc_dec_128_avx +#define AES_CBC_DEC_192 aes_cbc_dec_192_avx +#define AES_CBC_DEC_256 aes_cbc_dec_256_avx + +#define SUBMIT_JOB_AES128_DEC submit_job_aes128_dec_avx_t2 +#define SUBMIT_JOB_AES192_DEC submit_job_aes192_dec_avx_t2 +#define SUBMIT_JOB_AES256_DEC submit_job_aes256_dec_avx_t2 + +/* AES-CBCS */ +#define SUBMIT_JOB_AES128_CBCS_1_9_ENC submit_job_aes128_cbcs_1_9_enc_avx +#define FLUSH_JOB_AES128_CBCS_1_9_ENC flush_job_aes128_cbcs_1_9_enc_avx +#define SUBMIT_JOB_AES128_CBCS_1_9_DEC submit_job_aes128_cbcs_1_9_dec_avx +#define AES_CBCS_1_9_DEC_128 aes_cbcs_1_9_dec_128_avx + +/* AES-ECB */ +#define SUBMIT_JOB_AES_ECB_128_ENC submit_job_aes_ecb_128_enc_avx_t2 +#define SUBMIT_JOB_AES_ECB_128_DEC submit_job_aes_ecb_128_dec_avx_t2 +#define SUBMIT_JOB_AES_ECB_192_ENC submit_job_aes_ecb_192_enc_avx_t2 +#define SUBMIT_JOB_AES_ECB_192_DEC submit_job_aes_ecb_192_dec_avx_t2 +#define SUBMIT_JOB_AES_ECB_256_ENC submit_job_aes_ecb_256_enc_avx_t2 +#define SUBMIT_JOB_AES_ECB_256_DEC submit_job_aes_ecb_256_dec_avx_t2 + +#define AES_ECB_ENC_128 aes_ecb_enc_128_avx +#define AES_ECB_ENC_192 aes_ecb_enc_192_avx +#define AES_ECB_ENC_256 aes_ecb_enc_256_avx +#define AES_ECB_DEC_128 aes_ecb_dec_128_avx +#define AES_ECB_DEC_192 aes_ecb_dec_192_avx +#define AES_ECB_DEC_256 aes_ecb_dec_256_avx + +/* AES-CTR */ +#define AES_CTR_128 aes_cntr_128_avx +#define AES_CTR_192 aes_cntr_192_avx +#define AES_CTR_256 aes_cntr_256_avx +#define AES_CTR_128_BIT aes_cntr_bit_128_avx +#define AES_CTR_192_BIT aes_cntr_bit_192_avx +#define AES_CTR_256_BIT aes_cntr_bit_256_avx + +/* AES-CCM */ +#define AES_CNTR_CCM_128 aes_cntr_ccm_128_avx +#define AES_CNTR_CCM_256 aes_cntr_ccm_256_avx + +#define FLUSH_JOB_AES128_CCM_AUTH flush_job_aes128_ccm_auth_avx +#define SUBMIT_JOB_AES128_CCM_AUTH submit_job_aes128_ccm_auth_avx + +#define FLUSH_JOB_AES256_CCM_AUTH flush_job_aes256_ccm_auth_avx +#define SUBMIT_JOB_AES256_CCM_AUTH submit_job_aes256_ccm_auth_avx + +/* AES-CMAC */ +#define FLUSH_JOB_AES128_CMAC_AUTH flush_job_aes128_cmac_auth_avx +#define SUBMIT_JOB_AES128_CMAC_AUTH submit_job_aes128_cmac_auth_avx + +#define FLUSH_JOB_AES256_CMAC_AUTH flush_job_aes256_cmac_auth_avx +#define SUBMIT_JOB_AES256_CMAC_AUTH submit_job_aes256_cmac_auth_avx + +/* AES-CFB */ +#define AES_CFB_128_ONE aes_cfb_128_one_avx +#define AES_CFB_256_ONE aes_cfb_256_one_avx + +/* AES-XCBC */ +#define SUBMIT_JOB_AES_XCBC submit_job_aes_xcbc_avx +#define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx + +/* PON */ +#define SUBMIT_JOB_PON_ENC submit_job_pon_enc_avx +#define SUBMIT_JOB_PON_DEC submit_job_pon_dec_avx +#define SUBMIT_JOB_PON_ENC_NO_CTR submit_job_pon_enc_no_ctr_avx +#define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx + +/* SHA1/224/256/384/512 */ +#define SUBMIT_JOB_SHA1 submit_job_sha1_ni_sse +#define FLUSH_JOB_SHA1 flush_job_sha1_ni_sse +#define SUBMIT_JOB_SHA224 submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_ni_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_ni_sse +#define SUBMIT_JOB_SHA384 submit_job_sha384_avx +#define FLUSH_JOB_SHA384 flush_job_sha384_avx +#define SUBMIT_JOB_SHA512 submit_job_sha512_avx +#define FLUSH_JOB_SHA512 flush_job_sha512_avx + +/* HMAC-SHA1/224/256/384/512 */ +#define SUBMIT_JOB_HMAC submit_job_hmac_ni_sse +#define FLUSH_JOB_HMAC flush_job_hmac_ni_sse +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_ni_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_ni_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_ni_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_ni_sse +#define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx +#define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx +#define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx +#define FLUSH_JOB_HMAC_SHA_512 flush_job_hmac_sha_512_avx +#define SUBMIT_JOB_HMAC_MD5 submit_job_hmac_md5_avx +#define FLUSH_JOB_HMAC_MD5 flush_job_hmac_md5_avx + +/* CHACHA20 & POLY1305 */ +#define SUBMIT_JOB_CHACHA20_ENC_DEC submit_job_chacha20_enc_dec_avx +#define SUBMIT_JOB_CHACHA20_POLY1305 aead_chacha20_poly1305_avx +#define SUBMIT_JOB_CHACHA20_POLY1305_SGL aead_chacha20_poly1305_sgl_avx +#define POLY1305_MAC poly1305_mac_scalar + +/* ZUC EEA3 & EIA3 */ +#define SUBMIT_JOB_ZUC_EEA3 submit_job_zuc_eea3_avx +#define FLUSH_JOB_ZUC_EEA3 flush_job_zuc_eea3_avx +#define SUBMIT_JOB_ZUC_EIA3 submit_job_zuc_eia3_avx +#define FLUSH_JOB_ZUC_EIA3 flush_job_zuc_eia3_avx +#define SUBMIT_JOB_ZUC256_EEA3 submit_job_zuc256_eea3_avx +#define FLUSH_JOB_ZUC256_EEA3 flush_job_zuc256_eea3_avx +#define SUBMIT_JOB_ZUC256_EIA3 submit_job_zuc256_eia3_avx +#define FLUSH_JOB_ZUC256_EIA3 flush_job_zuc256_eia3_avx + +/* SNOW-V */ +#define SUBMIT_JOB_SNOW_V snow_v_avx +#define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx + +/* AES-DOCSIS */ +#define ETHERNET_FCS ethernet_fcs_avx_local + +static void reset_ooo_mgrs(IMB_MGR *state) +{ + /* Init AES out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_ooo, 8); + ooo_mgr_aes_reset(state->aes192_ooo, 8); + ooo_mgr_aes_reset(state->aes256_ooo, 8); + + /* DOCSIS SEC BPI (AES CBC + AES CFB for partial block) + * uses same settings as AES CBC. + */ + ooo_mgr_docsis_aes_reset(state->docsis128_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis128_crc32_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_sec_ooo, 8); + ooo_mgr_docsis_aes_reset(state->docsis256_crc32_sec_ooo, 8); + + /* Init ZUC out-of-order fields */ + ooo_mgr_zuc_reset(state->zuc_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc_eia3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eea3_ooo, 4); + ooo_mgr_zuc_reset(state->zuc256_eia3_ooo, 4); + + /* Init HMAC/SHA1 out-of-order fields */ + ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, 2); + + /* Init HMAC/SHA224 out-of-order fields */ + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); + + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); + + /* Init HMAC/SHA384 out-of-order fields */ + ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, + AVX_NUM_SHA512_LANES); + + /* Init HMAC/SHA512 out-of-order fields */ + ooo_mgr_hmac_sha512_reset(state->hmac_sha_512_ooo, + AVX_NUM_SHA512_LANES); + + /* Init HMAC/MD5 out-of-order fields */ + ooo_mgr_hmac_md5_reset(state->hmac_md5_ooo, AVX_NUM_MD5_LANES); + + /* Init AES/XCBC OOO fields */ + ooo_mgr_aes_xcbc_reset(state->aes_xcbc_ooo, 8); + + /* Init AES-CCM auth out-of-order fields */ + ooo_mgr_ccm_reset(state->aes_ccm_ooo, 8); + ooo_mgr_ccm_reset(state->aes256_ccm_ooo, 8); + + /* Init AES-CMAC auth out-of-order fields */ + ooo_mgr_cmac_reset(state->aes_cmac_ooo, 8); + ooo_mgr_cmac_reset(state->aes256_cmac_ooo, 8); + + /* Init AES CBC-S out-of-order fields */ + ooo_mgr_aes_reset(state->aes128_cbcs_ooo, 8); + + /* Init SHA1 out-of-order fields */ + ooo_mgr_sha1_reset(state->sha_1_ooo, 2); + + /* Init SHA224 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); + + /* Init SHA256 out-of-order fields */ + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); + + /* Init SHA384 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_384_ooo, AVX_NUM_SHA512_LANES); + + /* Init SHA512 out-of-order fields */ + ooo_mgr_sha512_reset(state->sha_512_ooo, AVX_NUM_SHA512_LANES); +} + +IMB_DLL_LOCAL void +init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs) +{ + /* Check if CPU flags needed for AVX interface are present */ + if ((state->features & IMB_CPUFLAGS_AVX) != IMB_CPUFLAGS_AVX) { + imb_set_errno(state, IMB_ERR_MISSING_CPUFLAGS_INIT_MGR); + return; + } + + /* Set architecture for future checks */ + state->used_arch = (uint32_t) IMB_ARCH_AVX; + + if (reset_mgrs) { + reset_ooo_mgrs(state); + + /* Init "in order" components */ + state->next_job = 0; + state->earliest_job = -1; + } + + /* set handlers */ + state->get_next_job = GET_NEXT_JOB; + state->submit_job = SUBMIT_JOB; + state->submit_job_nocheck = SUBMIT_JOB_NOCHECK; + state->get_completed_job = GET_COMPLETED_JOB; + state->flush_job = FLUSH_JOB; + state->queue_size = QUEUE_SIZE; + state->get_next_burst = GET_NEXT_BURST; + state->submit_burst = SUBMIT_BURST; + state->submit_burst_nocheck= SUBMIT_BURST_NOCHECK; + state->flush_burst = FLUSH_BURST; + state->submit_cipher_burst = SUBMIT_CIPHER_BURST; + state->submit_cipher_burst_nocheck = SUBMIT_CIPHER_BURST_NOCHECK; + state->submit_hash_burst = SUBMIT_HASH_BURST; + state->submit_hash_burst_nocheck = SUBMIT_HASH_BURST_NOCHECK; + + state->keyexp_128 = aes_keyexp_128_avx; + state->keyexp_192 = aes_keyexp_192_avx; + state->keyexp_256 = aes_keyexp_256_avx; + + state->cmac_subkey_gen_128 = aes_cmac_subkey_gen_avx; + state->cmac_subkey_gen_256 = aes_cmac_256_subkey_gen_avx; + + state->xcbc_keyexp = aes_xcbc_expand_key_avx; + state->des_key_sched = des_key_schedule; + + state->sha1_one_block = sha1_one_block_avx; + state->sha1 = sha1_avx; + state->sha224_one_block = sha224_one_block_avx; + state->sha224 = sha224_avx; + state->sha256_one_block = sha256_one_block_avx; + state->sha256 = sha256_avx; + state->sha384_one_block = sha384_one_block_avx; + state->sha384 = sha384_avx; + state->sha512_one_block = sha512_one_block_avx; + state->sha512 = sha512_avx; + state->md5_one_block = md5_one_block_avx; + + state->aes128_cfb_one = aes_cfb_128_one_avx; + + state->eea3_1_buffer = zuc_eea3_1_buffer_avx; + state->eea3_4_buffer = zuc_eea3_4_buffer_avx; + state->eea3_n_buffer = zuc_eea3_n_buffer_avx; + state->eia3_1_buffer = zuc_eia3_1_buffer_avx; + state->eia3_n_buffer = zuc_eia3_n_buffer_avx; + + state->f8_1_buffer = kasumi_f8_1_buffer_avx; + state->f8_1_buffer_bit = kasumi_f8_1_buffer_bit_avx; + state->f8_2_buffer = kasumi_f8_2_buffer_avx; + state->f8_3_buffer = kasumi_f8_3_buffer_avx; + state->f8_4_buffer = kasumi_f8_4_buffer_avx; + state->f8_n_buffer = kasumi_f8_n_buffer_avx; + state->f9_1_buffer = kasumi_f9_1_buffer_avx; + state->f9_1_buffer_user = kasumi_f9_1_buffer_user_avx; + state->kasumi_init_f8_key_sched = kasumi_init_f8_key_sched_avx; + state->kasumi_init_f9_key_sched = kasumi_init_f9_key_sched_avx; + state->kasumi_key_sched_size = kasumi_key_sched_size_avx; + + state->snow3g_f8_1_buffer_bit = snow3g_f8_1_buffer_bit_avx; + state->snow3g_f8_1_buffer = snow3g_f8_1_buffer_avx; + state->snow3g_f8_2_buffer = snow3g_f8_2_buffer_avx; + state->snow3g_f8_4_buffer = snow3g_f8_4_buffer_avx; + state->snow3g_f8_8_buffer = snow3g_f8_8_buffer_avx; + state->snow3g_f8_n_buffer = snow3g_f8_n_buffer_avx; + state->snow3g_f8_8_buffer_multikey = snow3g_f8_8_buffer_multikey_avx; + state->snow3g_f8_n_buffer_multikey = snow3g_f8_n_buffer_multikey_avx; + state->snow3g_f9_1_buffer = snow3g_f9_1_buffer_avx; + state->snow3g_init_key_sched = snow3g_init_key_sched_avx; + state->snow3g_key_sched_size = snow3g_key_sched_size_avx; + + state->hec_32 = hec_32_avx; + state->hec_64 = hec_64_avx; + + state->crc32_ethernet_fcs = ethernet_fcs_avx; + state->crc16_x25 = crc16_x25_avx; + state->crc32_sctp = crc32_sctp_avx; + state->crc24_lte_a = crc24_lte_a_avx; + state->crc24_lte_b = crc24_lte_b_avx; + state->crc16_fp_data = crc16_fp_data_avx; + state->crc11_fp_header = crc11_fp_header_avx; + state->crc7_fp_header = crc7_fp_header_avx; + state->crc10_iuup_data = crc10_iuup_data_avx; + state->crc6_iuup_header = crc6_iuup_header_avx; + state->crc32_wimax_ofdma_data = crc32_wimax_ofdma_data_avx; + state->crc8_wimax_ofdma_hcs = crc8_wimax_ofdma_hcs_avx; + + state->chacha20_poly1305_init = init_chacha20_poly1305_avx; + state->chacha20_poly1305_enc_update = update_enc_chacha20_poly1305_avx; + state->chacha20_poly1305_dec_update = update_dec_chacha20_poly1305_avx; + state->chacha20_poly1305_finalize = finalize_chacha20_poly1305_avx; + + state->gcm128_enc = aes_gcm_enc_128_avx_gen2; + state->gcm192_enc = aes_gcm_enc_192_avx_gen2; + state->gcm256_enc = aes_gcm_enc_256_avx_gen2; + state->gcm128_dec = aes_gcm_dec_128_avx_gen2; + state->gcm192_dec = aes_gcm_dec_192_avx_gen2; + state->gcm256_dec = aes_gcm_dec_256_avx_gen2; + state->gcm128_init = aes_gcm_init_128_avx_gen2; + state->gcm192_init = aes_gcm_init_192_avx_gen2; + state->gcm256_init = aes_gcm_init_256_avx_gen2; + state->gcm128_init_var_iv = aes_gcm_init_var_iv_128_avx_gen2; + state->gcm192_init_var_iv = aes_gcm_init_var_iv_192_avx_gen2; + state->gcm256_init_var_iv = aes_gcm_init_var_iv_256_avx_gen2; + state->gcm128_enc_update = aes_gcm_enc_128_update_avx_gen2; + state->gcm192_enc_update = aes_gcm_enc_192_update_avx_gen2; + state->gcm256_enc_update = aes_gcm_enc_256_update_avx_gen2; + state->gcm128_dec_update = aes_gcm_dec_128_update_avx_gen2; + state->gcm192_dec_update = aes_gcm_dec_192_update_avx_gen2; + state->gcm256_dec_update = aes_gcm_dec_256_update_avx_gen2; + state->gcm128_enc_finalize = aes_gcm_enc_128_finalize_avx_gen2; + state->gcm192_enc_finalize = aes_gcm_enc_192_finalize_avx_gen2; + state->gcm256_enc_finalize = aes_gcm_enc_256_finalize_avx_gen2; + state->gcm128_dec_finalize = aes_gcm_dec_128_finalize_avx_gen2; + state->gcm192_dec_finalize = aes_gcm_dec_192_finalize_avx_gen2; + state->gcm256_dec_finalize = aes_gcm_dec_256_finalize_avx_gen2; + state->gcm128_precomp = aes_gcm_precomp_128_avx_gen2; + state->gcm192_precomp = aes_gcm_precomp_192_avx_gen2; + state->gcm256_precomp = aes_gcm_precomp_256_avx_gen2; + state->gcm128_pre = aes_gcm_pre_128_avx_gen2; + state->gcm192_pre = aes_gcm_pre_192_avx_gen2; + state->gcm256_pre = aes_gcm_pre_256_avx_gen2; + + state->ghash = ghash_avx_gen2; + state->ghash_pre = ghash_pre_avx_gen2; + + state->gmac128_init = imb_aes_gmac_init_128_avx_gen2; + state->gmac192_init = imb_aes_gmac_init_192_avx_gen2; + state->gmac256_init = imb_aes_gmac_init_256_avx_gen2; + state->gmac128_update = imb_aes_gmac_update_128_avx_gen2; + state->gmac192_update = imb_aes_gmac_update_192_avx_gen2; + state->gmac256_update = imb_aes_gmac_update_256_avx_gen2; + state->gmac128_finalize = imb_aes_gmac_finalize_128_avx_gen2; + state->gmac192_finalize = imb_aes_gmac_finalize_192_avx_gen2; + state->gmac256_finalize = imb_aes_gmac_finalize_256_avx_gen2; +} + +#include "mb_mgr_code.h" diff --git a/lib/include/ipsec_ooo_mgr.h b/lib/include/ipsec_ooo_mgr.h index cafcfe18..1c376482 100644 --- a/lib/include/ipsec_ooo_mgr.h +++ b/lib/include/ipsec_ooo_mgr.h @@ -426,6 +426,8 @@ init_mb_mgr_avx_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx_t1_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void +init_mb_mgr_avx_t2_internal(IMB_MGR *state, const int reset_mgrs); +IMB_DLL_LOCAL void init_mb_mgr_avx2_internal(IMB_MGR *state, const int reset_mgrs); IMB_DLL_LOCAL void init_mb_mgr_avx2_t1_internal(IMB_MGR *state, const int reset_mgrs); @@ -447,6 +449,8 @@ get_next_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +get_next_burst_avx_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t get_next_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t get_next_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -464,6 +468,8 @@ submit_burst_sse_t3(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_avx_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t submit_burst_avx2_t2(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); @@ -485,6 +491,9 @@ IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +submit_burst_nocheck_avx_t2(IMB_MGR *state, const uint32_t n_jobs, + IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t submit_burst_nocheck_avx2_t1(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t @@ -506,6 +515,8 @@ flush_burst_sse_t3(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t +flush_burst_avx_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); +IMB_DLL_EXPORT uint32_t flush_burst_avx2_t1(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); IMB_DLL_EXPORT uint32_t flush_burst_avx2_t2(IMB_MGR *state, const uint32_t max_jobs, IMB_JOB **jobs); @@ -539,6 +550,12 @@ submit_cipher_burst_avx_t1(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_avx_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -588,6 +605,12 @@ submit_cipher_burst_nocheck_avx_t1(IMB_MGR *state, IMB_JOB *jobs, const IMB_CIPHER_DIRECTION dir, const IMB_KEY_SIZE_BYTES key_size); IMB_DLL_EXPORT uint32_t +submit_cipher_burst_nocheck_avx_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_CIPHER_MODE cipher, + const IMB_CIPHER_DIRECTION dir, + const IMB_KEY_SIZE_BYTES key_size); +IMB_DLL_EXPORT uint32_t submit_cipher_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_CIPHER_MODE cipher, @@ -629,6 +652,10 @@ submit_hash_burst_avx_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_avx_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -662,6 +689,10 @@ submit_hash_burst_nocheck_avx_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); IMB_DLL_EXPORT uint32_t +submit_hash_burst_nocheck_avx_t2(IMB_MGR *state, IMB_JOB *jobs, + const uint32_t n_jobs, + const IMB_HASH_ALG hash); +IMB_DLL_EXPORT uint32_t submit_hash_burst_nocheck_avx2_t1(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const IMB_HASH_ALG hash); @@ -710,6 +741,14 @@ IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_next_job_avx_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx_t1(IMB_MGR *state); +/* AVX TYPE2 manager functions */ +IMB_DLL_EXPORT IMB_JOB *submit_job_avx_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *flush_job_avx_t2(IMB_MGR *state); +IMB_DLL_EXPORT uint32_t queue_size_avx_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *submit_job_nocheck_avx_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_next_job_avx_t2(IMB_MGR *state); +IMB_DLL_EXPORT IMB_JOB *get_completed_job_avx_t2(IMB_MGR *state); + /* AVX2 TYPE1 manager functions */ IMB_DLL_EXPORT IMB_JOB *submit_job_avx2_t1(IMB_MGR *state); IMB_DLL_EXPORT IMB_JOB *flush_job_avx2_t1(IMB_MGR *state); diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index d84127e2..b54a787b 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -549,118 +549,131 @@ EXPORTS get_next_burst_sse_t3 @523 get_next_burst_sse_no_aesni @524 get_next_burst_avx_t1 @525 - get_next_burst_avx2_t1 @526 - get_next_burst_avx2_t2 @527 - get_next_burst_avx512_t1 @528 - get_next_burst_avx512_t2 @529 - submit_burst_sse_t1 @530 - submit_burst_sse_t2 @531 - submit_burst_sse_t3 @532 - submit_burst_sse_no_aesni @533 - submit_burst_avx_t1 @534 - submit_burst_avx2_t1 @535 - submit_burst_avx2_t2 @536 - submit_burst_avx512_t1 @537 - submit_burst_avx512_t2 @538 - submit_burst_nocheck_sse_t1 @539 - submit_burst_nocheck_sse_t2 @540 - submit_burst_nocheck_sse_t3 @541 - submit_burst_nocheck_sse_no_aesni @542 - submit_burst_nocheck_avx @543 - submit_burst_nocheck_avx2_t1 @544 - submit_burst_nocheck_avx2_t2 @545 - submit_burst_nocheck_avx512_t1 @546 - submit_burst_nocheck_avx512_t2 @547 - flush_burst_sse_t1 @548 - flush_burst_sse_t2 @549 - flush_burst_sse_t3 @550 - flush_burst_sse_no_aesni @551 - flush_burst_avx_t1 @552 - flush_burst_avx2_t1 @553 - flush_burst_avx2_t2 @554 - flush_burst_avx512_t1 @555 - flush_burst_avx512_t2 @556 - submit_cipher_burst_sse_t1 @557 - submit_cipher_burst_sse_t2 @558 - submit_cipher_burst_sse_t3 @559 - submit_cipher_burst_sse_no_aesni @560 - submit_cipher_burst_avx_t1 @561 - submit_cipher_burst_avx2_t1 @562 - submit_cipher_burst_avx2_t2 @563 - submit_cipher_burst_avx512_t1 @564 - submit_cipher_burst_avx512_t2 @565 - submit_cipher_burst_nocheck_sse_t1 @566 - submit_cipher_burst_nocheck_sse_t2 @567 - submit_cipher_burst_nocheck_sse_t3 @568 - submit_cipher_burst_nocheck_sse_no_aesni @569 - submit_cipher_burst_nocheck_avx @570 - submit_cipher_burst_nocheck_avx2_t1 @571 - submit_cipher_burst_nocheck_avx2_t2 @572 - submit_cipher_burst_nocheck_avx512_t1 @573 - submit_cipher_burst_nocheck_avx512_t2 @574 - submit_hash_burst_sse_t1 @575 - submit_hash_burst_sse_t2 @576 - submit_hash_burst_sse_t3 @577 - submit_hash_burst_sse_no_aesni @578 - submit_hash_burst_avx_t1 @579 - submit_hash_burst_avx2_t1 @580 - submit_hash_burst_avx2_t2 @581 - submit_hash_burst_avx512_t1 @582 - submit_hash_burst_avx512_t2 @583 - submit_hash_burst_nocheck_sse_t1 @584 - submit_hash_burst_nocheck_sse_t2 @585 - submit_hash_burst_nocheck_sse_t3 @586 - submit_hash_burst_nocheck_sse_no_aesni @587 - submit_hash_burst_nocheck_avx_t1 @588 - submit_hash_burst_nocheck_avx2_t1 @589 - submit_hash_burst_nocheck_avx2_t2 @590 - submit_hash_burst_nocheck_avx512_t1 @591 - submit_hash_burst_nocheck_avx512_t2 @592 - flush_job_sse_t1 @593 - flush_job_sse_t2 @594 - flush_job_sse_t3 @595 - flush_job_avx_t1 @596 - flush_job_avx2_t1 @597 - flush_job_avx2_t2 @598 - flush_job_avx512_t1 @599 - flush_job_avx512_t2 @601 - queue_size_sse_t1 @602 - queue_size_sse_t2 @603 - queue_size_sse_t3 @604 - queue_size_avx_t1 @605 - queue_size_avx2_t1 @606 - queue_size_avx2_t2 @607 - queue_size_avx512_t1 @608 - queue_size_avx512_t2 @609 - submit_job_sse_t1 @610 - submit_job_sse_t2 @611 - submit_job_sse_t3 @612 - submit_job_avx_t1 @613 - submit_job_avx2_t1 @614 - submit_job_avx2_t2 @615 - submit_job_avx512_t1 @616 - submit_job_avx512_t2 @617 - submit_job_nocheck_sse_t1 @618 - submit_job_nocheck_sse_t2 @619 - submit_job_nocheck_sse_t3 @620 - submit_job_nocheck_avx_t1 @621 - submit_job_nocheck_avx2_t1 @622 - submit_job_nocheck_avx2_t2 @623 - submit_job_nocheck_avx512_t1 @624 - submit_job_nocheck_avx512_t2 @625 - get_next_job_sse_t1 @626 - get_next_job_sse_t2 @627 - get_next_job_sse_t3 @628 - get_next_job_avx_t1 @629 - get_next_job_avx2_t1 @630 - get_next_job_avx2_t2 @631 - get_next_job_avx512_t1 @632 - get_next_job_avx512_t2 @633 - get_completed_job_sse_t1 @634 - get_completed_job_sse_t2 @635 - get_completed_job_sse_t3 @636 - get_completed_job_avx_t1 @637 - get_completed_job_avx2_t1 @638 - get_completed_job_avx2_t2 @639 - get_completed_job_avx512_t1 @640 - get_completed_job_avx512_t2 @641 + get_next_burst_avx_t2 @526 + get_next_burst_avx2_t1 @527 + get_next_burst_avx2_t2 @528 + get_next_burst_avx512_t1 @529 + get_next_burst_avx512_t2 @530 + submit_burst_sse_t1 @531 + submit_burst_sse_t2 @532 + submit_burst_sse_t3 @533 + submit_burst_sse_no_aesni @534 + submit_burst_avx_t1 @535 + submit_burst_avx_t2 @536 + submit_burst_avx2_t1 @537 + submit_burst_avx2_t2 @538 + submit_burst_avx512_t1 @539 + submit_burst_avx512_t2 @540 + submit_burst_nocheck_sse_t1 @541 + submit_burst_nocheck_sse_t2 @542 + submit_burst_nocheck_sse_t3 @543 + submit_burst_nocheck_sse_no_aesni @544 + submit_burst_nocheck_avx @545 + submit_burst_nocheck_avx2_t1 @546 + submit_burst_nocheck_avx2_t2 @547 + submit_burst_nocheck_avx512_t1 @548 + submit_burst_nocheck_avx512_t2 @549 + flush_burst_sse_t1 @550 + flush_burst_sse_t2 @551 + flush_burst_sse_t3 @552 + flush_burst_sse_no_aesni @553 + flush_burst_avx_t1 @554 + flush_burst_avx_t2 @555 + flush_burst_avx2_t1 @556 + flush_burst_avx2_t2 @557 + flush_burst_avx512_t1 @558 + flush_burst_avx512_t2 @559 + submit_cipher_burst_sse_t1 @560 + submit_cipher_burst_sse_t2 @561 + submit_cipher_burst_sse_t3 @562 + submit_cipher_burst_sse_no_aesni @563 + submit_cipher_burst_avx_t1 @564 + submit_cipher_burst_avx_t2 @565 + submit_cipher_burst_avx2_t1 @566 + submit_cipher_burst_avx2_t2 @567 + submit_cipher_burst_avx512_t1 @568 + submit_cipher_burst_avx512_t2 @569 + submit_cipher_burst_nocheck_sse_t1 @570 + submit_cipher_burst_nocheck_sse_t2 @571 + submit_cipher_burst_nocheck_sse_t3 @572 + submit_cipher_burst_nocheck_sse_no_aesni @573 + submit_cipher_burst_nocheck_avx_t1 @574 + submit_cipher_burst_nocheck_avx_t2 @575 + submit_cipher_burst_nocheck_avx2_t1 @576 + submit_cipher_burst_nocheck_avx2_t2 @577 + submit_cipher_burst_nocheck_avx512_t1 @578 + submit_cipher_burst_nocheck_avx512_t2 @579 + submit_hash_burst_sse_t1 @580 + submit_hash_burst_sse_t2 @581 + submit_hash_burst_sse_t3 @582 + submit_hash_burst_sse_no_aesni @583 + submit_hash_burst_avx_t1 @584 + submit_hash_burst_avx_t2 @585 + submit_hash_burst_avx2_t1 @586 + submit_hash_burst_avx2_t2 @587 + submit_hash_burst_avx512_t1 @588 + submit_hash_burst_avx512_t2 @589 + submit_hash_burst_nocheck_sse_t1 @590 + submit_hash_burst_nocheck_sse_t2 @591 + submit_hash_burst_nocheck_sse_t3 @592 + submit_hash_burst_nocheck_sse_no_aesni @593 + submit_hash_burst_nocheck_avx_t1 @594 + submit_hash_burst_nocheck_avx_t2 @595 + submit_hash_burst_nocheck_avx2_t1 @596 + submit_hash_burst_nocheck_avx2_t2 @597 + submit_hash_burst_nocheck_avx512_t1 @598 + submit_hash_burst_nocheck_avx512_t2 @599 + flush_job_sse_t1 @600 + flush_job_sse_t2 @601 + flush_job_sse_t3 @602 + flush_job_avx_t1 @603 + flush_job_avx_t2 @604 + flush_job_avx2_t1 @605 + flush_job_avx2_t2 @606 + flush_job_avx512_t1 @607 + flush_job_avx512_t2 @608 + queue_size_sse_t1 @609 + queue_size_sse_t2 @610 + queue_size_sse_t3 @611 + queue_size_avx_t1 @612 + queue_size_avx_t2 @613 + queue_size_avx2_t1 @614 + queue_size_avx2_t2 @615 + queue_size_avx512_t1 @616 + queue_size_avx512_t2 @617 + submit_job_sse_t1 @618 + submit_job_sse_t2 @619 + submit_job_sse_t3 @620 + submit_job_avx_t1 @621 + submit_job_avx_t2 @622 + submit_job_avx2_t1 @623 + submit_job_avx2_t2 @624 + submit_job_avx512_t1 @625 + submit_job_avx512_t2 @626 + submit_job_nocheck_sse_t1 @627 + submit_job_nocheck_sse_t2 @628 + submit_job_nocheck_sse_t3 @629 + submit_job_nocheck_avx_t1 @630 + submit_job_nocheck_avx_t2 @631 + submit_job_nocheck_avx2_t1 @632 + submit_job_nocheck_avx2_t2 @633 + submit_job_nocheck_avx512_t1 @634 + submit_job_nocheck_avx512_t2 @635 + get_next_job_sse_t1 @636 + get_next_job_sse_t2 @637 + get_next_job_sse_t3 @638 + get_next_job_avx_t1 @639 + get_next_job_avx_t2 @640 + get_next_job_avx2_t1 @641 + get_next_job_avx2_t2 @642 + get_next_job_avx512_t1 @643 + get_next_job_avx512_t2 @644 + get_completed_job_sse_t1 @645 + get_completed_job_sse_t2 @646 + get_completed_job_sse_t3 @647 + get_completed_job_avx_t1 @648 + get_completed_job_avx_t2 @649 + get_completed_job_avx2_t1 @650 + get_completed_job_avx2_t2 @651 + get_completed_job_avx512_t1 @652 + get_completed_job_avx512_t2 @653 diff --git a/lib/win_x64.mak b/lib/win_x64.mak index c269dd19..b33f5b12 100644 --- a/lib/win_x64.mak +++ b/lib/win_x64.mak @@ -448,6 +448,7 @@ lib_objs2 = \ $(OBJ_DIR)\mb_mgr_zuc_submit_flush_gfni_avx512.obj \ $(OBJ_DIR)\mb_mgr_avx.obj \ $(OBJ_DIR)\mb_mgr_avx_t1.obj \ + $(OBJ_DIR)\mb_mgr_avx_t2.obj \ $(OBJ_DIR)\mb_mgr_avx2.obj \ $(OBJ_DIR)\mb_mgr_avx2_t1.obj \ $(OBJ_DIR)\mb_mgr_avx2_t2.obj \ -- GitLab From 5ab739e11d593e1acfb56e49a3dd6725aaa66ec1 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 12 Sep 2022 15:51:32 +0000 Subject: [PATCH 343/369] avx: [snow3g] use improved SSE implementation for AVX architecture --- lib/avx_t1/mb_mgr_avx_t1.c | 36 +++++- lib/avx_t2/mb_mgr_avx_t2.c | 35 ++++++ lib/libIPSec_MB.def | 219 +++++++++++++++++++------------------ 3 files changed, 180 insertions(+), 110 deletions(-) diff --git a/lib/avx_t1/mb_mgr_avx_t1.c b/lib/avx_t1/mb_mgr_avx_t1.c index cb64f28b..6c51ecd2 100644 --- a/lib/avx_t1/mb_mgr_avx_t1.c +++ b/lib/avx_t1/mb_mgr_avx_t1.c @@ -36,6 +36,7 @@ #include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" +#include "include/snow3g_submit.h" #include "include/gcm.h" #include "include/chacha20_poly1305.h" #include "include/save_xmms.h" @@ -43,7 +44,7 @@ #include "include/cpu_feature.h" #include "include/aesni_emu.h" #include "include/error.h" -#include "include/arch_x86_64.h" /* self-test */ +#include "include/arch_sse_type1.h" /* snow3g */ #include "include/arch_avx_type1.h" #include "include/ooo_mgr_reset.h" @@ -213,6 +214,33 @@ #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_avx_t1(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_avx_t1(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx_t1 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx_t1 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx_t1 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx_t1 + /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local @@ -287,6 +315,12 @@ static void reset_ooo_mgrs(IMB_MGR *state) /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/avx_t2/mb_mgr_avx_t2.c b/lib/avx_t2/mb_mgr_avx_t2.c index 37a3c810..08a63a52 100644 --- a/lib/avx_t2/mb_mgr_avx_t2.c +++ b/lib/avx_t2/mb_mgr_avx_t2.c @@ -36,6 +36,7 @@ #include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" +#include "include/snow3g_submit.h" #include "include/gcm.h" #include "include/chacha20_poly1305.h" #include "include/save_xmms.h" @@ -44,6 +45,7 @@ #include "include/aesni_emu.h" #include "include/error.h" +#include "include/arch_sse_type1.h" /* snow3g */ #include "include/arch_sse_type2.h" /* shani */ #include "include/arch_avx_type1.h" @@ -215,6 +217,33 @@ #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_avx_t2(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_avx_t2(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx_t2 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx_t2 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx_t2 +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx_t2 + /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local @@ -287,6 +316,12 @@ static void reset_ooo_mgrs(IMB_MGR *state) /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/libIPSec_MB.def b/lib/libIPSec_MB.def index b54a787b..903b3368 100644 --- a/lib/libIPSec_MB.def +++ b/lib/libIPSec_MB.def @@ -568,112 +568,113 @@ EXPORTS submit_burst_nocheck_sse_t2 @542 submit_burst_nocheck_sse_t3 @543 submit_burst_nocheck_sse_no_aesni @544 - submit_burst_nocheck_avx @545 - submit_burst_nocheck_avx2_t1 @546 - submit_burst_nocheck_avx2_t2 @547 - submit_burst_nocheck_avx512_t1 @548 - submit_burst_nocheck_avx512_t2 @549 - flush_burst_sse_t1 @550 - flush_burst_sse_t2 @551 - flush_burst_sse_t3 @552 - flush_burst_sse_no_aesni @553 - flush_burst_avx_t1 @554 - flush_burst_avx_t2 @555 - flush_burst_avx2_t1 @556 - flush_burst_avx2_t2 @557 - flush_burst_avx512_t1 @558 - flush_burst_avx512_t2 @559 - submit_cipher_burst_sse_t1 @560 - submit_cipher_burst_sse_t2 @561 - submit_cipher_burst_sse_t3 @562 - submit_cipher_burst_sse_no_aesni @563 - submit_cipher_burst_avx_t1 @564 - submit_cipher_burst_avx_t2 @565 - submit_cipher_burst_avx2_t1 @566 - submit_cipher_burst_avx2_t2 @567 - submit_cipher_burst_avx512_t1 @568 - submit_cipher_burst_avx512_t2 @569 - submit_cipher_burst_nocheck_sse_t1 @570 - submit_cipher_burst_nocheck_sse_t2 @571 - submit_cipher_burst_nocheck_sse_t3 @572 - submit_cipher_burst_nocheck_sse_no_aesni @573 - submit_cipher_burst_nocheck_avx_t1 @574 - submit_cipher_burst_nocheck_avx_t2 @575 - submit_cipher_burst_nocheck_avx2_t1 @576 - submit_cipher_burst_nocheck_avx2_t2 @577 - submit_cipher_burst_nocheck_avx512_t1 @578 - submit_cipher_burst_nocheck_avx512_t2 @579 - submit_hash_burst_sse_t1 @580 - submit_hash_burst_sse_t2 @581 - submit_hash_burst_sse_t3 @582 - submit_hash_burst_sse_no_aesni @583 - submit_hash_burst_avx_t1 @584 - submit_hash_burst_avx_t2 @585 - submit_hash_burst_avx2_t1 @586 - submit_hash_burst_avx2_t2 @587 - submit_hash_burst_avx512_t1 @588 - submit_hash_burst_avx512_t2 @589 - submit_hash_burst_nocheck_sse_t1 @590 - submit_hash_burst_nocheck_sse_t2 @591 - submit_hash_burst_nocheck_sse_t3 @592 - submit_hash_burst_nocheck_sse_no_aesni @593 - submit_hash_burst_nocheck_avx_t1 @594 - submit_hash_burst_nocheck_avx_t2 @595 - submit_hash_burst_nocheck_avx2_t1 @596 - submit_hash_burst_nocheck_avx2_t2 @597 - submit_hash_burst_nocheck_avx512_t1 @598 - submit_hash_burst_nocheck_avx512_t2 @599 - flush_job_sse_t1 @600 - flush_job_sse_t2 @601 - flush_job_sse_t3 @602 - flush_job_avx_t1 @603 - flush_job_avx_t2 @604 - flush_job_avx2_t1 @605 - flush_job_avx2_t2 @606 - flush_job_avx512_t1 @607 - flush_job_avx512_t2 @608 - queue_size_sse_t1 @609 - queue_size_sse_t2 @610 - queue_size_sse_t3 @611 - queue_size_avx_t1 @612 - queue_size_avx_t2 @613 - queue_size_avx2_t1 @614 - queue_size_avx2_t2 @615 - queue_size_avx512_t1 @616 - queue_size_avx512_t2 @617 - submit_job_sse_t1 @618 - submit_job_sse_t2 @619 - submit_job_sse_t3 @620 - submit_job_avx_t1 @621 - submit_job_avx_t2 @622 - submit_job_avx2_t1 @623 - submit_job_avx2_t2 @624 - submit_job_avx512_t1 @625 - submit_job_avx512_t2 @626 - submit_job_nocheck_sse_t1 @627 - submit_job_nocheck_sse_t2 @628 - submit_job_nocheck_sse_t3 @629 - submit_job_nocheck_avx_t1 @630 - submit_job_nocheck_avx_t2 @631 - submit_job_nocheck_avx2_t1 @632 - submit_job_nocheck_avx2_t2 @633 - submit_job_nocheck_avx512_t1 @634 - submit_job_nocheck_avx512_t2 @635 - get_next_job_sse_t1 @636 - get_next_job_sse_t2 @637 - get_next_job_sse_t3 @638 - get_next_job_avx_t1 @639 - get_next_job_avx_t2 @640 - get_next_job_avx2_t1 @641 - get_next_job_avx2_t2 @642 - get_next_job_avx512_t1 @643 - get_next_job_avx512_t2 @644 - get_completed_job_sse_t1 @645 - get_completed_job_sse_t2 @646 - get_completed_job_sse_t3 @647 - get_completed_job_avx_t1 @648 - get_completed_job_avx_t2 @649 - get_completed_job_avx2_t1 @650 - get_completed_job_avx2_t2 @651 - get_completed_job_avx512_t1 @652 - get_completed_job_avx512_t2 @653 + submit_burst_nocheck_avx_t1 @545 + submit_burst_nocheck_avx_t2 @546 + submit_burst_nocheck_avx2_t1 @547 + submit_burst_nocheck_avx2_t2 @548 + submit_burst_nocheck_avx512_t1 @549 + submit_burst_nocheck_avx512_t2 @550 + flush_burst_sse_t1 @551 + flush_burst_sse_t2 @552 + flush_burst_sse_t3 @553 + flush_burst_sse_no_aesni @554 + flush_burst_avx_t1 @555 + flush_burst_avx_t2 @556 + flush_burst_avx2_t1 @557 + flush_burst_avx2_t2 @558 + flush_burst_avx512_t1 @559 + flush_burst_avx512_t2 @560 + submit_cipher_burst_sse_t1 @561 + submit_cipher_burst_sse_t2 @562 + submit_cipher_burst_sse_t3 @563 + submit_cipher_burst_sse_no_aesni @564 + submit_cipher_burst_avx_t1 @565 + submit_cipher_burst_avx_t2 @566 + submit_cipher_burst_avx2_t1 @567 + submit_cipher_burst_avx2_t2 @568 + submit_cipher_burst_avx512_t1 @569 + submit_cipher_burst_avx512_t2 @570 + submit_cipher_burst_nocheck_sse_t1 @571 + submit_cipher_burst_nocheck_sse_t2 @572 + submit_cipher_burst_nocheck_sse_t3 @573 + submit_cipher_burst_nocheck_sse_no_aesni @574 + submit_cipher_burst_nocheck_avx_t1 @575 + submit_cipher_burst_nocheck_avx_t2 @576 + submit_cipher_burst_nocheck_avx2_t1 @577 + submit_cipher_burst_nocheck_avx2_t2 @578 + submit_cipher_burst_nocheck_avx512_t1 @579 + submit_cipher_burst_nocheck_avx512_t2 @580 + submit_hash_burst_sse_t1 @581 + submit_hash_burst_sse_t2 @582 + submit_hash_burst_sse_t3 @583 + submit_hash_burst_sse_no_aesni @584 + submit_hash_burst_avx_t1 @585 + submit_hash_burst_avx_t2 @586 + submit_hash_burst_avx2_t1 @587 + submit_hash_burst_avx2_t2 @588 + submit_hash_burst_avx512_t1 @589 + submit_hash_burst_avx512_t2 @590 + submit_hash_burst_nocheck_sse_t1 @591 + submit_hash_burst_nocheck_sse_t2 @592 + submit_hash_burst_nocheck_sse_t3 @593 + submit_hash_burst_nocheck_sse_no_aesni @594 + submit_hash_burst_nocheck_avx_t1 @595 + submit_hash_burst_nocheck_avx_t2 @596 + submit_hash_burst_nocheck_avx2_t1 @597 + submit_hash_burst_nocheck_avx2_t2 @598 + submit_hash_burst_nocheck_avx512_t1 @599 + submit_hash_burst_nocheck_avx512_t2 @600 + flush_job_sse_t1 @601 + flush_job_sse_t2 @602 + flush_job_sse_t3 @603 + flush_job_avx_t1 @604 + flush_job_avx_t2 @605 + flush_job_avx2_t1 @606 + flush_job_avx2_t2 @607 + flush_job_avx512_t1 @608 + flush_job_avx512_t2 @609 + queue_size_sse_t1 @610 + queue_size_sse_t2 @611 + queue_size_sse_t3 @612 + queue_size_avx_t1 @613 + queue_size_avx_t2 @614 + queue_size_avx2_t1 @615 + queue_size_avx2_t2 @616 + queue_size_avx512_t1 @617 + queue_size_avx512_t2 @618 + submit_job_sse_t1 @619 + submit_job_sse_t2 @620 + submit_job_sse_t3 @621 + submit_job_avx_t1 @622 + submit_job_avx_t2 @623 + submit_job_avx2_t1 @624 + submit_job_avx2_t2 @625 + submit_job_avx512_t1 @626 + submit_job_avx512_t2 @627 + submit_job_nocheck_sse_t1 @628 + submit_job_nocheck_sse_t2 @629 + submit_job_nocheck_sse_t3 @630 + submit_job_nocheck_avx_t1 @631 + submit_job_nocheck_avx_t2 @632 + submit_job_nocheck_avx2_t1 @633 + submit_job_nocheck_avx2_t2 @634 + submit_job_nocheck_avx512_t1 @635 + submit_job_nocheck_avx512_t2 @636 + get_next_job_sse_t1 @637 + get_next_job_sse_t2 @638 + get_next_job_sse_t3 @639 + get_next_job_avx_t1 @640 + get_next_job_avx_t2 @641 + get_next_job_avx2_t1 @642 + get_next_job_avx2_t2 @643 + get_next_job_avx512_t1 @644 + get_next_job_avx512_t2 @645 + get_completed_job_sse_t1 @646 + get_completed_job_sse_t2 @647 + get_completed_job_sse_t3 @648 + get_completed_job_avx_t1 @649 + get_completed_job_avx_t2 @650 + get_completed_job_avx2_t1 @651 + get_completed_job_avx2_t2 @652 + get_completed_job_avx512_t1 @653 + get_completed_job_avx512_t2 @654 \ No newline at end of file -- GitLab From 5bab8502073fee9d36ed611e34f60803de13729d Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Mon, 22 Aug 2022 15:29:16 +0100 Subject: [PATCH 344/369] burst_api_fuzz_test --- test/job_api_fuzz_test.c | 89 +++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/test/job_api_fuzz_test.c b/test/job_api_fuzz_test.c index 606097a3..a55583dd 100644 --- a/test/job_api_fuzz_test.c +++ b/test/job_api_fuzz_test.c @@ -454,12 +454,16 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_MGR *p_mgr = NULL; IMB_ARCH arch; unsigned i; - const unsigned num_jobs = 20; + const char *n_jobs = getenv("NUM_JOBS"); + const unsigned num_jobs = strtol(n_jobs, NULL, 10); const size_t buffsize = BUFF_SIZE; /* Setting minimum datasize to always fill job structure */ if (dataSize < sizeof(IMB_JOB)) return 0; + if (num_jobs > 32 || num_jobs == 0) + return 0; + /* allocate multi-buffer manager */ p_mgr = alloc_mb_mgr(0); if (p_mgr == NULL) { @@ -483,31 +487,68 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) } IMB_JOB *job = NULL; + /* create job array */ + IMB_JOB *jobs[32] = {NULL}; - for (i = 0; i < num_jobs; i++) { - hash = hash_selection(); - cipher = cipher_selection(); - job = IMB_GET_NEXT_JOB(p_mgr); - memcpy(job, data, sizeof(*job)); - /* - * setenv is invalid or unset - receive flag and fuzz random - * else a specific algo has been selected to fuzz. - */ - if (hash == 0) - job->hash_alg %= (IMB_AUTH_NUM + 1); - else - job->hash_alg = hash; - if (cipher == 0) - job->cipher_mode %= (IMB_CIPHER_NUM + 1); - else - job->cipher_mode = cipher; - clamp_lengths(job, buffsize); - static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); + if (strcmp(getenv("API"), "BURST") == 0) { + while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) + < (uint32_t)num_jobs) + IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + hash = hash_selection(); + cipher = cipher_selection(); + memcpy(job, data, sizeof(*job)); + /* + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been selected to fuzz. + */ + if (hash == 0) + job->hash_alg %= (IMB_AUTH_NUM + 1); + else + job->hash_alg = hash; + if (cipher == 0) + job->cipher_mode %= (IMB_CIPHER_NUM + 1); + else + job->cipher_mode = cipher; + clamp_lengths(job, buffsize); + static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); - fill_job_data(job, buff); - fill_additional_cipher_data(job, buff, buffsize); - fill_additional_hash_data(job, buff, buffsize); - job = IMB_SUBMIT_JOB(p_mgr); + fill_job_data(job, buff); + fill_additional_cipher_data(job, buff, buffsize); + fill_additional_hash_data(job, buff, buffsize); + } + + IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); + } else { + for (i = 0; i < num_jobs; i++) { + hash = hash_selection(); + cipher = cipher_selection(); + job = IMB_GET_NEXT_JOB(p_mgr); + memcpy(job, data, sizeof(*job)); + /* + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been selected to fuzz. + */ + if (hash == 0) + job->hash_alg %= (IMB_AUTH_NUM + 1); + else + job->hash_alg = hash; + if (cipher == 0) + job->cipher_mode %= (IMB_CIPHER_NUM + 1); + else + job->cipher_mode = cipher; + clamp_lengths(job, buffsize); + static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); + + fill_job_data(job, buff); + fill_additional_cipher_data(job, buff, buffsize); + fill_additional_hash_data(job, buff, buffsize); + job = IMB_SUBMIT_JOB(p_mgr); + } } free_mb_mgr(p_mgr); return 0; -- GitLab From 677fbf53ddae9a6b9212b598b7d01f0ed2460eab Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Thu, 25 Aug 2022 12:24:29 +0100 Subject: [PATCH 345/369] submit_cipher_burst --- test/job_api_fuzz_test.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/test/job_api_fuzz_test.c b/test/job_api_fuzz_test.c index a55583dd..6067f39d 100644 --- a/test/job_api_fuzz_test.c +++ b/test/job_api_fuzz_test.c @@ -449,6 +449,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) { IMB_HASH_ALG hash; IMB_CIPHER_MODE cipher; + IMB_CIPHER_DIRECTION dir; char *ar; IMB_MGR *p_mgr = NULL; @@ -456,12 +457,21 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) unsigned i; const char *n_jobs = getenv("NUM_JOBS"); const unsigned num_jobs = strtol(n_jobs, NULL, 10); + const char *key_length = getenv("KEY_LEN"); + const unsigned key_len = strtol(key_length, NULL, 10); const size_t buffsize = BUFF_SIZE; /* Setting minimum datasize to always fill job structure */ if (dataSize < sizeof(IMB_JOB)) return 0; - if (num_jobs > 32 || num_jobs == 0) + if (num_jobs > 32 || num_jobs == 0 || key_len == 0) + return 0; + + if (strcmp(getenv("DIR"), "ENCRYPT") == 0) + dir = IMB_DIR_ENCRYPT; + else if (strcmp(getenv("DIR"), "DECRYPT") == 0) + dir = IMB_DIR_DECRYPT; + else return 0; /* allocate multi-buffer manager */ @@ -522,6 +532,31 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) } IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); + } else if (strcmp(getenv("API"), "CIPHER_BURST") == 0) { + while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) + < (uint32_t)num_jobs) + IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + cipher = cipher_selection(); + memcpy(job, data, sizeof(*job)); + /* + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been selected to fuzz. + */ + if (cipher == 0) + cipher = (job->cipher_mode % + (IMB_CIPHER_NUM + 1)); + clamp_lengths(job, buffsize); + static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); + + fill_job_data(job, buff); + } + + IMB_SUBMIT_CIPHER_BURST(p_mgr, jobs[0], num_jobs, + cipher, dir, key_len); } else { for (i = 0; i < num_jobs; i++) { hash = hash_selection(); -- GitLab From ad0bba2b90ab076f52dacd0bed5d05febfe1181d Mon Sep 17 00:00:00 2001 From: "Ibrahim, Imad" Date: Thu, 25 Aug 2022 16:32:39 +0100 Subject: [PATCH 346/369] submit_hash_burst --- test/job_api_fuzz_test.c | 152 +++++++++++++++++++++++++++------------ 1 file changed, 108 insertions(+), 44 deletions(-) diff --git a/test/job_api_fuzz_test.c b/test/job_api_fuzz_test.c index 6067f39d..a2b7fbb3 100644 --- a/test/job_api_fuzz_test.c +++ b/test/job_api_fuzz_test.c @@ -31,7 +31,7 @@ #include #include #include - +#include #include #define BUFF_SIZE (32*1024*1024) @@ -451,15 +451,28 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_CIPHER_MODE cipher; IMB_CIPHER_DIRECTION dir; char *ar; + const char *api; IMB_MGR *p_mgr = NULL; IMB_ARCH arch; unsigned i; const char *n_jobs = getenv("NUM_JOBS"); - const unsigned num_jobs = strtol(n_jobs, NULL, 10); const char *key_length = getenv("KEY_LEN"); - const unsigned key_len = strtol(key_length, NULL, 10); + unsigned num_jobs; + unsigned key_len; const size_t buffsize = BUFF_SIZE; + bool single = false, cipher_burst = false, + hash_burst = false, burst = false; + + if (n_jobs == NULL) + num_jobs = 10; + else + num_jobs = strtoul(n_jobs, NULL, 10); + if (key_length == NULL) + key_len = 16; + else + key_len = strtoul(key_length, NULL, 10); + /* Setting minimum datasize to always fill job structure */ if (dataSize < sizeof(IMB_JOB)) return 0; @@ -467,12 +480,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) if (num_jobs > 32 || num_jobs == 0 || key_len == 0) return 0; - if (strcmp(getenv("DIR"), "ENCRYPT") == 0) + if (getenv("DIR")) { + if (strcmp(getenv("DIR"), "ENCRYPT") == 0) + dir = IMB_DIR_ENCRYPT; + else if (strcmp(getenv("DIR"), "DECRYPT") == 0) + dir = IMB_DIR_DECRYPT; + } else { dir = IMB_DIR_ENCRYPT; - else if (strcmp(getenv("DIR"), "DECRYPT") == 0) - dir = IMB_DIR_DECRYPT; - else - return 0; + } /* allocate multi-buffer manager */ p_mgr = alloc_mb_mgr(0); @@ -500,21 +515,32 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) /* create job array */ IMB_JOB *jobs[32] = {NULL}; - if (strcmp(getenv("API"), "BURST") == 0) { - while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) - < (uint32_t)num_jobs) - IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + api = getenv("API"); + + if (api == NULL || (strcmp(api, "SINGLE") == 0)) { + single = true; + } else if (strcmp(api, "BURST") == 0) { + burst = true; + } else if (strcmp(api, "CIPHER_BURST") == 0) { + cipher_burst = true; + } else if (strcmp(api, "HASH_BURST") == 0) { + hash_burst = true; + } else { + printf("Invalid API passed to application. Terminating\n"); + return 0; + } + if (single) { for (i = 0; i < num_jobs; i++) { - job = jobs[i]; hash = hash_selection(); cipher = cipher_selection(); + job = IMB_GET_NEXT_JOB(p_mgr); memcpy(job, data, sizeof(*job)); /* - * setenv is invalid or unset - - * receive flag and fuzz random - * else a specific algo has been selected to fuzz. - */ + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been selected to fuzz. + */ if (hash == 0) job->hash_alg %= (IMB_AUTH_NUM + 1); else @@ -529,12 +555,48 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) fill_job_data(job, buff); fill_additional_cipher_data(job, buff, buffsize); fill_additional_hash_data(job, buff, buffsize); + job = IMB_SUBMIT_JOB(p_mgr); + } + } else if (burst) { + while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) + < (uint32_t)num_jobs) + IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; + hash = hash_selection(); + cipher = cipher_selection(); + memcpy(job, data, sizeof(*job)); + /* + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been + * selected to fuzz. + */ + if (hash == 0) + job->hash_alg %= (IMB_AUTH_NUM + 1); + else + job->hash_alg = hash; + if (cipher == 0) + job->cipher_mode %= + (IMB_CIPHER_NUM + 1); + else + job->cipher_mode = cipher; + clamp_lengths(job, buffsize); + static DECLARE_ALIGNED + (uint8_t buff[2*BUFF_SIZE], 64); + + fill_job_data(job, buff); + fill_additional_cipher_data + (job, buff, buffsize); + fill_additional_hash_data + (job, buff, buffsize); } IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); - } else if (strcmp(getenv("API"), "CIPHER_BURST") == 0) { + } else if (cipher_burst) { while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) - < (uint32_t)num_jobs) + < (uint32_t)num_jobs) IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); for (i = 0; i < num_jobs; i++) { @@ -542,49 +604,51 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) cipher = cipher_selection(); memcpy(job, data, sizeof(*job)); /* - * setenv is invalid or unset - - * receive flag and fuzz random - * else a specific algo has been selected to fuzz. - */ + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has been + * selected to fuzz. + */ if (cipher == 0) cipher = (job->cipher_mode % - (IMB_CIPHER_NUM + 1)); + (IMB_CIPHER_NUM + 1)); clamp_lengths(job, buffsize); - static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); + static DECLARE_ALIGNED + (uint8_t buff[2*BUFF_SIZE], 64); fill_job_data(job, buff); } - IMB_SUBMIT_CIPHER_BURST(p_mgr, jobs[0], num_jobs, + IMB_SUBMIT_CIPHER_BURST(p_mgr, job, num_jobs, cipher, dir, key_len); - } else { + } else if (hash_burst) { + while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) + < (uint32_t)num_jobs) + IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); + for (i = 0; i < num_jobs; i++) { + job = jobs[i]; hash = hash_selection(); - cipher = cipher_selection(); - job = IMB_GET_NEXT_JOB(p_mgr); memcpy(job, data, sizeof(*job)); /* - * setenv is invalid or unset - - * receive flag and fuzz random - * else a specific algo has been selected to fuzz. - */ + * setenv is invalid or unset - + * receive flag and fuzz random + * else a specific algo has + * been selected to fuzz. + */ if (hash == 0) - job->hash_alg %= (IMB_AUTH_NUM + 1); - else - job->hash_alg = hash; - if (cipher == 0) - job->cipher_mode %= (IMB_CIPHER_NUM + 1); - else - job->cipher_mode = cipher; + hash = (job->hash_alg % + (IMB_AUTH_NUM + 1)); clamp_lengths(job, buffsize); - static DECLARE_ALIGNED(uint8_t buff[2*BUFF_SIZE], 64); + static DECLARE_ALIGNED + (uint8_t buff[2*BUFF_SIZE], 64); fill_job_data(job, buff); - fill_additional_cipher_data(job, buff, buffsize); - fill_additional_hash_data(job, buff, buffsize); - job = IMB_SUBMIT_JOB(p_mgr); } + + IMB_SUBMIT_HASH_BURST(p_mgr, jobs[0], num_jobs, hash); } + free_mb_mgr(p_mgr); return 0; } -- GitLab From 2b35d83be14817421410168da3c20ff9e3560380 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Tue, 13 Sep 2022 19:30:37 +0100 Subject: [PATCH 347/369] avx2_t1: [CHACHA20_POLY1305] fix AVX-SSE transitions --- lib/avx2_t1/chacha20_avx2.asm | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/avx2_t1/chacha20_avx2.asm b/lib/avx2_t1/chacha20_avx2.asm index 9e11bf81..25e116fd 100644 --- a/lib/avx2_t1/chacha20_avx2.asm +++ b/lib/avx2_t1/chacha20_avx2.asm @@ -927,7 +927,7 @@ partial_block: no_partial_block: endbranch64 %ifdef SAFE_DATA - clear_all_ymms_asm + vpxor ymm0, ymm0 ; Clear stack frame %assign i 0 %rep 16 @@ -949,6 +949,12 @@ no_partial_block: %endif mov rsp, [rsp + _RSP_SAVE] +%ifdef SAFE_DATA + clear_scratch_ymms_asm +%else + vzeroupper +%endif + exit: mov rax, job or dword [rax + _status], IMB_STATUS_COMPLETED_CIPHER @@ -1378,7 +1384,7 @@ no_partial_block_ks: mov [ctx + LastBlkCount], blk_cnt %ifdef SAFE_DATA - clear_all_ymms_asm + vpxor ymm0, ymm0 ; Clear stack frame %assign i 0 %rep 16 @@ -1407,6 +1413,12 @@ no_partial_block_ks: %endif mov rsp, [rsp + _RSP_SAVE] +%ifdef SAFE_DATA + clear_scratch_ymms_asm +%else + vzeroupper +%endif + exit_ks: ret -- GitLab From 3d6f201ff598c95622df797c22ae68ee43e2dc36 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Wed, 14 Sep 2022 13:42:47 +0100 Subject: [PATCH 348/369] test: [ACVP] use separate handler for GMAC This handler uses dedicated GMAC direct API and GMAC algorithms, through job API. --gmac-api is not needed anymore, as this handler exercises these APIs. --- test/acvp_app_main.c | 235 ++++++++++++++++++++++++++++--------------- 1 file changed, 153 insertions(+), 82 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 86c98c34..da7c246e 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -45,7 +45,6 @@ static ACVP_RESULT logger(char *msg) IMB_MGR *mb_mgr = NULL; int verbose = 0; int direct_api = 0; /* job API by default */ -int gmac_api = 0; static int aes_cbc_handler(ACVP_TEST_CASE *test_case) { @@ -152,13 +151,8 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) aes_gcm_enc_dec_update_t gcm_update_dec = mb_mgr->gcm128_dec_update; aes_gcm_enc_dec_finalize_t gcm_finalize_dec = mb_mgr->gcm128_dec_finalize; - aes_gmac_init_t gmac_init_var = mb_mgr->gmac128_init; - aes_gmac_update_t gmac_update = mb_mgr->gmac128_update; - aes_gmac_finalize_t gmac_finalize = mb_mgr->gmac128_finalize; struct gcm_key_data key; struct gcm_context_data ctx; - IMB_HASH_ALG hash_mode; - int use_gmac_api = 0; if (test_case == NULL) return EXIT_FAILURE; @@ -170,26 +164,15 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) return EXIT_FAILURE; } - if (gmac_api) { - if ((tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT && - tc->pt_len == 0) || - (tc->direction == ACVP_SYM_CIPH_DIR_DECRYPT && - tc->ct_len == 0)) - use_gmac_api = 1; - } - switch (tc->key_len) { case 128: IMB_AES128_GCM_PRE(mb_mgr, tc->key, &key); - hash_mode = IMB_AUTH_AES_GMAC_128; break; case 192: IMB_AES192_GCM_PRE(mb_mgr, tc->key, &key); - hash_mode = IMB_AUTH_AES_GMAC_192; break; case 256: IMB_AES256_GCM_PRE(mb_mgr, tc->key, &key); - hash_mode = IMB_AUTH_AES_GMAC_256; break; default: fprintf(stderr, "Unsupported AES key length\n"); @@ -207,9 +190,6 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) gcm_finalize_enc = mb_mgr->gcm192_enc_finalize; gcm_update_dec = mb_mgr->gcm192_dec_update; gcm_finalize_dec = mb_mgr->gcm192_dec_finalize; - gmac_init_var = mb_mgr->gmac192_init; - gmac_update = mb_mgr->gmac192_update; - gmac_finalize = mb_mgr->gmac192_finalize; break; case 256: gcm_init_var_iv = mb_mgr->gcm256_init_var_iv; @@ -217,6 +197,143 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) gcm_finalize_enc = mb_mgr->gcm256_enc_finalize; gcm_update_dec = mb_mgr->gcm256_dec_update; gcm_finalize_dec = mb_mgr->gcm256_dec_finalize; + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + } else { + job = IMB_GET_NEXT_JOB(mb_mgr); + job->key_len_in_bytes = tc->key_len >> 3; + job->cipher_mode = IMB_CIPHER_GCM; + job->hash_alg = IMB_AUTH_AES_GMAC; + job->u.GCM.aad = tc->aad; + job->u.GCM.aad_len_in_bytes = tc->aad_len; + job->enc_keys = &key; + job->dec_keys = &key; + job->iv = tc->iv; + job->iv_len_in_bytes = tc->iv_len; + job->cipher_start_src_offset_in_bytes = 0; + job->hash_start_src_offset_in_bytes = 0; + job->auth_tag_output_len_in_bytes = tc->tag_len; + } + + if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { + if (direct_api == 1) { + gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, + tc->aad, tc->aad_len); + gcm_update_enc(&key, &ctx, tc->ct, + tc->pt, tc->pt_len); + gcm_finalize_enc(&key, &ctx, tc->tag, + tc->tag_len); + } else { + job->src = tc->pt; + job->dst = tc->ct; + job->msg_len_to_cipher_in_bytes = tc->pt_len; + job->msg_len_to_hash_in_bytes = tc->pt_len; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->auth_tag_output = tc->tag; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + } else /* DECRYPT */ { + uint8_t res_tag[MAX_TAG_LENGTH]; + + if (direct_api == 1) { + gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, + tc->aad, tc->aad_len); + gcm_update_dec(&key, &ctx, tc->pt, + tc->ct, tc->ct_len); + gcm_finalize_dec(&key, &ctx, + res_tag, tc->tag_len); + } else { + job->src = tc->ct; + job->dst = tc->pt; + job->msg_len_to_cipher_in_bytes = tc->ct_len; + job->msg_len_to_hash_in_bytes = tc->ct_len; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->auth_tag_output = res_tag; + + job = IMB_SUBMIT_JOB(mb_mgr); + if (job == NULL) + job = IMB_FLUSH_JOB(mb_mgr); + if (job->status != IMB_STATUS_COMPLETED) { + fprintf(stderr, "Invalid job\n"); + return EXIT_FAILURE; + } + } + if (memcmp(res_tag, tc->tag, tc->tag_len) != 0) { + if (verbose) { + hexdump(stdout, "result tag: ", + res_tag, tc->tag_len); + hexdump(stdout, "reference tag: ", + tc->tag, tc->tag_len); + fprintf(stderr, "Invalid tag\n"); + } + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; +} + +static int aes_gmac_handler(ACVP_TEST_CASE *test_case) +{ + ACVP_SYM_CIPHER_TC *tc; + IMB_JOB *job = NULL; + aes_gmac_init_t gmac_init_var = mb_mgr->gmac128_init; + aes_gmac_update_t gmac_update = mb_mgr->gmac128_update; + aes_gmac_finalize_t gmac_finalize = mb_mgr->gmac128_finalize; + struct gcm_key_data key; + struct gcm_context_data ctx; + IMB_HASH_ALG hash_mode; + + if (test_case == NULL) + return EXIT_FAILURE; + + tc = test_case->tc.symmetric; + + if (tc->direction != ACVP_SYM_CIPH_DIR_ENCRYPT && + tc->direction != ACVP_SYM_CIPH_DIR_DECRYPT) { + return EXIT_FAILURE; + } + + switch (tc->key_len) { + case 128: + IMB_AES128_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_128; + break; + case 192: + IMB_AES192_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_192; + break; + case 256: + IMB_AES256_GCM_PRE(mb_mgr, tc->key, &key); + hash_mode = IMB_AUTH_AES_GMAC_256; + break; + default: + fprintf(stderr, "Unsupported AES key length\n"); + return EXIT_FAILURE; + } + + if (direct_api == 1) { + switch (tc->key_len) { + case 128: + /* Function pointers already set for 128-bit key */ + break; + case 192: + gmac_init_var = mb_mgr->gmac192_init; + gmac_update = mb_mgr->gmac192_update; + gmac_finalize = mb_mgr->gmac192_finalize; + break; + case 256: gmac_init_var = mb_mgr->gmac256_init; gmac_update = mb_mgr->gmac256_update; gmac_finalize = mb_mgr->gmac256_finalize; @@ -228,22 +345,11 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) } else { job = IMB_GET_NEXT_JOB(mb_mgr); job->key_len_in_bytes = tc->key_len >> 3; - if (use_gmac_api == 1) { - job->cipher_mode = IMB_CIPHER_NULL; - job->hash_alg = hash_mode; - job->u.GMAC._iv = tc->iv; - job->u.GMAC.iv_len_in_bytes = tc->iv_len; - job->u.GMAC._key = &key; - } else { - job->cipher_mode = IMB_CIPHER_GCM; - job->hash_alg = IMB_AUTH_AES_GMAC; - job->u.GCM.aad = tc->aad; - job->u.GCM.aad_len_in_bytes = tc->aad_len; - job->enc_keys = &key; - job->dec_keys = &key; - job->iv = tc->iv; - job->iv_len_in_bytes = tc->iv_len; - } + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = hash_mode; + job->u.GMAC._iv = tc->iv; + job->u.GMAC.iv_len_in_bytes = tc->iv_len; + job->u.GMAC._key = &key; job->cipher_start_src_offset_in_bytes = 0; job->hash_start_src_offset_in_bytes = 0; job->auth_tag_output_len_in_bytes = tc->tag_len; @@ -251,29 +357,13 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) if (tc->direction == ACVP_SYM_CIPH_DIR_ENCRYPT) { if (direct_api == 1) { - if (use_gmac_api == 1) { - gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); - gmac_update(&key, &ctx, tc->aad, tc->aad_len); - gmac_finalize(&key, &ctx, tc->tag, + gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); + gmac_update(&key, &ctx, tc->aad, tc->aad_len); + gmac_finalize(&key, &ctx, tc->tag, tc->tag_len); - } else { - gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, - tc->aad, tc->aad_len); - gcm_update_enc(&key, &ctx, tc->ct, - tc->pt, tc->pt_len); - gcm_finalize_enc(&key, &ctx, tc->tag, - tc->tag_len); - } } else { - if (use_gmac_api == 1) { - job->src = tc->aad; - job->msg_len_to_hash_in_bytes = tc->aad_len; - } else { - job->src = tc->pt; - job->dst = tc->ct; - job->msg_len_to_cipher_in_bytes = tc->pt_len; - job->msg_len_to_hash_in_bytes = tc->pt_len; - } + job->src = tc->aad; + job->msg_len_to_hash_in_bytes = tc->aad_len; job->cipher_direction = IMB_DIR_ENCRYPT; job->chain_order = IMB_ORDER_CIPHER_HASH; job->auth_tag_output = tc->tag; @@ -290,28 +380,12 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) uint8_t res_tag[MAX_TAG_LENGTH]; if (direct_api == 1) { - if (use_gmac_api == 1) { - gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); - gmac_update(&key, &ctx, tc->aad, tc->aad_len); - gmac_finalize(&key, &ctx, res_tag, tc->tag_len); - } else { - gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, - tc->aad, tc->aad_len); - gcm_update_dec(&key, &ctx, tc->pt, - tc->ct, tc->ct_len); - gcm_finalize_dec(&key, &ctx, - res_tag, tc->tag_len); - } + gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); + gmac_update(&key, &ctx, tc->aad, tc->aad_len); + gmac_finalize(&key, &ctx, res_tag, tc->tag_len); } else { - if (use_gmac_api == 1) { - job->src = tc->aad; - job->msg_len_to_hash_in_bytes = tc->aad_len; - } else { - job->src = tc->ct; - job->dst = tc->pt; - job->msg_len_to_cipher_in_bytes = tc->ct_len; - job->msg_len_to_hash_in_bytes = tc->ct_len; - } + job->src = tc->aad; + job->msg_len_to_hash_in_bytes = tc->aad_len; job->cipher_direction = IMB_DIR_DECRYPT; job->chain_order = IMB_ORDER_HASH_CIPHER; job->auth_tag_output = res_tag; @@ -1214,7 +1288,6 @@ static void usage(const char *app_name) "--req FILENAME: request file in JSON format (required)\n" "--resp FILENAME: response file in JSON format (required)\n" "--direct-api: uses direct API instead of job API if available\n" - "--gmac-api: uses GMAC API\n" "--arch ARCH: select arch to test (SSE/AVX/AVX2/AVX512)\n" "-h: print this message\n" "-v: verbose, prints extra information\n\n" @@ -1282,8 +1355,6 @@ int main(int argc, char **argv) i++; } else if (strcmp(argv[i], "--direct-api") == 0) { direct_api = 1; - } else if (strcmp(argv[i], "--gmac-api") == 0) { - gmac_api = 1; } else if (strcmp(argv[i], "-h") == 0) { usage(argv[0]); ret = EXIT_SUCCESS; @@ -1322,7 +1393,7 @@ int main(int argc, char **argv) &aes_ctr_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_GMAC, - &aes_gcm_handler) != ACVP_SUCCESS) + &aes_gmac_handler) != ACVP_SUCCESS) goto exit; if (acvp_cap_sym_cipher_enable(ctx, ACVP_AES_CCM, -- GitLab From 304d8b812bccb0996affb6207102f5cd71faa83a Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 14 Sep 2022 16:07:29 +0000 Subject: [PATCH 349/369] lib: [aes-ccm] fix for issue #107 on SSE and AVX architectures --- ...gr_aes128_ccm_auth_submit_flush_x8_avx.asm | 27 +++++++++++++++++++ ...gr_aes128_ccm_auth_submit_flush_x4_sse.asm | 27 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm index a5d6dac3..fc82044e 100644 --- a/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_ccm_auth_submit_flush_x8_avx.asm @@ -386,6 +386,23 @@ APPEND(skip_,I): vmovdqa [state + _aes_ccm_lens], ccm_lens ;; Find min length vphminposuw min_len_idx, ccm_lens + jmp %%_ccm_round + +%%_ccm_round_flush: + ;; This is identical to the above block but optimized for + ;; a repeat flush operation when keys etc. are already set + ;; - vphminposuw was already executed + ;; - good_lane is already known + ;; - copy input pointer from good lane to empty lanes + mov tmp2, [state + _aes_ccm_args_in + good_lane*8] +%assign I 0 +%rep 8 + cmp qword [state + _aes_ccm_job_in_lane + I*8], 0 + jne APPEND(skip2_,I) + mov [state + _aes_ccm_args_in + I*8], tmp2 +APPEND(skip2_,I): +%assign I (I+1) +%endrep %endif ; end FLUSH @@ -524,12 +541,17 @@ APPEND(skip_clear_,I): ; Reset NULL lane lens to UINT16_MAX %ifidn %%SUBMIT_FLUSH, FLUSH SET_NULL_JOB_LENS_TO_MAX ccm_lens, xtmp0, xtmp1, xtmp2, xtmp3, xtmp4 + mov good_lane, min_idx %endif XVPINSRW ccm_lens, xtmp0, tmp2, min_idx, tmp, scale_x16 vphminposuw min_len_idx, ccm_lens vmovdqa [state + _aes_ccm_lens], ccm_lens +%ifidn %%SUBMIT_FLUSH, FLUSH + jmp %%_ccm_round_flush +%else jmp %%_ccm_round +%endif %%_prepare_partial_block_to_auth: ; Check if partial block needs to be hashed @@ -559,7 +581,12 @@ APPEND(skip_clear_,I): vmovdqa [init_block_addr], xtmp0 mov [state + _aes_ccm_args_in + min_idx * 8], init_block_addr +%ifidn %%SUBMIT_FLUSH, FLUSH + mov good_lane, min_idx + jmp %%_ccm_round_flush +%else jmp %%_ccm_round +%endif %endmacro align 64 diff --git a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm index bfff5c2b..7e12dedd 100644 --- a/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_ccm_auth_submit_flush_x4_sse.asm @@ -399,6 +399,23 @@ APPEND(skip_,I): movdqa [state + _aes_ccm_lens], ccm_lens ;; Find min length phminposuw min_len_idx, ccm_lens + jmp %%_ccm_round + +%%_ccm_round_flush: + ;; This is identical to the above block but optimized for + ;; a repeat flush operation when keys etc. are already set + ;; - vphminposuw was already executed + ;; - good_lane is already known + ;; - copy input pointer from good lane to empty lanes + mov tmp2, [state + _aes_ccm_args_in + good_lane*8] +%assign I 0 +%rep NUM_LANES + cmp qword [state + _aes_ccm_job_in_lane + I*8], 0 + jne APPEND(skip2_,I) + mov [state + _aes_ccm_args_in + I*8], tmp2 +APPEND(skip2_,I): +%assign I (I+1) +%endrep %endif ; end FLUSH @@ -542,12 +559,17 @@ APPEND(skip_clear_,I): ; Reset NULL lane lens to UINT16_MAX %ifidn %%SUBMIT_FLUSH, FLUSH SET_NULL_JOB_LENS_TO_MAX ccm_lens, xtmp0, xtmp1, xtmp2, xtmp3 + mov good_lane, min_idx %endif XPINSRW ccm_lens, xtmp0, tmp2, min_idx, tmp, scale_x16 phminposuw min_len_idx, ccm_lens movdqa [state + _aes_ccm_lens], ccm_lens +%ifidn %%SUBMIT_FLUSH, FLUSH + jmp %%_ccm_round_flush +%else jmp %%_ccm_round +%endif %%_prepare_partial_block_to_auth: ; Check if partial block needs to be hashed @@ -578,7 +600,12 @@ APPEND(skip_clear_,I): movdqa [init_block_addr], xtmp0 mov [state + _aes_ccm_args_in + min_idx * 8], init_block_addr +%ifidn %%SUBMIT_FLUSH, FLUSH + mov good_lane, min_idx + jmp %%_ccm_round_flush +%else jmp %%_ccm_round +%endif %endmacro align 64 -- GitLab From b7c4f7753fdac444ca64125aa05ba6da0160a8ce Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 6 Sep 2022 16:30:19 +0000 Subject: [PATCH 350/369] test: new wycheproof application added It exercises test vectors from https://github.com/google/wycheproof project. Tested algorithms are: aes-gcm, aes-ccm, chacha20-poly1305, aes-cmac, gmac, hmac-sha1, hmac-sha224, hmac-sha256, hmac-sha384 and hmac-sha512. --- .gitignore | 2 + Makefile | 3 + test/wycheproof/Makefile | 127 + test/wycheproof/aead_test.h | 50 + test/wycheproof/aes_ccm_test.json.c | 4168 +++++++++++++++++ test/wycheproof/aes_cmac_test.json.c | 1774 +++++++ test/wycheproof/aes_gcm_test.json.c | 2528 ++++++++++ test/wycheproof/chacha20_poly1305_test.json.c | 4058 ++++++++++++++++ test/wycheproof/gmac_test.json.c | 3263 +++++++++++++ test/wycheproof/hmac_sha1_test.json.c | 1119 +++++ test/wycheproof/hmac_sha224_test.json.c | 1219 +++++ test/wycheproof/hmac_sha256_test.json.c | 1262 +++++ test/wycheproof/hmac_sha384_test.json.c | 1604 +++++++ test/wycheproof/hmac_sha512_test.json.c | 1853 ++++++++ test/wycheproof/mac_test.h | 47 + test/wycheproof/win_x64.mak | 98 + test/wycheproof/wycheproof.c | 1470 ++++++ win_x64.mak | 2 + 18 files changed, 24647 insertions(+) create mode 100644 test/wycheproof/Makefile create mode 100644 test/wycheproof/aead_test.h create mode 100644 test/wycheproof/aes_ccm_test.json.c create mode 100644 test/wycheproof/aes_cmac_test.json.c create mode 100644 test/wycheproof/aes_gcm_test.json.c create mode 100644 test/wycheproof/chacha20_poly1305_test.json.c create mode 100644 test/wycheproof/gmac_test.json.c create mode 100644 test/wycheproof/hmac_sha1_test.json.c create mode 100644 test/wycheproof/hmac_sha224_test.json.c create mode 100644 test/wycheproof/hmac_sha256_test.json.c create mode 100644 test/wycheproof/hmac_sha384_test.json.c create mode 100644 test/wycheproof/hmac_sha512_test.json.c create mode 100644 test/wycheproof/mac_test.h create mode 100644 test/wycheproof/win_x64.mak create mode 100644 test/wycheproof/wycheproof.c diff --git a/.gitignore b/.gitignore index 7c4f0f6c..24e1c5d1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ job_api_fuzz_test* *_lnk.def TAGS acvp_app +test/wycheproof/wycheproof +!wycheproof.c diff --git a/Makefile b/Makefile index e4a298d5..37f1f92b 100644 --- a/Makefile +++ b/Makefile @@ -30,16 +30,19 @@ all: $(MAKE) -C lib $(MAKE) -C test + $(MAKE) -C test/wycheproof $(MAKE) -C perf clean: $(MAKE) -C lib clean $(MAKE) -C test clean + $(MAKE) -C test/wycheproof clean $(MAKE) -C perf clean style: $(MAKE) -C lib style $(MAKE) -C test style + $(MAKE) -C test/wycheproof style $(MAKE) -C perf style install: diff --git a/test/wycheproof/Makefile b/test/wycheproof/Makefile new file mode 100644 index 00000000..faf051df --- /dev/null +++ b/test/wycheproof/Makefile @@ -0,0 +1,127 @@ +# +# Copyright (c) 2022, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP := wycheproof +INSTPATH ?= /usr/include/intel-ipsec-mb.h +LIB_DIR ?= ../../lib + +MINGW ?= $(shell $(CC) -dM -E - < /dev/null | grep -i mingw | wc -l | sed 's/^ *//') + +CFLAGS = -MMD -D_GNU_SOURCE -DNO_COMPAT_IMB_API_053 \ + -W -Wall -Wextra -Wmissing-declarations -Wpointer-arith \ + -Wcast-qual -Wundef -Wwrite-strings \ + -Wformat -Wformat-security \ + -Wunreachable-code -Wmissing-noreturn -Wsign-compare -Wno-endif-labels \ + -Wstrict-prototypes -Wmissing-prototypes -Wold-style-definition \ + -fno-delete-null-pointer-checks -fwrapv + +# -fno-strict-overflow is not supported by clang +ifneq ($(CC),clang) +CFLAGS += -fno-strict-overflow +endif + +# if "-z ibt" is supported then assume "-z shstk, -z cet-report=error" are also supported +# "-fcf-protection" needs to be checked separately +ifeq ($(MINGW),0) +CC_HAS_CET = $(and $(shell $(CC) --target-help 2> /dev/null | grep -m1 -e "-z ibt" | wc -l), \ + $(shell $(CC) --help=common 2> /dev/null | grep -m1 -e "-fcf-protection" | wc -l)) +CET_LDFLAGS=-r -z ibt -z shstk +endif + +ifeq ($(CC_HAS_CET),1) +CFLAGS += -fcf-protection=full +endif + +ifeq ($(MINGW),0) +LDFLAGS = -fPIE -z noexecstack -z relro -z now +else +LDFLAGS = -fPIE +endif + +ifeq ($(CC_HAS_CET),1) +LDFLAGS += -fcf-protection=full -Wl,-z,ibt -Wl,-z,shstk -Wl,-z,cet-report=error +endif +LDLIBS = -lIPSec_MB + +ifeq ("$(shell test -r $(INSTPATH) && echo -n yes)","yes") +# library installed +CFLAGS += +else +# library not installed +CFLAGS += -I../../lib +LDFLAGS += -L$(LIB_DIR) +endif + +DEBUG_OPT ?= -O0 +ifeq ($(DEBUG),y) +CFLAGS += $(DEBUG_OPT) -DDEBUG -g +LDFLAGS += -g +else +ifeq ($(MINGW),0) +CFLAGS += -O3 +else +CFLAGS += -O2 +endif +endif + +OBJ_FILES = aes_gcm_test.json.o aes_ccm_test.json.o \ + chacha20_poly1305_test.json.o \ + aes_cmac_test.json.o gmac_test.json.o gmac_test.json.o \ + hmac_sha1_test.json.o hmac_sha224_test.json.o hmac_sha256_test.json.o \ + hmac_sha384_test.json.o hmac_sha512_test.json.o \ + wycheproof.o + +all: $(APP) + +$(APP): $(OBJ_FILES) + +.PHONY: clean +clean: + -rm -f $(OBJ_FILES) *.d $(APP) + +# style check section +CHECKPATCH?=checkpatch.pl +CHECKPATCH_FLAGS = --no-tree --no-signoff --emacs --no-color --ignore CODE_INDENT,INITIALISED_STATIC,LEADING_SPACE,SPLIT_STRING,UNSPECIFIED_INT,ARRAY_SIZE,BLOCK_COMMENT_STYLE,GLOBAL_INITIALISERS,AVOID_EXTERNS,COMPLEX_MACRO,USE_FUNC,CONSTANT_COMPARISON,MISSING_SPACE,NEW_TYPEDEFS + +%.c_style_check : %.c + $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< + +%.h_style_check : %.h + $(CHECKPATCH) $(CHECKPATCH_FLAGS) -f $< + +SOURCES_ALL := $(wildcard *.[ch]) +SOURCES_STYLE := $(foreach infile,$(SOURCES_ALL),$(infile)_style_check) + +.PHONY: style +style: $(SOURCES_STYLE) + +# if target not clean or rinse then make dependencies +ifneq ($(MAKECMDGOALS),clean) +ifneq ($(MAKECMDGOALS),style) +-include $(wildcard *.d) +endif +endif diff --git a/test/wycheproof/aead_test.h b/test/wycheproof/aead_test.h new file mode 100644 index 00000000..dd2f45d5 --- /dev/null +++ b/test/wycheproof/aead_test.h @@ -0,0 +1,50 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef AEAD_TEST_H +#define AEAD_TEST_H + +#include +#include + +struct aead_test { + size_t ivSize; /* bits */ + size_t keySize; /* bits */ + size_t tagSize; /* bits */ + size_t tcId; + const char *key; + const char *iv; + const char *aad; + const char *msg; + const char *ct; + const char *tag; + int resultValid; + size_t aadSize; /* bits */ + size_t msgSize; /* bits */ +}; + +#endif /* AEAD_TEST_H */ diff --git a/test/wycheproof/aes_ccm_test.json.c b/test/wycheproof/aes_ccm_test.json.c new file mode 100644 index 00000000..da8f1791 --- /dev/null +++ b/test/wycheproof/aes_ccm_test.json.c @@ -0,0 +1,4168 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* AES-CCM, 0.8r12 */ +#include "aead_test.h" +const struct aead_test aes_ccm_test_json[] = { + { 96, 128, 128, 1, + "\xbe\xdc\xfb\x5a\x01\x1e\xbc\x84\x60\x0f\xcb\x29\x6c\x15\xaf\x0d", + "\x43\x8a\x54\x7a\x94\xea\x88\xdc\xe4\x6c\x6c\x85", "", "", "", + "\x25\xd1\xa3\x84\x95\xa7\xde\xa4\x5b\xda\x04\x97\x05\x62\x7d\x10", 1, + 0, 0 }, + { 96, 128, 128, 2, + "\x38\x4e\xa4\x16\xac\x3c\x2f\x51\xa7\x6e\x7d\x82\x26\x34\x6d\x4e", + "\xb3\x0c\x08\x47\x27\xad\x1c\x59\x2a\xc2\x1d\x12", "", "\x35", + "\xd7", + "\x6b\xe3\xfd\x13\xb7\x06\x5a\xfc\x19\xe3\xb8\xa3\xb9\x6b\x39\xfb", 1, + 0, 8 }, + { 96, 128, 128, 3, + "\xca\xe3\x1c\xd9\xf5\x55\x26\xeb\x03\x82\x41\xfc\x44\xca\xc1\xe5", + "\xb5\xe0\x06\xde\xd5\x53\x11\x0e\x6d\xc5\x65\x29", "", + "\xd1\x09\x89\xf2\xc5\x2e\x94\xad", + "\xe6\x4d\x0b\x64\xeb\xb3\x81\xec", + "\x25\x40\x9c\x79\x5d\x49\x1d\x80\x4e\x58\x39\x17\x22\x7b\x73\xc7", 1, + 0, 64 }, + { 96, 128, 128, 4, + "\xff\xdf\x42\x28\x36\x1e\xa1\xf8\x16\x58\x52\x13\x6b\x34\x80\xf7", + "\x0e\x16\x66\xf2\xdc\x65\x2f\x77\x08\xfb\x8f\x0d", "", + "\x25\xb1\x2e\x28\xac\x0e\xf6\xea\xd0\x22\x6a\x3b\x22\x88\xc8\x00", + "\xaa\xf5\x96\xfa\x5b\x00\xaa\xac\x27\x70\x01\x46\xae\xc9\x32\xa9", + "\x84\x8b\x67\x35\xd3\x2c\x96\xe4\xa0\x53\x2b\xcd\xfa\xf3\x35\x82", 1, + 0, 128 }, + { 96, 128, 128, 5, + "\xc1\x5e\xd2\x27\xdd\x2e\x23\x7e\xcd\x08\x7e\xaa\xaa\xd1\x9e\xa4", + "\x96\x5f\xf6\x64\x31\x16\xac\x14\x43\xa2\xde\xc7", "", + "\xfe\xe6\x2f\xde\x97\x3f\xe0\x25\xad\x6b\x32\x2d\xcd\xf3\xc6\x3f" + "\xc7", + "\x03\x33\xdf\x2a\x86\xd7\xf0\x94\xdd\x8b\xce\x75\xda\x6c\x38\xc5" + "\xc1", + "\x41\x7d\xa2\x9d\xf8\x5a\x1d\x13\x4f\xee\xe8\xaa\x35\x56\x90\x81", 1, + 0, 136 }, + { 96, 128, 128, 6, + "\xa8\xee\x11\xb2\x6d\x7c\xeb\x7f\x17\xea\xa1\xe4\xb8\x3a\x2c\xf6", + "\xfb\xbc\x04\xfd\x6e\x02\x5b\x71\x93\xeb\x57\xf6", "", + "\xc0\x8f\x08\x5e\x6a\x9e\x0e\xf3\x63\x62\x80\xc1\x1e\xcf\xad\xf0\xc1" + "\xe7\x29\x19\xff\xc1\x7e\xaf", + "\xfc\xaa\xa3\x8f\xed\xa3\xac\xa9\x75\xac\x76\x55\x3c\x3e\x7e\xf3\x6b" + "\x88\x7a\x8c\x4d\x82\x41\xf9", + "\xc2\xc6\xdc\xae\xae\xb9\xf3\x8a\x3a\x42\xd2\xf4\xe8\xa1\x7d\xe4", 1, + 0, 192 }, + { 96, 128, 128, 7, + "\x16\x55\xbf\x66\x2f\x7e\xe6\x85\x61\x57\x01\xfd\x37\x79\xd6\x28", + "\x42\xb5\x13\x88\xf6\xf9\x04\x7a\x2a\x99\x45\x75", "", + "\x85\x7b\x2f\x6c\xd6\x08\xc9\xce\xa0\x24\x6c\x74\x0c\xaa\x4c\xa1\x9c" + "\x5f\x1c\x7d\x71\xcb\x92\x73\xf0\xd8\xc8\xbb\x65\xb7\x0a", + "\xb3\xfb\x66\xd3\xf2\xcb\x75\x90\xad\x5e\xf5\x60\x48\x13\xc1\x25\x02" + "\x0e\xe3\xd7\x91\xcb\x0e\xc6\x7e\xb5\xeb\x86\x70\x9b\x6b", + "\x15\x55\x77\xb9\x8a\x81\x1e\x45\x32\x46\x16\x04\x39\x97\xbc\x03", 1, + 0, 248 }, + { 96, 128, 128, 8, + "\x3f\xd5\x07\x41\xec\x47\xdd\xbf\xc2\xfc\x09\x09\x75\xd1\x54\xf0", + "\xee\xf1\xa6\xe6\x51\x32\x18\x52\xf0\xb2\x5a\x31", "", + "\xd6\xf6\xa9\xa2\x4d\xb6\xa7\xa6\x17\x6d\x43\x62\x63\x9c\x4f\xd7\x7f" + "\x70\xf3\xe0\x89\xdd\x94\x00\x86\xe1\x2a\x9b\xec\xba\xf9\x7f\x82", + "\x53\x26\x94\xbb\x28\x51\xea\x7f\x3b\xdd\x37\xc4\xe8\x06\xbe\x5b\x95" + "\x3e\xa7\x9d\x08\x10\x0e\x74\xaf\x3f\xa6\x7e\xca\x88\x90\xdb\x28", + "\x9e\x1a\xce\xe6\x88\x84\x8b\xca\x45\x4c\x6d\x04\x75\x3d\x3c\x7d", 1, + 0, 264 }, + { 96, 128, 128, 9, + "\x42\xe3\x8a\xbe\xf2\xdd\x75\x73\x24\x8c\x5a\xef\xb3\xec\xca\x54", + "\x06\x4b\x3c\xfb\xe0\x4d\x94\xd4\xd5\xc1\x9b\x30", "", + "\x2c\x76\x3b\x9e\xc8\x49\x03\xbc\xbb\x8a\xec\x15\xe6\x78\xa3\xa9\x55" + "\xe4\x87\x0e\xdb\xf6\x2d\x9d\x3c\x81\xc4\xf9\xed\x61\x54\x87\x78\x75" + "\x77\x9c\xa3\x3c\xce\x8f\x73\xa5\x5c\xa7\xaf\x1d\x8d\x81\x7f\xc6\xba" + "\xac\x00\xef\x96\x2c\x5a\x0d\xa3\x39\xce\x81\x42\x7a\x3d\x59", + "\x4c\xa0\x1b\x5b\x2a\x5e\x57\xbc\xc1\xa4\xb7\xf6\x3f\x04\x9d\xc4\x77" + "\xe3\xee\x2e\x5c\x26\x8e\xfb\x34\x6f\xf9\x5b\x7d\xcd\x67\xf8\x6e\xd0" + "\xf1\x1b\xb1\x7c\x1d\xd7\xfb\x51\x1d\x2f\x37\xb9\x68\x45\x50\xc0\xd8" + "\x4b\xe0\xf1\x00\x30\xcc\xc4\xe0\xde\x5b\x74\xef\x92\xea\x54", + "\xc5\xa5\x7d\xd6\xfa\x16\xaa\x9d\xe8\xde\x20\xe6\xbd\x32\x13\x96", 1, + 0, 528 }, + { 96, 128, 128, 10, + "\x59\xab\x7e\xc1\xc0\x2b\xb2\x06\xaf\x5a\x91\x31\xf1\x13\x43\x11", + "\x55\x08\xf5\xce\xa1\x97\x38\x69\x86\xd9\x2d\xbe", + "\xa4\x3d\x39\xf7\x8a\x2e\x9a\x8a", "", "", + "\x09\xec\x70\xfa\xae\x33\x35\x37\xa7\x31\x49\x29\xdd\xfb\x52\x5b", 1, + 64, 0 }, + { 96, 128, 128, 11, + "\x94\x15\xf9\x25\xbc\xb4\x1d\xc2\x5e\x86\xc8\x26\xdb\xc8\xbf\x68", + "\xbd\xff\xaa\x76\x3b\x91\x6f\xf0\xee\x3f\x3c\xe4", + "\x70\x5d\x67\x6c\xd8\xa9\x44\x51", + "\xfe\xb3\x61\x67\xea\xfc\x02\xc8\xe2\xbd\x6e\x13\x81\x76\x86\xba", + "\x08\xdb\x32\x7a\x88\xbe\x7b\x48\xf4\x30\xfd\x7b\xfc\xcd\xf5\x02", + "\xb7\xc2\x49\xf8\x10\xad\xac\xf9\x9a\xbd\xed\x1f\x3b\x91\x30\xf2", 1, + 64, 128 }, + { 96, 128, 128, 12, + "\xd9\x7c\x9b\x04\x3b\xdc\xcf\xd5\x94\x91\xa9\x95\xe7\x8f\x16\x96", + "\xef\x42\x32\x40\x35\x88\x30\xdf\x91\x55\x06\xa3", + "\x3d\xdb\xa7\xb3\xab\x69\xc8\xb2", + "\xf0\x47\x59\x4a\x5c\xff\xda\x64\x30\x3a\x80\xb2\xfa\x6a\x95\x71" + "\x69", + "\xe0\xca\xf2\xa9\xd5\x0f\x70\xec\xaa\x43\xb4\xa2\x87\xc3\xb3\x4a" + "\x99", + "\xcf\xf4\xc6\x18\x82\xb4\x13\xb6\x86\xff\x35\xb6\x3a\x3a\x73\xde", 1, + 64, 136 }, + { 96, 128, 128, 13, + "\x16\xbe\x38\xc0\x5c\x7b\xc5\xc6\x8e\xe6\x20\x38\x71\x79\x92\x40", + "\xac\xca\x8a\xe9\x16\x11\x9e\x49\xd8\x7c\x33\xa7", "\x28", "", "", + "\x21\x7d\x40\xef\xd9\x72\x70\x1f\xcc\x33\xdf\x53\x62\xe1\xea\x9c", 1, + 8, 0 }, + { 96, 128, 128, 14, + "\x7c\x89\x68\x0b\x4b\xca\x11\xa6\x43\x14\xf4\xca\xc5\x7a\x95\xdf", + "\x07\xc8\xef\x98\x1b\xea\x99\x52\x57\xd3\xd6\x5a", "\xb8\xe8", "", + "", + "\xde\xa6\x36\xde\xd8\xb9\xef\x2a\x08\xff\xdf\x58\xa0\x5b\x78\x71", 1, + 16, 0 }, + { 96, 128, 128, 15, + "\x43\x9f\xd5\xc3\xb7\x65\x87\xd5\xa6\x01\xba\x6e\xf8\xfa\xd2\x14", + "\xed\x1d\x31\x6d\x08\x34\xd1\x74\xc1\xb5\xb4\x38", + "\xea\xe2\x52\xf4\x2d\x2c\x71", "", "", + "\xe8\x53\x04\x26\xcb\xab\xf6\x36\x33\xff\x37\x31\x59\x24\x7e\x38", 1, + 56, 0 }, + { 96, 128, 128, 16, + "\x1a\x44\xf3\x55\x06\x88\xfd\xdb\xc1\xe5\x04\x1d\xc9\x89\x52\xc0", + "\x5d\x29\x04\x29\x8f\x66\x8b\xa9\x5e\xaa\x17\x97", + "\xd5\x59\x08\x95\x8b\x70\xab\xee\x81\x05\x4c\xdf\x3d\x3d\xf5", "", + "", + "\x5c\x71\xb4\xf0\x69\xcf\xa1\x3b\x76\x34\xdb\x4b\x13\xe7\xbe\x7d", 1, + 120, 0 }, + { 96, 128, 128, 17, + "\x7d\xb6\xd8\xe5\x8e\x3c\x55\x2a\x64\x45\x20\xaa\x80\x5e\x2f\x48", + "\xe9\x86\x93\xe9\xf6\x63\x2d\x11\x5b\x5d\x5a\x74", + "\x6f\xc1\xca\x24\xe6\x97\x86\xaa\x26\xbf\xb5\xd4\x6e\xf8\xcb\x56", + "", "", + "\xb9\xa8\xa6\xd4\x61\xa4\x41\xfb\xd5\xbb\x6a\x8a\xc0\xd4\x7e\x9d", 1, + 128, 0 }, + { 96, 128, 128, 18, + "\xde\x6e\xd1\x69\xd3\x96\xcf\xb7\x37\x8e\x89\x2c\x7f\xaf\x1d\x5d", + "\xef\xc1\x87\x02\x82\xe7\x7c\xa8\x06\x3f\x1b\xeb", + "\xee\xdf\x6e\x77\x6a\xd3\x7d\xc6\x10\x82\x5a\x61\x68\xe2\x13\x56" + "\xc2", + "", "", + "\x0d\x65\x0a\x97\x4e\xbe\xa2\x2f\xed\x07\x7d\x22\x9e\x0c\x9e\x65", 1, + 136, 0 }, + { 96, 128, 128, 19, + "\x6c\xf0\x95\x99\x18\x1c\x07\xae\xb2\x1d\x78\x20\xbf\x70\x65\x95", + "\x4c\x4c\x52\x5a\x8c\x7e\xe6\x87\x9a\xef\xa7\x9e", + "\xbd\x91\x39\x67\xdb\x07\xb9\xeb\x59\x07\xf0\xbe\x71\xce\x88\x6c\x41" + "\xff\x92\x3c\x29\x6c\x0e\xf3\xf7\x04\xe9\x8f\x64\x9e\x59", + "", "", + "\x80\x6e\x48\xe7\xd4\x52\xb6\x3b\x61\x26\xf5\x76\xef\xbd\xf4\xc4", 1, + 248, 0 }, + { 96, 128, 128, 20, + "\xef\x35\xb5\xc7\x97\xbb\x6b\xee\xdb\x51\x3b\xa3\xd8\xae\xbd\x25", + "\x05\x76\xa1\x01\x7a\xc0\x0e\x49\x11\x0c\x4c\xac", + "\xa3\x86\xd5\xc4\x4d\xe8\xc6\xa5\x06\x3a\xdf\x5b\xa9\xf0\xb7\x5e\x9a" + "\xd1\xf2\x39\xa5\x30\xdd\x76\xd7\x97\x55\x4d\x7b\x03\x7d\x7d", + "", "", + "\x69\x66\xa1\xcf\x57\x29\x33\x2b\x26\xfd\x3e\x38\x50\xb7\x48\x65", 1, + 256, 0 }, + { 96, 128, 128, 21, + "\x64\x9f\x3d\xfd\xdb\xf1\xaf\x60\x87\x67\x45\x68\xe2\xe6\xd7\xc3", + "\x6c\xa6\xf8\x7b\x7a\x85\x84\xdf\x4f\x46\x87\xb9", + "\x35\x31\x2c\xa2\x3e\x4e\xb3\x6c\xb0\xa6\x6c\x6f\x38\x6b\x8e\xc2\x9f" + "\x6d\x11\xe8\x2f\xbf\xca\xad\xfd\x6c\xbc\x9b\x59\xd5\x1a\x6c\x02\x70" + "\x86\x82\x74\xd9\x1f\x60\x97\x8d\x1f\x0f\x37\x28\x09\x30\xd3\xfd\xcb" + "\x3e\x90\xea\x46\x1e\xcc\xc8\x3f\xa0\xd9\x75\x54\x88\x16", + "", "", + "\x3d\xaa\x00\x03\xde\x38\x4d\x78\x44\x3f\xfd\x3a\x5e\xa4\x81\x79", 1, + 520, 0 }, + { 96, 128, 128, 22, + "\xa5\xb5\xb6\xba\xe4\x5b\x74\x1f\xe4\x66\x38\x90\x09\x8f\x32\x6a", + "\x4b\xad\x10\xc6\xd8\x4f\xd4\x3f\xd1\x3a\xd3\x6f", "\x30", + "\x12\x7b\x15\x00\x80\xec\x0b\xc7\x70\x4e\x26\xf4\xab\x11\xab\xb6", + "\x75\xe6\xff\xcb\x61\x14\x83\x3b\x67\xcd\x93\xbd\xf2\xc2\x2b\x55", + "\xc9\x0e\x18\xea\xf8\x10\xb7\xbc\xef\xe7\xa5\x26\xb1\x78\x3b\x20", 1, + 8, 128 }, + { 96, 128, 128, 23, + "\x0c\xec\xb9\xf5\x12\x93\x2d\x68\xe2\xc7\xc0\xbc\x4b\xd6\x21\xc8", + "\x21\x86\xa3\x09\x12\x37\xad\xae\x83\x54\x0e\x24", "\x74\x3e", + "\x43\x7a\xeb\x94\xd8\x42\x28\x3b\xa5\x7b\xb7\x58\xe3\xd2\x29\xf0", + "\x64\x6c\xef\x72\x90\x6e\x2b\x8f\x69\xac\x31\x34\xb4\x96\x59\x8e", + "\x9d\xab\x1e\xe9\x31\x4a\x04\x30\xab\xf5\x4c\x37\xc8\x8c\x79\x0f", 1, + 16, 128 }, + { 96, 128, 128, 24, + "\xa3\xfd\x2f\xdc\xce\x8a\x63\xbf\xe4\xeb\x2d\xb7\xe4\x2a\xdb\xe1", + "\x69\x0e\x7a\xd1\xe0\x5d\x0d\x4a\xb4\x55\x2c\xf7", + "\xab\x91\xec\x8c\xc7\x33\x73", + "\xbe\x02\x31\xb5\xc7\x86\x1f\x0a\xf7\xb6\x38\x14\x79\xd2\x5b\x77", + "\xa8\x84\xf7\x69\xfc\xc7\x27\x83\x9d\x59\x71\x1f\xa3\xcb\x5e\xe0", + "\xf2\x01\x7e\x3b\xd1\x0b\xb1\xb4\x3f\xdc\xc0\xfe\xef\xfc\x9c\x68", 1, + 56, 128 }, + { 96, 128, 128, 25, + "\x55\xe0\x4c\x12\x27\x80\xbe\x52\xed\x93\x28\x92\x80\x39\x00\x8c", + "\x0c\x90\x8e\x58\xcd\xda\xd6\x9d\xea\x1a\x32\xc3", + "\x25\x59\x17\x07\xc0\x04\xf5\x06\xf4\xb5\x1e\x85\xe2\x9f\x6a", + "\x26\xeb\x70\x67\x2e\xef\x03\x66\x7b\x34\xcc\x7d\x0d\xf0\x58\x72", + "\x89\x16\x6d\xcd\x7d\x74\xa4\x45\xdf\xd3\x52\x6c\x51\x80\xd8\x25", + "\x8b\x8e\xd5\xf9\x7a\x16\x88\x81\xc3\xb6\xef\xe9\x1c\xfe\x70\x43", 1, + 120, 128 }, + { 96, 128, 128, 26, + "\x5f\x0a\x1b\x5f\x8f\x86\x73\xd5\x66\xec\x7f\x54\xe7\xdc\xa4\xf2", + "\xc3\x09\x68\xc9\x67\xe5\x35\x05\x62\x16\x28\xdb", + "\xc0\x70\x92\xd7\x99\xda\xc2\xb4\xc0\x5f\xbd\xdd\x04\x74\x3c\x34", + "\xf6\x53\x84\x76\xda\xf0\x45\x24\xcf\x13\x43\x09\xdd\x84\xe1\x87", + "\x23\x15\x11\x0f\x7e\xc6\x4e\x7a\x23\xe5\xa7\x62\x82\x2f\x71\xab", + "\xdc\x7b\x12\xfa\x2d\xbf\xbd\xc6\xd8\x5f\xaa\x77\xa2\xeb\x76\x7e", 1, + 128, 128 }, + { 96, 128, 128, 27, + "\x67\x1a\x70\xe8\x83\xfb\x06\x11\xdf\xfd\x0b\x1d\xd9\xb8\xcc\xa2", + "\xa5\x1c\x37\xf4\x67\x89\x3c\x16\x08\xe5\x62\x74", + "\x3e\xa1\x2d\x80\xf4\x0f\x34\xf8\x12\x47\x9d\x2e\xcc\x13\xd2\xd6" + "\xdf", + "\x3b\xaf\x3e\xdf\x04\xdc\x0c\x97\xaa\xe0\x81\xcd\xeb\x08\x02\x1d", + "\x5d\x56\x30\xfc\x72\x8f\xfb\x08\xce\x69\x3f\x72\x99\xe6\x72\x8b", + "\x00\x02\x3f\x11\xa0\x23\xc0\x78\x6c\x10\x5f\xe4\xc0\x03\xaf\x6e", 1, + 136, 128 }, + { 96, 128, 128, 28, + "\x20\xbb\xf7\x4c\x1e\x63\x98\x2c\x47\x2c\x47\x43\x56\x9e\x4c\x84", + "\x45\x9f\xc7\xc0\x04\xbf\x46\x32\x3a\x02\xd8\x46", + "\x4f\x22\x85\xce\x3d\xaf\xa5\x28\xc6\x94\xa5\x27\x2d\x3b\x7b\x92\x90" + "\x97\xdb\x39\x87\x72\x65\x3b\xd9\xbb\xbd\xb3\xb2\xc8\xe1", + "\x6d\xb5\x09\x92\xe8\xfb\xbe\xe1\x5d\x49\x79\xd3\xe3\x22\xda\xcd", + "\x87\x03\xe4\x46\x97\x13\x8c\x58\x53\x2d\x97\xee\x99\x23\x1d\x94", + "\xf1\x4c\x2f\x39\xa4\x87\x1a\x4a\x16\xc4\x2f\x6f\xe8\x78\xde\xef", 1, + 248, 128 }, + { 96, 128, 128, 29, + "\x63\xf0\x31\x72\x50\x5d\x90\xe9\x49\x00\x12\x5c\xb8\xa4\xb0\xdd", + "\x52\xc2\x09\x79\xcd\xaa\xad\xe5\x73\xdb\xa6\x50", + "\x51\x89\xea\x6f\x39\xb2\xa7\x8c\x02\x02\xfd\xff\x14\x6c\x5c\xc6\xbd" + "\xc7\x49\x1d\x47\x86\xf8\x0c\x6c\x6a\xef\x65\x63\x4c\x05\xda", + "\x60\x2c\x98\x99\x7e\xe0\x3f\xd1\x1c\xe0\x0e\x92\xde\x19\x39\x77", + "\x55\x90\x15\x5f\x3e\x70\x1b\x4a\x96\x09\x89\xd0\x25\x1b\xac\x65", + "\xfd\x6a\x2c\x92\x73\xd1\x24\xb5\x55\x3b\xe4\x2e\x78\x93\x14\x65", 1, + 256, 128 }, + { 96, 128, 128, 30, + "\x5b\xf0\x08\xf6\xf2\x7c\xc2\x1f\x5a\xe8\x2f\xb7\x90\x7b\x1d\x92", + "\x58\x0a\xf4\x8b\xc1\x10\x86\x04\xd5\x55\x13\x43", + "\x48\x2d\xa2\x4b\xb4\xfb\x9e\xaa\x0d\xbf\x40\x37\x33\x59\x7f\x5b\x3e" + "\xe8\x33\x8b\x5d\x09\xa1\xd6\xf9\x07\x0b\xb0\x69\x26\x4a\xbb\xca\xcc" + "\x56\x57\xaa\x63\x53\xf1\x79\xd1\xbb\x4c\x7f\xa0\x05\x26\x78\x9e\xaf" + "\x08\xe0\xda\x25\x8c\xbd\xb3\x9e\x98\x77\xc6\x8b\x4a\x75", + "\xca\x89\xd6\xae\x28\x4a\xfb\x67\x92\xcd\x89\x4e\x07\xaa\x83\x36", + "\x1b\x89\xc6\xbc\xdd\xef\xbe\x92\x33\xee\x40\x93\x46\x8a\x5f\x61", + "\xa4\x9c\x77\x47\xda\xd4\x2d\xf6\xd7\x29\xa0\x1f\x4c\x50\xcf\x34", 1, + 520, 128 }, + { 96, 128, 128, 31, + "\x30\xd8\x69\x2e\xb3\xb6\x2d\xb6\x14\x4f\x74\xee\x9d\xec\x52\x96", + "\xfe\x9f\x6f\xb4\x41\x5c\xfb\x41\x89\xf9\xc7\x6d", "", "", "", + "\x80\x4f\x91\x5f\xc7\xfe\xa2\xca\x7d\x8b\xaf\x13\x50\xc5\x22\x7b", 1, + 0, 0 }, + { 96, 128, 128, 32, + "\x21\x51\x7f\xd9\xeb\xfd\x38\x7d\xff\x2a\x0c\x05\x18\xab\x82\x67", + "\x61\xf6\xc4\xec\x9e\x20\x91\xd4\xa0\x31\x80\x4a", + "\x76\xd3\x32\xba\x08\x1b\x3d\x3c\xfb\xa2\x71\x16\x7b\xa1\x08\xcd", + "", "", + "\x2c\xc4\xb9\x05\xa4\xd3\x9e\x35\xd4\xbe\xae\xbd\xed\x9b\x59\x66", 1, + 128, 0 }, + { 96, 128, 128, 33, + "\xa7\x16\xf9\x31\xc8\xf9\xd9\x77\xf7\xda\x85\x73\xbc\x65\xf2\xbf", + "\x91\x77\x36\x59\xad\xac\x8f\x12\xe5\x52\x63\x16", "", + "\x9c\x98\x03\x8c\x5e\x8d\x1a\xf5\x97\xb3\xb9\x18\x8b\x36\x24\xfb", + "\x99\xae\x76\xcf\xff\x55\x2c\xe3\x7b\x21\x0e\x26\xe8\x10\x78\x7c", + "\x2b\xda\xb5\xe6\xf0\x08\xb0\xcc\x75\x1d\x5b\x06\x74\x87\xeb\x2b", 1, + 0, 128 }, + { 96, 128, 128, 34, + "\x87\x5b\x0b\x4a\x84\x15\x05\x24\xeb\x1f\x50\xf9\xd8\xde\x13\x88", + "\xfd\xc5\x01\x4e\xd1\xad\x70\x61\x29\xd5\x73\x22", + "\x35\xa6\xd9\x82\x9c\x84\x49\xc4\x40\x2e\x38\x5c\xc5\xc6\xfe\x98", + "\x2f\x57\x5d\xfb\x2d\xbe\x9d\x23\x8d\xe5\x76\xfc\x63\xe4\xac\x32", + "\x34\x69\xc3\xff\x73\x8a\xa3\x2a\xac\xc1\xac\x48\xd8\x9b\x1d\x75", + "\xfa\x68\x72\x0a\x31\x71\xa5\x4c\x4b\x36\x90\xbf\xfd\xe7\xb6\x10", 1, + 128, 128 }, + { 96, 128, 128, 35, + "\xd6\x29\x8f\xff\x67\xdb\xa1\xec\x25\x03\x08\xe0\xbc\x5f\x4f\xae", + "\xd9\x6f\x9b\xbb\xfa\x14\xe9\x61\x6c\x45\x8d\xf5", "", + "\x8e\xeb\x44\x45\xa3\x4c\x81\xfd\xbc\x47\x8b\x83\xdf\x71\x11\x6c" + "\xe6", + "\x47\x68\x69\xa3\xdd\xb3\x86\xbf\x42\x47\x8d\x0c\x84\x17\x90\x45" + "\xbe", + "\x91\x36\xd9\x94\xda\xa2\x2e\xad\x4d\x08\x27\xe5\x82\x50\x01\xac", 1, + 0, 136 }, + { 96, 128, 128, 36, + "\xe3\xd3\xec\x41\xf2\x8e\xb3\x5f\xb5\x3f\x5f\xa9\x18\x04\xe0\x51", + "\x89\xd2\x70\xfc\x8b\x58\x3b\xc6\x31\xce\xfd\x39", + "\x3d\x2f\x45\x8c\x67\xc5\xb6\xc7\x94\xb1\xf1\x2d\xad\x40\x9e\x0f", + "\x84\x7a\xcf\x52\x19\x95\xb3\x3f\x8b\xc4\x74\xc8\xbe\xfb\xca\x3b" + "\xb2", + "\x28\xaa\xec\x53\x49\x3c\xd6\x25\x2c\xf6\x41\x0e\xd1\x41\xbd\xaf" + "\xb7", + "\x47\xbf\x3e\x16\xc2\x27\xca\x11\xfd\x68\xa1\x6d\x40\x7c\x2c\xc3", 1, + 128, 136 }, + { 96, 128, 128, 37, + "\xfa\xf3\x6a\x66\xf8\xe5\x4f\x2f\xb2\xa0\x2f\x3a\x30\xf0\x18\x0b", + "\x2f\xfa\x98\x2a\x47\x84\x79\x7c\xf4\x6b\x07\xab", "", + "\x50\xa5\x9e\xdc\x01\xb7\xbd\x0d\xb6\xec\x43\xfe\x23\xf7\x2e\x70\xed" + "\x4d\x42\x33\x7a\xb1\x92\x6c\xc6\x95\x6a\xa4\x4d\xbe\xbf", + "\xa7\x89\x90\x7a\xeb\x23\x44\xf0\x25\xb1\xb4\x26\xc9\xde\xe5\x2b\x10" + "\x6f\xf2\x11\x0c\xb2\x00\xcf\xb8\x5a\xea\x60\xfd\xdf\x6a", + "\x72\x2e\x5c\x45\x0c\x5e\xd9\x49\x28\x59\xa3\x23\x6a\x22\x0f\x76", 1, + 0, 248 }, + { 96, 128, 128, 38, + "\x2c\x9b\x9f\xf4\x7d\x74\x2c\x4a\xb2\x24\xe9\xca\x1e\xd5\x7c\x4c", + "\x91\x79\x62\xca\xf3\x93\x24\x41\xc2\x59\x28\x2f", + "\x72\x17\x5b\xdf\xdb\x4a\x23\xe9\x7f\xdc\xbd\x26\x3b\xaf\x43\x16", + "\xb5\x42\xc2\xf3\xf8\x16\x70\xdd\xf7\x4f\x15\x18\x4a\xb7\xde\x17\xe0" + "\x57\xcd\xe9\xee\xf9\x2b\xab\xdb\x83\x75\x00\x77\x4c\x19", + "\x32\x0a\xe0\xc1\x1e\x92\xd1\x0d\x5b\xf5\x48\x5c\x85\x4b\x2d\x8f\x63" + "\x18\xe3\x3f\x16\xb5\x20\xcf\xfd\x35\xad\xa3\x81\xc9\x67", + "\xa4\x86\x69\x08\xe6\x64\xee\x14\x0c\x6a\xe2\xb9\xd2\xab\x84\x16", 1, + 128, 248 }, + { 96, 128, 128, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3d\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3e\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\xbc\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6c\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\xe6\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x42\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x41\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\xdb\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb1\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\x30\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x14\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x06\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x96\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x95\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x17\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x9b", 0, + 0, 128 }, + { 96, 128, 128, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x98", 0, + 0, 128 }, + { 96, 128, 128, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\xda", 0, + 0, 128 }, + { 96, 128, 128, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\x5b\xb0\x34\x07\x72\x97\xf0\x92\x1a", 0, + 0, 128 }, + { 96, 128, 128, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3d\x6d\x5f\x66\x43\x0a\xd6\x5b\xb1\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\xe6\x43\x0a\xd6\xdb\xb0\x34\x07\x72\x97\xf0\x92\x9a", 0, + 0, 128 }, + { 96, 128, 128, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3c\x6d\x5f\x66\x43\x0a\xd6\xdb\xb0\x34\x07\x72\x97\xf0\x92\x1a", 0, + 0, 128 }, + { 96, 128, 128, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\xc3\x92\xa0\x99\xbc\xf5\x29\xa4\x4f\xcb\xf8\x8d\x68\x0f\x6d\x65", 0, + 0, 128 }, + { 96, 128, 128, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 128, 128, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 128, 128, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\xbc\xed\xdf\xe6\xc3\x8a\x56\xdb\x30\xb4\x87\xf2\x17\x70\x12\x1a", 0, + 0, 128 }, + { 96, 128, 128, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3e\xe9\xf3\x43\x0f\x3e\x80\x3c\x0a\x46\xb7\xa8\x4c\xd8\x03\xde", + "\x3d\x6c\x5e\x67\x42\x0b\xd7\x5a\xb1\x35\x06\x73\x96\xf1\x93\x9b", 0, + 0, 128 }, + { 96, 192, 128, 66, + "\x50\x19\xeb\x9f\xef\x82\xe5\x75\x0b\x63\x17\x58\xf0\x21\x3e\x3e\x5f" + "\xcc\xa1\x27\x48\xb4\x0e\xb4", + "\xff\x0d\xdb\x0a\x0d\x7b\x36\xd2\x19\xda\x12\xb5", "", "", "", + "\x2d\x03\xf5\xe8\xc2\xe5\xa1\xb4\x3c\x77\x08\xdd\x0c\xbf\x0a\xcd", 1, + 0, 0 }, + { 96, 192, 128, 67, + "\x21\x21\x8a\xf7\x90\x42\x8f\x80\x24\xd3\xe7\xe1\x42\x8c\x9f\xcf\x57" + "\x8c\x21\x66\x36\xd6\x0e\x73", + "\x34\x04\x7b\xc3\x9b\x9c\x60\x83\x84\xdf\xf5\xb8", "", "\xe3", + "\x39", + "\x74\x50\xf5\x5a\x21\xe7\x17\xa1\x10\x6e\xa0\xc1\x18\x71\xf5\xff", 1, + 0, 8 }, + { 96, 192, 128, 68, + "\x3a\x8b\xf5\x43\xc4\x80\x92\x56\x32\x11\x82\x45\xbc\xbf\x5d\x01\x52" + "\x2b\x98\x7a\x31\xa3\x3d\xa3", + "\x4e\xbc\x13\xcf\x46\x36\xcc\x7c\x45\xe5\x60\xa7", "", + "\x53\xfc\x72\xe7\x1b\x59\xee\xb3", + "\x5d\x24\xd0\xe1\xa2\xee\x9f\xce", + "\xe7\x70\xf9\x1a\x51\xf5\xb5\x87\xa4\x4c\xd9\xd3\x63\x4b\x97\x06", 1, + 0, 64 }, + { 96, 192, 128, 69, + "\xbc\xb6\xbc\x5e\xe6\x74\x3d\xf1\x39\x6a\x34\x63\x93\x27\xb2\x58\x09" + "\xec\x9c\x81\xdd\x6a\x0c\x0e", + "\xbe\x03\x26\xd2\x3b\xdc\x2c\x64\x64\x8d\x13\xf4", "", + "\x80\x47\x4a\x3a\x3b\x80\x95\x60\xee\xe2\xce\x7a\x7a\x33\xea\x07", + "\xdb\x58\x93\xdc\x8d\xa3\x36\x61\x4a\xa0\xff\x76\x8d\x46\x95\x35", + "\x90\x2c\x2a\x83\x25\xcb\x55\xbc\x95\xf0\xe1\x3c\xaf\xe9\xaa\x8d", 1, + 0, 128 }, + { 96, 192, 128, 70, + "\x5e\x1d\x28\x21\x3e\x09\x25\x36\x52\x5b\xba\xe0\x9e\x21\x4a\xf4\xc8" + "\x91\xe2\x02\xb2\xb4\xfa\x4f", + "\xb6\xbe\x6c\xd0\x68\x12\x35\xd8\x26\xaa\x28\xea", "", + "\x53\xd5\x94\x33\xa7\xdb\x7f\x41\xb3\x1c\xcb\x6d\x4a\x2d\x78\x99" + "\x65", + "\x20\xeb\xc1\xf5\xa2\xc9\xf8\x8d\x1c\xdb\x18\x2e\x81\x32\x9c\xc0" + "\x3e", + "\xbc\x54\x5e\x91\xc9\x74\xa7\x44\xba\xea\xb2\xdd\x8c\xe6\x09\x60", 1, + 0, 136 }, + { 96, 192, 128, 71, + "\x7f\x67\x2d\x85\xe1\x51\xaa\x49\x0b\xc0\xee\xc8\xf6\x6b\x5e\x5b\xee" + "\x74\xaf\x11\x64\x2b\xe3\xff", + "\xb0\x22\x06\x70\x48\x50\x5b\x20\x94\x62\x16\xef", "", + "\xef\x64\x12\xc7\x2b\x03\xc6\x43\xfa\x02\x56\x5a\x0a\xe2\x37\x8a\x93" + "\x11\xc1\x1a\x84\x06\x5f\x80", + "\x1b\xc9\x90\x29\xa0\x9c\x08\x01\x40\x60\x8a\x62\xc3\x3b\xc7\xae\x69" + "\xff\x81\x1f\xef\xb2\x0b\x2d", + "\x80\xf0\x91\x03\x48\x5f\x95\xf8\x6a\xd1\xf0\x72\xa2\x14\xc5\x5e", 1, + 0, 192 }, + { 96, 192, 128, 72, + "\xf7\xac\xe6\xc3\xc1\x0c\x3f\xf9\x77\xfe\xbe\x7d\xc8\x82\xb8\xe7\x79" + "\xef\x3a\x17\xef\x93\x24\xa8", + "\x6e\x2b\xa2\x83\x3c\x5d\xce\x6b\xec\xc4\xf6\xd8", "", + "\x2e\x11\xe4\x19\x51\xc2\x04\x60\xc7\x68\xb0\xd7\x1a\xd5\x6e\x77\xbe" + "\xc0\x5e\x04\x78\xf9\x9d\x5b\x62\xe7\x99\xf7\x32\xe4\x67", + "\xb3\xce\xc7\x77\xf8\x07\xd1\x6b\x69\x71\x63\xd0\xc6\xa4\x5d\x00\x29" + "\x36\x71\x4d\x60\x0a\x15\x6d\x7e\x53\x65\xd1\xaa\xca\xd0", + "\x11\xa5\x6e\xdb\xe2\xfb\xbb\xb2\xb0\x11\xc4\x3a\x62\x00\x08\x30", 1, + 0, 248 }, + { 96, 192, 128, 73, + "\xa9\x54\x1a\x96\xb8\x6d\x32\xb4\x52\x09\x2e\x8b\x92\x09\x9e\xa3\xf4" + "\x5f\x98\xca\x05\xce\x69\x2b", + "\x90\x15\xb4\xbc\xd6\x98\x90\x83\x04\x6b\xe8\x6d", "", + "\x9d\x35\x9a\xad\x3f\xf5\xce\x37\x35\xa8\xcf\xfe\x4f\x08\x71\x14\xd4" + "\xd6\xc5\xe0\x1d\xce\xb1\x96\x9f\x40\xc8\xe0\xdb\x6b\xb9\x02\x81", + "\x0a\x6b\x84\xde\x44\xcc\xe1\x42\x55\xa9\xcb\x19\x16\x96\x95\xcf\x46" + "\x60\x48\x9f\x1e\x3f\x60\x53\x34\x35\x58\x28\xc5\xc0\x9f\xb3\x0e", + "\xa1\x9a\x60\x10\x5f\xc7\xa0\x3b\xe1\x78\x3f\x55\x8e\xa2\x3e\x9c", 1, + 0, 264 }, + { 96, 192, 128, 74, + "\x23\x91\x95\xb5\x86\x68\xeb\x89\x63\x6b\x1e\xc2\xb3\x31\x33\x69\x46" + "\x36\x9f\xc6\xc8\x7b\x88\x49", + "\x14\xa6\x28\x1a\x43\xb4\xeb\x05\x6a\x67\xb9\xe6", "", + "\x39\xd8\x73\xd4\xca\xd7\x1c\xb2\x52\x78\x4b\xd1\x46\x48\xa4\x94\xce" + "\xb5\x17\xeb\x9e\x3e\x6f\x32\xd1\x9b\xd1\x8d\xfa\xf8\x77\xc7\xae\xc2" + "\x21\x03\xd2\x42\x99\x3e\xd7\xba\xb1\x23\x32\x61\x10\xdf\xdb\x72\x29" + "\x14\x3a\x0c\x60\x1e\x16\xaa\x4e\xcd\xde\x80\x8c\xd8\x3b\xb2", + "\x8f\xce\xa9\xe2\xfa\xa5\x23\x29\x84\x72\xb5\x58\x3e\x35\x6d\x18\x75" + "\x39\x3e\xa3\xbc\x1b\x4f\x8e\xa4\xaa\xd5\x97\x14\x7a\x7c\xa9\x4e\x26" + "\x09\xfe\x6b\xf0\xab\x86\x1e\x06\x31\xa3\x12\x4e\xb1\x5d\x0d\xe2\x65" + "\xef\x11\xa3\x3e\x45\x07\xe3\x07\x70\xce\x37\xbb\xb4\xb6\xc3", + "\xa4\x45\x68\x28\xb4\x9c\xdb\xf8\xf3\xc2\x00\x42\x9c\x33\x9a\x89", 1, + 0, 528 }, + { 96, 192, 128, 75, + "\x03\xb4\x67\x58\x51\xb7\x8b\x69\xfb\x7b\x55\x89\x88\x2e\x71\x8b\x07" + "\x5e\x9a\x54\x02\xb5\x20\xfb", + "\xc4\xca\x2d\x67\x8e\x51\x74\x2e\xc5\xe5\x60\xab", + "\x91\xe1\x0a\xc5\x63\x6f\xe9\x9b", "", "", + "\x93\x7f\x15\xff\xd1\xcc\xd6\x45\xd9\xc7\xcd\xd6\x67\x73\x11\xcd", 1, + 64, 0 }, + { 96, 192, 128, 76, + "\x40\x0e\xec\x9b\x06\xa8\x0a\x84\x03\xd4\x5d\xae\x5d\x58\xcc\x91\x7b" + "\xc8\x54\xf5\x1c\xd3\xce\x0d", + "\x44\x7d\xd0\x9a\x23\x70\x8f\x3b\x66\x64\xe1\x5b", + "\x73\x20\x36\x7d\x5b\x07\x05\x59", + "\xb7\x84\x92\x5a\x69\x5f\x0e\xd1\x4c\xa4\x02\x49\xc1\xfd\x5d\x1a", + "\x91\x2d\x05\xc4\x02\x38\x39\x50\xe1\xc5\xa5\x18\x8e\x62\x41\xd8", + "\xab\x30\x9b\xe2\xc0\x5c\x94\x1f\xbf\xb3\x38\xba\x06\x4b\x19\xa1", 1, + 64, 128 }, + { 96, 192, 128, 77, + "\xe2\x58\xb1\x17\xc2\xfd\xd7\x55\x87\xf0\x7b\x40\x0a\xe4\xaf\x3e\x67" + "\x3a\x51\xdc\xf7\x61\xe4\xca", + "\x5e\xad\x03\xaa\x8c\x72\x0d\x21\xb7\x70\x75\xdb", + "\x27\x70\x29\x50\x96\x0b\x9c\x79", + "\xaf\xe9\x61\x13\xa6\x84\xbc\x52\xa6\xd9\x62\xcf\x27\x24\xf6\x79" + "\x1d", + "\x78\x30\x44\x6f\x33\x30\x57\xd9\x96\xa1\xa7\x9b\x21\xc6\x8d\x8b" + "\x43", + "\x72\xac\x47\x8a\x66\xf5\x63\x75\x63\xf1\xf1\x2c\x1d\x02\x67\xca", 1, + 64, 136 }, + { 96, 192, 128, 78, + "\x6c\x2a\x03\xe9\xed\x8e\x42\x1e\x07\xdf\xc3\x6b\x99\xc0\xd0\xdc\x9b" + "\xb8\x74\xea\x3a\xf8\xa8\xb7", + "\x8f\x01\x5e\xce\x4e\x03\x38\xe7\x82\xfa\x3a\x2f", "\xf1", "", "", + "\x92\x26\xc4\xc3\x91\x66\xdf\x5a\xf4\xe0\xc9\x1b\x64\xb4\x63\xa2", 1, + 8, 0 }, + { 96, 192, 128, 79, + "\x14\x86\xb5\xf1\x50\x52\x4c\xc6\x01\xb2\xea\x7d\xa4\x7d\x7c\x8a\xfb" + "\x06\xd6\x42\x0d\xd3\x3f\x8d", + "\x93\x07\x31\x7d\x2f\x42\x3b\x57\xb3\x72\x0f\x8f", "\x3c\x09", "", + "", + "\x36\x00\xe0\x6d\xef\x58\x5e\x20\x12\x35\x0e\xfe\x04\x78\x26\xe9", 1, + 16, 0 }, + { 96, 192, 128, 80, + "\xe0\x9c\x83\xff\x0f\xc0\xb6\xa3\x0f\x93\x8e\x50\xe2\x66\x85\x24\x7e" + "\x9d\xed\x2e\x7d\x6d\xbc\x7c", + "\x3e\xc6\x1e\x9c\x16\x6d\x67\x83\x99\x23\x91\x52", + "\xc1\xa1\x3c\x74\xc1\x1c\xb8", "", "", + "\x9f\x5b\x3e\x48\xdd\xda\x9a\xf3\x75\x15\x01\x50\x9c\x94\x0a\xc1", 1, + 56, 0 }, + { 96, 192, 128, 81, + "\xd0\xc6\x88\xc5\x25\x80\xd8\xf8\x00\xac\xa3\x4f\xa7\x4c\xec\x48\x7b" + "\x67\x1a\xaf\x85\x02\x7b\x9b", + "\x9c\x46\x0a\xbf\x56\x29\x2d\xcb\x1b\x35\xb3\xb4", + "\x35\x96\xce\x98\x9f\xf9\x75\xf3\x25\x0e\x6c\x9e\xce\xd2\x5b", "", + "", + "\xcd\xa4\x34\xc2\x29\xe5\x4b\xf9\xbf\xd5\x4c\x8d\x8c\xe4\x73\x0c", 1, + 120, 0 }, + { 96, 192, 128, 82, + "\x18\x92\xdb\xd7\xe6\xb3\xfe\x18\xeb\xdc\x81\xbb\x27\x1a\xb0\x3a\x8f" + "\x32\xaf\x04\xf1\x33\x00\xd2", + "\x0e\x87\x2d\xe5\x8a\xd1\x0d\xa2\x48\x40\x3f\x21", + "\xe8\xb1\xc6\xcc\x6c\x45\x10\x5e\x0c\x32\x58\x7a\x0d\xe3\x69\xe3", + "", "", + "\x0c\x7e\x14\xdc\x49\xa8\x1e\x6b\xe1\x23\xb9\xcf\xbb\x28\x17\x87", 1, + 128, 0 }, + { 96, 192, 128, 83, + "\xef\x79\x92\xb0\xf8\xec\x7a\x10\x1d\x34\x00\x01\x00\xeb\x7d\x9b\x2e" + "\xae\xe3\x33\xd0\xaa\x2f\xf8", + "\x31\x6d\x38\xa9\x00\x19\xb9\xa3\x7a\xd0\x80\xb7", + "\x10\x14\xed\x78\x89\x69\x4c\xff\x76\x78\x76\xc0\x69\xae\x1f\x91" + "\x85", + "", "", + "\x17\xff\x8e\x79\x97\x60\x55\x8f\x1d\x4c\xf8\x92\x7d\x5e\xc6\x99", 1, + 136, 0 }, + { 96, 192, 128, 84, + "\xe4\x57\x08\x15\xa1\x49\x59\x9d\x13\xbd\x8d\xca\xad\xbe\xc9\x3c\xf0" + "\x90\x19\xba\xa2\xd4\x07\x0b", + "\x5a\x31\xa3\xa0\x26\x78\x6c\x49\xdb\x9d\x09\x58", + "\x90\xa7\x35\x75\x19\xe3\x5e\x8d\xbd\x89\x76\xd4\xb3\x67\x10\xff\xc1" + "\xeb\x0d\x9a\x4a\xe7\xd5\x31\x5a\xe7\x32\x4e\xb1\xd1\x8c", + "", "", + "\x06\xcc\xbf\x76\x7e\x0a\x63\xc8\x9d\x50\xb8\x14\x11\x87\xa5\x55", 1, + 248, 0 }, + { 96, 192, 128, 85, + "\x9f\x0c\x07\x6b\x06\x30\xca\xa1\x0e\x7b\xdc\x07\xdc\xdc\x89\xa2\x70" + "\xf0\x39\x30\x99\x7a\xde\x0b", + "\x3f\x5d\xef\x08\x80\xb8\x89\xdb\x0b\x3f\x2b\xf0", + "\xf5\x36\x8b\x9d\x8f\xdc\x1e\xfa\xb2\xb1\x7a\x45\xf4\x60\x42\x45\x98" + "\x35\x72\xf8\xc1\x67\xaa\x31\xfa\x3f\x53\x0f\x1c\x5e\x17\x81", + "", "", + "\x33\x4c\x13\x25\xfa\x96\x9a\x07\x17\x90\x11\xd2\xf8\x61\x36\x36", 1, + 256, 0 }, + { 96, 192, 128, 86, + "\x80\x3f\x4e\xbb\xed\x8b\x1a\x4f\x34\x87\x13\x46\x1c\x0e\xb0\xbd\x30" + "\xca\xec\x55\xa1\xe7\x16\x28", + "\xb0\x5e\xc4\x9b\xc4\x05\xeb\x7e\x97\x29\x4f\x19", + "\xdf\xb7\x1f\x25\xe7\xf1\x1c\xca\x17\x70\x2e\xb8\x9a\x18\x4e\x57\xf2" + "\x2e\x4e\xa4\x74\x1f\xf6\x03\xab\xc9\x01\xfa\x02\x6b\xde\x7c\xe1\x10" + "\x7e\x2f\xfb\xa0\xa0\xa0\xf2\x4f\x47\xee\x62\x78\x32\xee\x5b\xc2\x19" + "\x2c\x18\x84\x56\x30\x00\x99\x10\xc0\x7f\x8d\x0a\xb4\x51", + "", "", + "\x2b\x71\xd2\xa8\x1f\x4c\x6e\xa2\x67\xa9\x86\x50\x94\xfe\x20\xe9", 1, + 520, 0 }, + { 96, 192, 128, 87, + "\xda\x6e\x3d\xf6\x73\x5f\x63\x2e\x03\x5a\xb8\xf1\x0c\x37\xb5\xf0\x0a" + "\x40\xe1\x8b\x17\x77\x8a\x85", + "\x80\x76\x5e\xac\x22\x81\x96\x9c\xba\x56\x9c\xe7", "\x6d", + "\x3d\x87\x65\xdf\x3a\x06\xf5\x24\x8b\x1a\xaa\x54\x12\x3b\x86\xbd", + "\xa4\x28\x63\xb9\x5a\xbe\xa3\x91\x94\x0a\xda\xc7\xfe\x0c\x41\x43", + "\x12\x2b\x46\xf8\x1a\x0b\x6e\x92\xcd\xa1\x95\x0c\xe6\xfe\x02\x6e", 1, + 8, 128 }, + { 96, 192, 128, 88, + "\x54\xe1\x71\xcf\x90\x72\x9c\x77\xd5\x00\xe1\xd2\x53\x33\x60\xe8\x41" + "\xe2\x60\x89\x45\x76\xb1\x29", + "\x8b\x1e\x57\xf9\x8e\xa4\xe7\x7d\xea\xe4\x57\x6c", "\x27\xff", + "\x2d\x71\x63\x66\xf2\x87\x38\x60\xd5\x04\x37\x00\xf1\xe9\xa9\xd8", + "\x34\x18\x15\x4a\xfc\x95\xd0\x55\x69\xee\x6f\xe4\xdb\x82\xeb\x6e", + "\x0a\x73\x4b\xac\x17\x84\x3b\x85\x73\xfb\xfe\xba\x4a\x5f\xc5\xd4", 1, + 16, 128 }, + { 96, 192, 128, 89, + "\x99\x2c\xab\xb6\x4f\x12\x21\x0c\x8b\x0b\x14\xd7\x3d\x39\x31\x7d\xdb" + "\x2b\x8a\x62\x8a\xc3\x51\x06", + "\x57\xc8\x78\x6e\x66\xd8\xb0\xbe\xc3\x36\x04\xb0", + "\x06\x8d\x94\x0e\x26\xb6\x78", + "\x9e\x76\x92\xf1\x21\x32\xcd\xd5\x3f\x50\x53\x16\x51\x41\x7b\xd2", + "\xba\x34\x09\x26\xde\x50\x0d\x01\xae\x3d\xff\x2e\x90\x56\x08\x16", + "\xdb\x9d\xba\x31\xa0\x30\x19\xce\x88\xce\x74\x1c\x03\x94\x06\x60", 1, + 56, 128 }, + { 96, 192, 128, 90, + "\x89\xbe\x64\x95\xc9\x17\xbd\x7a\xf0\xa3\xb7\xa6\xc8\xa4\xc6\xb5\xca" + "\xde\x76\x6d\x32\xde\x36\x04", + "\xf4\x7f\x6b\x65\xd6\x60\xf1\x0c\x04\x3e\xa6\x41", + "\xf6\xa2\x8d\x27\x68\x6a\xdc\xbf\x9f\xf8\xab\x80\xec\xc1\xc1", + "\xc3\xc5\x0f\x4b\x38\xaa\x37\x51\xf4\x91\x0a\x44\x67\x5d\x37\xe5", + "\xb1\x9b\xe8\xba\x6e\xbc\xdb\x74\xd3\x3c\x2d\xbd\xfd\xc0\x20\x74", + "\x93\xbb\x71\x70\x61\x33\x0a\xcc\xc7\x18\xb4\xa8\xef\x05\xab\x20", 1, + 120, 128 }, + { 96, 192, 128, 91, + "\x8e\xa9\x54\x55\x24\x17\x51\x6c\x97\x2e\x43\x11\x69\x2d\x65\x8d\xd7" + "\xac\x9a\x7f\xd6\xf3\xd0\x2c", + "\xea\x16\xc1\x04\xbc\xe5\xb7\xed\xd5\xa2\x5a\x46", + "\x7d\x4d\x7c\x27\x3a\x9a\xa0\xf3\x5d\x1f\x91\x57\x01\x41\xdb\x54", + "\x0f\xc6\xcc\x80\x0a\x57\x86\xe6\x3a\x45\x46\xfb\x33\x88\x7a\xf9", + "\xeb\x45\xae\x29\x80\xd9\x96\xf5\x02\x35\x93\xf7\x62\xd3\x70\x51", + "\x4a\xcc\x06\x24\x2c\xc4\xec\x4b\xa3\x6e\x8b\xfb\xe8\x4f\x3d\x5d", 1, + 128, 128 }, + { 96, 192, 128, 92, + "\xdf\xa7\xef\x72\x30\x2d\xfb\xcd\x26\x48\xb8\x89\x58\xfe\x0f\x04\x9f" + "\x1d\x60\x14\x3d\x86\xe3\x95", + "\xf3\x72\x3b\x9a\xb7\x28\xc9\x9b\xbd\x6f\x23\x04", + "\xcf\x75\xdd\x45\x36\xd0\x0f\x11\xed\xa4\x0d\xb4\xd2\x52\xe1\x72" + "\xe3", + "\x20\xb3\x88\x32\x44\x30\x0a\x82\x09\x4d\xdb\x9b\x3d\x1e\xfb\x81", + "\x5a\x83\x74\x39\x1a\xdc\x22\xe8\xc0\x66\x55\x7d\x9f\xf5\x86\xcf", + "\xd4\xa1\x11\xd6\x11\xef\xa6\x3f\x0c\x3f\x08\xb2\xfb\xb3\xb0\xac", 1, + 136, 128 }, + { 96, 192, 128, 93, + "\xb6\x3b\x52\xd1\xd1\x59\xa1\x75\x96\xdf\xbd\x9b\xe5\xc5\x08\x86\x99" + "\xd9\x4b\x9c\x5d\x95\xc2\x2d", + "\xb1\xbe\xee\x8a\xfb\x00\xe0\x1a\x9c\xbf\x59\x73", + "\x90\x20\xb5\x62\x56\xbc\xb0\x2c\x69\x07\x20\xe3\x23\x9d\x32\x5d\x25" + "\x9f\x18\x98\xea\x05\x17\x0e\x31\x5c\x14\x49\x60\xd2\x63", + "\xdf\xb4\x17\x90\x3c\x6d\x48\x27\x50\x0a\x3e\xca\x21\x84\xfe\x1c", + "\xe8\x44\x71\x60\x1f\xb8\xb4\xdb\xbd\xc8\x0d\x56\xe3\x7f\x69\xb8", + "\x52\x08\x9c\xb4\xe6\xc1\x1b\xd7\x64\xae\x7d\x44\x38\xcf\xd1\xf8", 1, + 248, 128 }, + { 96, 192, 128, 94, + "\xaa\xf6\x40\x23\xf5\x76\x2c\x4a\x54\xc3\xeb\x0e\xf3\xbb\x2e\xbd\x23" + "\xce\xaf\x38\xb3\xe0\x28\x5f", + "\x2a\x56\x23\x5f\x07\x9d\x53\xa3\x10\x07\x02\xd4", + "\x2d\x68\x8b\x3b\x33\x11\x77\x05\x79\xdd\x06\x46\x14\xa7\xd1\x1b\x17" + "\xa1\x69\x53\xbd\x97\x00\x75\x9b\x35\xa5\x03\x1a\x2d\x8b\xd0", + "\x34\xae\xc5\xa6\x57\x95\xcb\xee\xda\xc2\xee\x6f\xd7\x76\x5c\x6f", + "\xd1\x3c\xce\xb8\x37\x6e\xb4\x23\xf5\xb2\xea\x25\x7c\x11\x8f\xb5", + "\x7b\x6a\x3c\x76\xfd\x52\x93\x0a\x98\x9f\x03\x4e\x5d\xfe\x07\x4b", 1, + 256, 128 }, + { 96, 192, 128, 95, + "\xb5\x21\xe4\xfa\x92\xeb\x46\xfd\x49\x16\xc7\x1e\x3f\x99\x90\x27\xaa" + "\xf2\x14\x66\xfa\xd5\xf9\x6e", + "\xb2\xb4\x2f\xa6\x0a\x2a\x80\x41\x2e\xec\xc7\xfc", + "\xb4\x97\x22\x1c\x7f\xad\x55\xa0\x6e\xa9\xf5\x6f\x39\xb3\x60\x93\x30" + "\xed\xc4\x67\xb7\x9c\xbf\x33\x53\x63\x6b\xce\xb7\x84\xb6\x0e\xc6\x3a" + "\x83\x60\x74\xce\xb4\x86\x24\xa4\xa4\x1a\xc0\x49\x6d\x5a\xdf\xe2\x98" + "\x93\x13\xd7\x41\x2b\x9c\x2d\x89\xca\xfd\x9c\xd5\xa7\x34", + "\xa3\xa8\x0a\xc0\xf5\xb4\x65\x97\xa7\xf4\xa5\x83\xdd\xa0\x21\x24", + "\x99\x4b\xb3\x44\x70\xd4\xdd\xb7\xbb\x7a\x3c\x3a\xbb\x5d\xa3\xc5", + "\x02\x55\x0e\x2b\x27\x8c\x72\x36\x72\xa0\x1e\x4b\x6a\x46\xaf\xc7", 1, + 520, 128 }, + { 96, 192, 128, 96, + "\x77\x57\x75\x4a\xec\xcf\x85\xc9\x1e\x48\xe4\xd4\x97\x0d\x4d\x62\xda" + "\xe9\x4c\xf4\x4f\x9f\xf0\x26", + "\x3d\x92\x81\xc4\xac\xfc\x72\x38\x73\x46\xfd\x92", "", "", "", + "\xda\x9a\xe6\x0a\x12\xaa\x6f\x92\x6c\xf4\x6d\x2a\x33\x5f\xaa\xc7", 1, + 0, 0 }, + { 96, 192, 128, 97, + "\xa0\xca\xc7\xe8\x3c\x7e\xba\x22\x36\x52\x56\xcb\x8f\x23\x70\x39\xb9" + "\x4f\x1a\x26\x92\x63\x64\x8e", + "\x2d\x2a\x5c\x8b\x17\x21\x2d\x4c\x44\xce\xd4\x59", + "\x8b\x38\x8e\x2e\x72\x25\xc0\x87\x30\x00\x42\xf6\x02\x4a\x11\x1f", + "", "", + "\x84\x04\x34\x98\xce\x07\xd8\x74\x23\x30\xc6\x05\xd9\x6d\x96\x6c", 1, + 128, 0 }, + { 96, 192, 128, 98, + "\xfa\x07\xaa\x39\x32\xb9\x01\x69\x62\x69\xc8\xf8\xbf\x56\x66\x2f\x82" + "\xdf\xf2\x95\x7a\x4a\xca\x35", + "\xec\x1b\x6d\x7f\x09\x7a\x2c\xad\x8c\xbc\x9f\xe9", "", + "\xd9\xa6\x89\x79\x3c\x94\x79\x68\xf0\x7d\x4b\xa2\xeb\x1c\x25\xeb", + "\xa3\x22\xf7\x58\x00\xfc\xaf\xf6\x91\x25\x17\x62\xed\x39\xfb\x39", + "\x8a\x73\x48\x92\x21\x33\x83\xac\x54\xdc\x2c\x1f\x48\x84\x19\x38", 1, + 0, 128 }, + { 96, 192, 128, 99, + "\xf9\x97\xa7\x9b\x63\xb1\xcf\x64\x14\x83\x70\x60\x97\xff\x4a\xbe\xeb" + "\xa1\x39\x62\xdb\x05\x62\x06", + "\xba\x95\x38\xad\x15\x75\xa1\xdf\x78\x79\x78\x2b", + "\xaa\x13\xc1\x09\xb2\xf5\x7f\x70\x0a\x89\x93\x1d\xe7\x5b\x70\x80", + "\xf3\x76\x94\x34\x59\xb6\x04\x1e\xd5\x23\x2d\x7b\x9f\xb5\xe9\xe6", + "\x98\x2b\x14\xf6\x64\x82\xd0\xc9\x83\x71\xe0\x80\x78\xef\xa0\x12", + "\x14\xf1\xb2\xb2\x45\x75\xa1\xa3\x32\x06\xac\xdf\x50\x0e\x9d\x46", 1, + 128, 128 }, + { 96, 192, 128, 100, + "\xd7\xc6\xea\x0a\x28\x5a\x5d\x8c\x59\x64\x77\x30\x80\x48\x89\x67\xe7" + "\xe6\x59\x35\x89\x0c\x32\x65", + "\xf5\x11\xd1\x6e\x97\x2e\x13\x8d\x5a\xe8\xdd\xac", "", + "\x50\x34\xfa\x6d\xa3\xa9\xee\x38\x0b\xe7\xe8\xd0\x26\x05\xac\x20" + "\x23", + "\x34\x50\xa9\x92\xa6\xfb\xce\x9c\xe2\x9f\x6c\x4f\x9f\x41\xc3\x6e" + "\xf6", + "\xed\x6c\xa1\xff\x3e\xa1\xa7\xca\x88\x19\x50\x11\x39\xf8\xa0\xb8", 1, + 0, 136 }, + { 96, 192, 128, 101, + "\xb2\x93\x46\xa9\x5c\x3b\x65\x3c\x9b\xed\x02\x3d\xf2\xe0\x3b\x6d\xe4" + "\x5b\x8d\xe1\xa4\x06\x7d\x86", + "\xc5\xb4\x5d\xf3\xa5\xbf\x4e\xf5\x39\xc3\xdb\xd8", + "\x40\x40\x59\x18\x9f\x1e\xaf\x31\xb2\xe5\x05\xfe\xc0\x8c\x70\x53", + "\x22\xe6\x28\x1f\xba\x3e\x5b\x05\x68\x71\xa9\x8d\xd2\xef\x0e\x16" + "\x4d", + "\x6c\x26\x39\x28\x99\xe3\x65\x45\x00\x54\xbf\x0a\xb3\x3f\x98\x3f" + "\x27", + "\xc4\x72\x40\xbb\x8d\x1d\xbb\x68\x7f\xab\x77\x7f\x72\xad\xbd\x2a", 1, + 128, 136 }, + { 96, 192, 128, 102, + "\xa6\x1e\xd3\xb8\x1d\x78\x56\x06\x02\x77\x74\x07\x75\x9b\x1f\x2c\xa3" + "\x4c\xd7\x0c\x6b\x57\x79\x1b", + "\x07\xb8\x24\x97\xb8\x15\xd1\x61\x82\x48\x10\x45", "", + "\xdd\xc1\x86\x2e\x35\x31\x62\x2e\x69\x83\x22\xf0\xb1\xca\x6d\x22\x22" + "\x31\xef\x14\xdb\xee\xa3\x36\x79\xd3\x1c\x48\x77\x7c\x88", + "\xa0\xc1\x52\x6c\x88\xdc\xc2\x65\xf7\x5d\x4e\xf9\xb2\xa0\x00\xfb\x3c" + "\xce\x9e\x5d\x99\x4c\x47\x2c\x46\xbf\xac\x38\x21\xd6\x11", + "\x9a\xa6\xcd\xb8\x5b\x12\x6e\x1f\x21\xd0\x66\xa3\xc0\x5e\x82\xf6", 1, + 0, 248 }, + { 96, 192, 128, 103, + "\x49\xf3\x3b\xc3\xc1\xa4\x0e\x1c\xa3\xb5\x6a\x49\x9e\x4c\x91\x37\xc1" + "\x48\xd1\x25\x61\x55\xfd\xb6", + "\x47\xbc\x33\xd9\x13\x49\x05\x68\x38\xb6\x24\x74", + "\x44\x9c\x8c\xbb\x9a\x67\xad\xb0\x3f\x60\x64\x6e\x5b\x90\x46\x20", + "\xa9\x20\xb4\xfe\xa9\x08\xb1\x77\x1d\x58\xd4\xc1\x08\x83\x8f\x3a\xf7" + "\xb8\x41\x54\x97\x06\x3d\xd9\x69\x1a\x55\x23\x44\xd6\x42", + "\x67\xad\xef\x99\x61\x1f\x34\x1d\x14\xea\x27\xe7\x2d\xa9\xb6\x58\xc9" + "\xa7\x9e\x3b\x32\x8e\x79\x75\x8c\x9d\x34\xdb\x0b\xed\x06", + "\xb2\xa4\x4d\x0f\xc9\x46\x06\xc4\xe2\xb6\xc3\x9b\x24\x2b\x3a\xca", 1, + 128, 248 }, + { 96, 192, 128, 104, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x74\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 105, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x77\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 106, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\xf5\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 107, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x54\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 108, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\x65\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb4\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb7\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\x36\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7b\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\xfa\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\x9e\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x94\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbf\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbc\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\x3e\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x85", 0, + 0, 128 }, + { 96, 192, 128, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x86", 0, + 0, 128 }, + { 96, 192, 128, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\xc4", 0, + 0, 128 }, + { 96, 192, 128, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7a\xbe\x95\x75\xbe\x8e\x47\x04", 0, + 0, 128 }, + { 96, 192, 128, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x74\x55\xdf\xe5\xb5\xe6\xf2\xb6\x7b\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\x65\xb5\xe6\xf2\x36\x7a\xbe\x95\x75\xbe\x8e\x47\x84", 0, + 0, 128 }, + { 96, 192, 128, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x75\x55\xdf\xe5\xb5\xe6\xf2\x36\x7a\xbe\x95\x75\xbe\x8e\x47\x04", 0, + 0, 128 }, + { 96, 192, 128, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x8a\xaa\x20\x1a\x4a\x19\x0d\x49\x85\x41\x6a\x8a\x41\x71\xb8\x7b", 0, + 0, 128 }, + { 96, 192, 128, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 192, 128, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 192, 128, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\xf5\xd5\x5f\x65\x35\x66\x72\x36\xfa\x3e\x15\xf5\x3e\x0e\xc7\x04", 0, + 0, 128 }, + { 96, 192, 128, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xef\x67\x20\x50\x90\x35\x75\x0e\xca\xfa\xc4\xb3\xe4\xfa\xc3\x56", + "\x74\x54\xde\xe4\xb4\xe7\xf3\xb7\x7b\xbf\x94\x74\xbf\x8f\x46\x85", 0, + 0, 128 }, + { 96, 256, 128, 131, + "\x80\xba\x31\x92\xc8\x03\xce\x96\x5e\xa3\x71\xd5\xff\x07\x3c\xf0\xf4" + "\x3b\x6a\x2a\xb5\x76\xb2\x08\x42\x6e\x11\x40\x9c\x09\xb9\xb0", + "\x4d\xa5\xbf\x8d\xfd\x58\x52\xc1\xea\x12\x37\x9d", "", "", "", + "\x6d\xc4\xef\x59\xa7\x3e\xbc\xab\xb5\xe3\x4c\x0d\x34\xd9\xf2\xd7", 1, + 0, 0 }, + { 96, 256, 128, 132, + "\xcc\x56\xb6\x80\x55\x2e\xb7\x50\x08\xf5\x48\x4b\x4c\xb8\x03\xfa\x50" + "\x63\xeb\xd6\xea\xb9\x1f\x6a\xb6\xae\xf4\x91\x6a\x76\x62\x73", + "\x99\xe2\x3e\xc4\x89\x85\xbc\xcd\xee\xab\x60\xf1", "", "\x2a", + "\x06", + "\x85\xca\x1f\x6c\x46\x28\x3c\xb5\xdd\x59\x60\xbd\x34\xa8\xdc\x36", 1, + 0, 8 }, + { 96, 256, 128, 133, + "\x51\xe4\xbf\x2b\xad\x92\xb7\xaf\xf1\xa4\xbc\x05\x55\x0b\xa8\x1d\xf4" + "\xb9\x6f\xab\xf4\x1c\x12\xc7\xb0\x0e\x60\xe4\x8d\xb7\xe1\x52", + "\x4f\x07\xaf\xed\xfd\xc3\xb6\xc2\x36\x18\x23\xd3", "", + "\xbe\x33\x08\xf7\x2a\x2c\x6a\xed", + "\x7f\xd4\xb5\xd3\x09\x52\x35\xa3", + "\xf6\x4d\x10\xb4\x1e\x3d\x69\x28\x74\x19\x47\xc5\x0c\xa0\x39\x1f", 1, + 0, 64 }, + { 96, 256, 128, 134, + "\x59\xd4\xea\xfb\x4d\xe0\xcf\xc7\xd3\xdb\x99\xa8\xf5\x4b\x15\xd7\xb3" + "\x9f\x0a\xcc\x8d\xa6\x97\x63\xb0\x19\xc1\x69\x9f\x87\x67\x4a", + "\x2f\xcb\x1b\x38\xa9\x9e\x71\xb8\x47\x40\xad\x9b", "", + "\x54\x9b\x36\x5a\xf9\x13\xf3\xb0\x81\x13\x1c\xcb\x6b\x82\x55\x88", + "\x48\xdd\x95\x89\xa4\x7e\x63\x8b\xbb\xc2\xaa\x3e\x23\x2f\xa5\x29", + "\xdf\x69\xfb\xe1\xf0\x99\xf0\x13\x4f\xe2\x86\x91\x56\xab\x07\xdb", 1, + 0, 128 }, + { 96, 256, 128, 135, + "\x3b\x24\x58\xd8\x17\x6e\x16\x21\xc0\xcc\x24\xc0\xc0\xe2\x4c\x1e\x80" + "\xd7\x2f\x7e\xe9\x14\x9a\x4b\x16\x61\x76\x62\x96\x16\xd0\x11", + "\x45\xaa\xa3\xe5\xd1\x6d\x2d\x42\xdc\x03\x44\x5d", "", + "\x3f\xf1\x51\x4b\x1c\x50\x39\x15\x91\x8f\x0c\x0c\x31\x09\x4a\x6e" + "\x1f", + "\x20\x22\x97\xd3\x6c\xa6\x2c\x5a\x1d\x44\x37\xfa\xfc\x7b\x50\xe7" + "\x64", + "\x66\x5f\x05\xa9\x6b\xf8\xde\x45\x36\x1d\xbf\x33\xc9\x8b\x09\x05", 1, + 0, 136 }, + { 96, 256, 128, 136, + "\x02\x12\xa8\xde\x50\x07\xed\x87\xb3\x3f\x1a\x70\x90\xb6\x11\x4f\x9e" + "\x08\xce\xfd\x96\x07\xf2\xc2\x76\xbd\xcf\xdb\xc5\xce\x9c\xd7", + "\xe6\xb1\xad\xf2\xfd\x58\xa8\x76\x2c\x65\xf3\x1b", "", + "\x10\xf1\xec\xf9\xc6\x05\x84\x66\x5d\x9a\xe5\xef\xe2\x79\xe7\xf7\x37" + "\x7e\xea\x69\x16\xd2\xb1\x11", + "\x3c\x0a\x0b\x34\x94\xd7\x5c\xcb\xcf\xfa\xa9\x17\xd6\x15\x92\x94\xfd" + "\x93\xe8\xa2\xee\x66\x44\x7a", + "\xb0\xb0\x7b\xa0\x51\x67\xe8\x8c\x24\xe5\x48\x24\xa0\x70\x61\xb9", 1, + 0, 192 }, + { 96, 256, 128, 137, + "\x2e\xb5\x1c\x46\x9a\xa8\xeb\x9e\x6c\x54\xa8\x34\x9b\xae\x50\xa2\x0f" + "\x0e\x38\x27\x11\xbb\xa1\x15\x2c\x42\x4f\x03\xb6\x67\x1d\x71", + "\x04\xa9\xbe\x03\x50\x8a\x5f\x31\x37\x1a\x6f\xd2", "", + "\xb0\x53\x99\x92\x86\xa2\x82\x4f\x42\xcc\x8c\x20\x3a\xb2\x4e\x2c\x97" + "\xa6\x85\xad\xcc\x2a\xd3\x26\x62\x55\x8e\x55\xa5\xc7\x29", + "\x0e\x29\xb2\x33\x5b\x90\x07\x58\xfa\xd2\x78\xae\xfb\x9b\x3a\xfa\x07" + "\xfd\x42\xb5\xd2\xf7\xd3\x87\xe3\xea\x0e\x0c\xa4\x16\xe0", + "\x6e\xd7\xe1\xe1\x27\x8c\x40\xce\x2e\x78\x1d\x10\x05\xde\x88\xdd", 1, + 0, 248 }, + { 96, 256, 128, 138, + "\x51\x55\xde\xe9\xaa\xde\x1c\xc6\x1e\xe7\xe3\xf9\x26\x60\xf7\x59\x0f" + "\x5e\x5b\xa8\x2f\x1b\x59\xb8\x50\xe3\xfa\x45\x3d\x2f\xa6\xb3", + "\xc2\x6c\x4b\x3b\xfd\xb9\x7e\xe6\xb0\xf6\x3c\xa1", "", + "\x27\x34\xe0\x8e\xff\x8f\x5c\x4f\x84\xfa\x0c\x20\x7f\x49\xc7\xfd\x78" + "\xaf\x1a\xd5\x12\x3f\xf8\x1f\x83\xf5\x00\xed\xf4\xed\xa0\x9e\xdf", + "\xae\xd2\x4e\x00\x82\xe1\x3e\xe1\x5b\xa0\x50\x6a\x83\x6c\x78\xb9\x7e" + "\xf2\xfa\xa3\xc6\xe8\xeb\x37\x8d\xc6\x4d\xd4\xad\xc9\x98\xad\x68", + "\x5b\xa1\xb4\x8a\x70\x16\x84\xd9\x40\xbe\x24\x4c\x3d\xe9\x38\xd2", 1, + 0, 264 }, + { 96, 256, 128, 139, + "\x95\xe8\x7e\xda\x64\xd0\xdc\x2d\x4e\x85\x10\x30\xc3\xe1\xb2\x7c\xca" + "\x22\x65\xb3\x46\x4c\x2c\x57\x2b\xd8\xfc\x8c\xfb\x28\x2d\x1b", + "\xce\x03\xbb\xb5\x67\x78\xf2\x5d\x45\x28\x35\x0b", "", + "\x2e\x5a\xcc\x19\xac\xb9\x94\x0b\xb7\x4d\x41\x4b\x45\xe7\x13\x86\xa4" + "\x09\xb6\x41\x49\x0b\x13\x94\x93\xd7\xd6\x32\xcb\xf1\x67\x4f\xdf\x25" + "\x11\xc3\xfa\xd6\xc2\x73\x59\xe6\x13\x7b\x4c\xd5\x2e\xfc\x4b\xf8\x71" + "\xe6\x62\x34\x51\x51\x7d\x6a\x3c\x68\x24\x0f\x2a\x79\x91\x6a", + "\x2f\xa0\xbd\x2c\xdf\xe9\xe8\xd7\x91\x9b\x97\xba\x05\xd5\x9e\x33\x89" + "\xfd\xcb\xa7\x28\xec\x12\x4d\x0f\x28\x49\x48\x4f\x63\x5a\x00\x0e\x73" + "\x4c\x8c\x80\x94\x1b\x3d\xa3\x2d\x23\xee\xa5\x1e\xdc\xe8\xd6\x61\x7b" + "\x16\xeb\xb4\x3a\xc8\x11\x3a\x09\x2e\x9d\xda\xa3\x72\x1a\xe9", + "\xa9\x25\x2a\x50\xdd\xbb\xf1\x8c\xd3\xe4\x3a\xdc\x0b\xa1\xa4\x81", 1, + 0, 528 }, + { 96, 256, 128, 140, + "\x7a\x4c\xd7\x59\x17\x2e\x02\xeb\x20\x4d\xb2\xc3\xf5\xc7\x46\x22\x7d" + "\xf5\x84\xfc\x13\x45\x19\x63\x91\xdb\xb9\x57\x7a\x25\x07\x42", + "\xa9\x2e\xf0\xac\x99\x1d\xd5\x16\xa3\xc6\xf6\x89", + "\xbd\x50\x67\x64\xf2\xd2\xc4\x10", "", "", + "\x56\x9d\x82\x69\x18\x92\xe1\x03\xe6\x27\x40\x7c\x95\xf0\x8a\x0e", 1, + 64, 0 }, + { 96, 256, 128, 141, + "\xb9\x07\xa4\x50\x75\x51\x3f\xe8\xa8\x01\x9e\xde\xe3\xf2\x59\x14\x87" + "\xb2\xa0\x30\xb0\x3c\x6e\x1d\x77\x1c\x86\x25\x71\xd2\xea\x1e", + "\x11\x8a\x69\x64\xc2\xd3\xe3\x80\x07\x1f\x52\x66", + "\x03\x45\x85\x62\x1a\xf8\xd7\xff", + "\x55\xa4\x65\x64\x4f\x5b\x65\x09\x28\xcb\xee\x7c\x06\x32\x14\xd6", + "\xab\x01\xf9\x2d\xb4\xf2\x10\xbd\xb5\xed\xaf\x0a\x1b\xd1\x9e\xba", + "\x62\x16\x30\xc5\x05\xd2\x4e\x3b\x29\x29\x49\x77\xd8\xff\xa4\xb4", 1, + 64, 128 }, + { 96, 256, 128, 142, + "\xf6\x0c\x6a\x1b\x62\x57\x25\xf7\x6c\x70\x37\xb4\x8f\xe3\x57\x7f\xa7" + "\xf7\xb8\x7b\x1b\xd5\xa9\x82\x17\x6d\x18\x23\x06\xff\xb8\x70", + "\xf0\x38\x4f\xb8\x76\x12\x14\x10\x63\x3d\x99\x3d", + "\x9a\xaf\x29\x9e\xee\xa7\x8f\x79", + "\x63\x85\x8c\xa3\xe2\xce\x69\x88\x7b\x57\x8a\x3c\x16\x7b\x42\x1c" + "\x9c", + "\xf0\x5e\x29\x0b\xbb\xc6\x19\x27\xfa\x65\x76\x06\x48\xdc\xca\x88" + "\xb0", + "\xb7\x21\xbe\x96\xa6\xb9\x5c\x09\x31\xfb\x24\x3d\xd1\x28\x7c\x70", 1, + 64, 136 }, + { 96, 256, 128, 143, + "\xaf\x1a\xc1\x9b\x3b\x84\xea\xaf\x26\x03\x37\x9c\xdd\x1d\xc1\xae\xe4" + "\xa4\x84\xfd\xc2\xc1\x90\x69\x1a\xfc\xc5\xb7\x62\xf9\xb5\x26", + "\xda\xf9\x8f\x1b\xd4\xc0\x71\xc6\xb1\x00\xf9\xc4", "\x14", "", "", + "\xe7\x72\xcc\x77\x14\xef\xce\xfb\xd1\x15\x08\xde\x48\x9f\x7c\x61", 1, + 8, 0 }, + { 96, 256, 128, 144, + "\x95\xfb\x78\x61\xf8\xc7\x5e\x14\x24\xd8\x40\x1c\xa3\xb3\x45\x2c\x56" + "\x3b\x99\xb0\x02\xc2\x4a\xfa\xd4\xcf\x5e\x82\x8f\x23\x53\xed", + "\xc1\xac\x60\x8d\x1f\xda\x28\xeb\x40\x34\x07\x9c", "\xfb\xc8", "", + "", + "\x54\xf2\xd2\xa5\x4c\xbe\x6e\x95\x9d\x51\xba\x4f\xfa\x8e\x0e\x9e", 1, + 16, 0 }, + { 96, 256, 128, 145, + "\xd3\x0e\x68\x2b\x58\x4d\x41\x60\x88\xa8\x1b\xd6\xf8\x55\x51\xec\x1f" + "\x2e\x11\x89\x38\x8a\x7a\x9c\x05\x21\xe2\x5b\x72\x5f\x7d\xbe", + "\x92\x72\x14\xf6\x43\x36\x70\x1a\x3b\x4d\xb6\x03", + "\x5c\xdb\x70\x70\x08\xb0\x65", "", "", + "\x2c\x6b\x6b\xef\x6b\xa0\x82\xba\xa7\x24\x15\xaa\xaa\x88\x3c\x75", 1, + 56, 0 }, + { 96, 256, 128, 146, + "\xac\x5a\x03\x8c\xae\xa8\x47\x5e\x71\xca\x41\x03\x93\x88\xb8\x61\xf0" + "\x08\xb6\x0c\x62\xff\x2e\x91\x4f\xf0\x83\x39\x86\x2f\xb8\x50", + "\xac\x38\x11\x7b\x39\x6a\xa0\x68\x43\x31\xfe\x74", + "\x02\xd1\xd0\x0a\x8f\x1f\x05\x2c\x08\x35\x75\xeb\x0c\x2a\x09", "", + "", + "\xd2\xe3\xaa\xdf\x9e\xd6\x0d\x91\xda\x5a\x1d\xc1\x21\xdb\xfd\x24", 1, + 120, 0 }, + { 96, 256, 128, 147, + "\xef\xa5\xc5\x05\x3b\x45\x20\x02\xfc\x34\x53\x12\xa3\xbe\x65\x0e\x9f" + "\xf4\x24\x4a\x1e\x44\x55\x7d\x8a\x41\x55\x70\xd2\xdb\xe9\x02", + "\x1a\xbf\xb6\xe3\x18\x99\x5e\xa0\x22\xb1\xd3\x69", + "\x01\xa1\x0b\xc7\x1a\x88\xc9\x4a\x3f\xf9\x24\xfe\x74\xcc\xa2\x29", + "", "", + "\x8e\x97\x80\xcc\xa8\x6d\x3c\xa4\x02\xe1\xdf\xaa\x03\xa7\x2d\x77", 1, + 128, 0 }, + { 96, 256, 128, 148, + "\xbd\xf2\x99\x4d\xd0\xfe\xb3\xc8\x70\xb3\x9f\x52\xbc\xc7\x6d\x28\xee" + "\xd7\x12\xd9\x11\xd9\x56\xc0\x42\xe4\x70\x1c\x4b\x20\xe5\xd5", + "\x0a\x82\x3c\x80\x1d\x05\x7e\x84\x3a\xf7\xca\x55", + "\x03\xf3\xd0\xfc\x23\xdd\x8f\x3e\x20\x88\x4d\x3c\x6f\xff\x26\x08" + "\xb1", + "", "", + "\x36\x88\x6d\x89\xcd\xcc\xe1\x57\x49\x7f\xd0\x9d\xcd\x67\xf3\x29", 1, + 136, 0 }, + { 96, 256, 128, 149, + "\x5a\xe3\xc3\xff\x78\xce\xdc\x19\x2c\xa7\x04\x4b\x3f\x41\xa2\x42\x43" + "\x2b\x0e\xa7\xd3\x48\x8c\x68\x0c\xd4\x22\x51\x5b\x09\x3b\x5b", + "\x7c\x62\x04\x28\x69\xa2\xe5\x97\x01\x48\x16\x14", + "\x7b\xa3\xf3\xbb\xba\x5d\xff\x63\x74\x88\x06\x4b\x6a\x52\x49\xd2\xad" + "\x46\x17\x17\x27\x87\x19\xfe\x71\xfe\xbf\x71\x00\x82\x8e", + "", "", + "\x3f\x8b\x8f\x90\x4c\xeb\x30\x45\x05\xf9\x42\xf3\x6c\xca\xc5\xf8", 1, + 248, 0 }, + { 96, 256, 128, 150, + "\xa8\x10\x8c\x33\xda\x05\x9f\xed\xf6\x02\x2a\x6e\xc4\x95\x27\xbe\x0a" + "\xb6\x40\x02\x47\x2c\xb2\xf7\x03\xb9\x7e\x01\x79\xa3\x43\x12", + "\xbc\xe6\x36\xca\x40\x1a\x88\xfa\xc2\x36\x1e\xd1", + "\x35\xea\xc1\x65\x26\xc2\xf1\x0a\x12\x71\xb3\xa8\xf8\x10\xbb\xf2\x39" + "\xee\xb9\x61\xe1\xa7\xe9\x20\x5b\xea\xe6\x00\x45\xf0\x08\xe6", + "", "", + "\x43\x0c\xcb\xb1\xf7\x5d\xe0\x6b\x71\x63\x7d\x1a\x76\xb3\x5c\xf5", 1, + 256, 0 }, + { 96, 256, 128, 151, + "\x4c\x8e\x95\xa7\x36\x1b\xb3\x78\x49\xb1\x6f\x0e\x5f\x9a\x6e\xab\x87" + "\x39\x13\x39\xd9\x51\xd7\x40\x4f\xf5\xcd\x82\x9c\x08\x7a\x6b", + "\xb4\xa8\xde\x71\xfb\x0f\xe1\x72\xff\x6d\x89\xb6", + "\x94\xd0\x6e\xdc\xfa\x5a\xe3\xd2\x7b\x99\x53\xfe\x5d\xf0\xcc\xa6\x19" + "\x4f\xf6\xdf\xa9\x4d\x82\xb7\x35\x9c\xb3\x87\xdd\x5e\x80\xc6\x18\x6f" + "\xbf\x17\x48\xc1\x92\xbb\x0c\x68\x8e\xbb\x47\x1b\x90\x20\xfe\x8f\xbf" + "\xae\xe3\xde\xe8\x78\x7a\xce\x3c\x20\xdd\x50\xbe\x08\x3e", + "", "", + "\xda\x7f\xe2\xa4\x46\x9e\x39\x1c\x20\x5b\x6d\xe8\xe1\x82\xa9\x14", 1, + 520, 0 }, + { 96, 256, 128, 152, + "\x7d\x00\xb4\x80\x95\xad\xfa\x32\x72\x05\x06\x07\xb2\x64\x18\x50\x02" + "\xba\x99\x95\x7c\x49\x8b\xe0\x22\x77\x0f\x2c\xe2\xf3\x14\x3c", + "\x87\x34\x5f\x10\x55\xfd\x9e\x21\x02\xd5\x06\x56", "\x02", + "\xe5\xcc\xaa\x44\x1b\xc8\x14\x68\x8f\x8f\x6e\x8f\x28\xb5\x00\xb2", + "\x6f\xe8\x78\x84\xb9\x4e\xac\x04\x1c\xb4\xc7\x8c\x23\xf2\x83\xa3", + "\x7e\xeb\x30\x08\x25\x49\x63\x72\x24\xe9\x26\xe5\x27\xb6\x9a\xea", 1, + 8, 128 }, + { 96, 256, 128, 153, + "\x64\x32\x71\x7f\x1d\xb8\x5e\x41\xac\x78\x36\xbc\xe2\x51\x85\xa0\x80" + "\xd5\x76\x2b\x9e\x2b\x18\x44\x4b\x6e\xc7\x2c\x3b\xd8\xe4\xdc", + "\x87\xa3\x16\x3e\xc0\x59\x8a\xd9\x5b\x3a\xa7\x13", "\xb6\x48", + "\x02\xcd\xe1\x68\xfb\xa3\xf5\x44\xbb\xd0\x33\x2f\x7a\xde\xad\xa8", + "\xe0\x17\xbf\x1d\xdd\x27\x98\x86\xf7\x54\x53\x65\xf1\x46\x5c\xc7", + "\x6d\xca\xb7\x9d\x1d\xda\xb4\xf3\xad\x8b\x4a\xf7\x23\x18\xeb\x1b", 1, + 16, 128 }, + { 96, 256, 128, 154, + "\xda\xcd\x51\xa8\xa8\xe4\xd5\x90\x5b\x4c\xbb\x94\x7e\xf4\x01\x3e\xb2" + "\x96\x88\x93\x53\xf3\xc9\xee\x35\xf5\x57\x7b\x26\x73\x7a\x51", + "\x3f\xa3\x78\xa1\xbe\xfd\xdd\xd6\x1a\xe6\x8c\xf4", + "\xbb\x5a\x38\x12\xf0\xae\xfd", + "\xe1\x48\x31\x38\x83\xa7\x7d\xa1\x21\x12\x4d\x06\xb1\xc7\x7d\xca", + "\xdd\xf8\xad\xe1\x3d\x69\xf3\x64\x9e\x36\xc6\x69\xd2\x5b\x4d\x81", + "\x01\x86\x15\x57\xd4\x3a\xb0\x14\xc4\xed\xe1\x9f\xcd\x75\x48\xea", 1, + 56, 128 }, + { 96, 256, 128, 155, + "\x8e\x34\xcf\x73\xd2\x45\xa1\x08\x2a\x92\x0b\x86\x36\x4e\xb8\x96\xc4" + "\x94\x64\x67\xbc\xb3\xd5\x89\x29\xfc\xb3\x66\x90\xe6\x39\x4f", + "\x6f\x57\x3a\xa8\x6b\xaa\x49\x2b\xa4\x65\x96\xdf", + "\xbd\x4c\xd0\x2f\xc7\x50\x2b\xbd\xbd\xf6\xc9\xa3\xcb\xe8\xf0", + "\x16\xdd\xd2\x3f\xf5\x3f\x3d\x23\xc0\x63\x34\x48\x70\x40\xeb\x47", + "\xc6\x0d\x2a\x92\xe6\x0a\x1a\x73\xa9\xce\x4b\x22\x69\xe1\x3a\x45", + "\x71\xfa\x66\x5b\x61\x1f\xed\x6e\xf5\xe6\x7e\xe8\x27\xac\x20\x6d", 1, + 120, 128 }, + { 96, 256, 128, 156, + "\xcb\x55\x75\xf5\xc7\xc4\x5c\x91\xcf\x32\x0b\x13\x9f\xb5\x94\x23\x75" + "\x60\xd0\xa3\xe6\xf8\x65\xa6\x7d\x4f\x63\x3f\x2c\x08\xf0\x16", + "\x1a\x65\x18\xf0\x2e\xde\x1d\xa6\x80\x92\x66\xd9", + "\x89\xcc\xe9\xfb\x47\x44\x1d\x07\xe0\x24\x5a\x66\xfe\x8b\x77\x8b", + "\x62\x3b\x78\x50\xc3\x21\xe2\xcf\x0c\x6f\xbc\xc8\xdf\xd1\xaf\xf2", + "\x72\x2a\xc6\xa2\x26\xf4\x9c\x90\xab\x22\x52\x7a\x51\x38\xb4\x01", + "\x2e\x4e\xe9\x97\xc7\x52\x78\x3e\x74\x3b\x36\x6b\xb6\xb3\x50\xa5", 1, + 128, 128 }, + { 96, 256, 128, 157, + "\xa5\x56\x9e\x72\x9a\x69\xb2\x4b\xa6\xe0\xff\x15\xc4\x62\x78\x97\x43" + "\x68\x24\xc9\x41\xe9\xd0\x0b\x2e\x93\xfd\xdc\x4b\xa7\x76\x57", + "\x56\x4d\xee\x49\xab\x00\xd2\x40\xfc\x10\x68\xc3", + "\xd1\x9f\x2d\x98\x90\x95\xf7\xab\x03\xa5\xfd\xe8\x44\x16\xe0\x0c" + "\x0e", + "\x87\xb3\xa4\xd7\xb2\x6d\x8d\x32\x03\xa0\xde\x1d\x64\xef\x82\xe3", + "\x2f\xad\xf1\x6a\xd1\x6a\x21\xc3\x17\xaf\x9d\x0b\xc1\x87\xf1\x36", + "\x88\xca\xf7\x0f\x6b\x5d\x8f\x3e\xf6\xa3\x9d\x1a\xe4\x13\x77\x2b", 1, + 136, 128 }, + { 96, 256, 128, 158, + "\x07\x74\x33\x02\x2a\xb3\x4d\x38\x0f\xc1\x92\xfc\x24\xc2\xed\xc6\x30" + "\x1f\xec\x6f\x24\x44\x2f\x57\x2a\x10\x87\xff\x2e\x05\xb3\x9a", + "\x28\xad\xcb\xc7\x43\x64\xf2\x6d\xd4\xb3\x10\x8b", + "\xe0\x10\x0e\xb1\x16\xcd\xc5\xe2\x2a\x3b\x9f\x9b\x41\x26\xc1\x49\x59" + "\x5e\x75\x10\x7f\x6e\x23\x7c\x69\xe8\x29\x60\x05\x22\x70", + "\x03\xc8\x74\xee\xaa\xa6\xfa\x9f\x0d\xa6\x2c\x75\x8f\xb0\xad\x04", + "\x13\x6f\x04\x9e\xa8\x51\xc6\xdf\xd2\xe8\x73\x12\xd8\x2f\x08\x82", + "\xfd\xf9\xbc\x04\x12\xcf\xca\x2b\x03\x5c\x5a\xe6\x8a\xc6\xda\x79", 1, + 248, 128 }, + { 96, 256, 128, 159, + "\x39\x37\x98\x6a\xf8\x6d\xaf\xc1\xba\x0c\x46\x72\xd8\xab\xc4\x6c\x20" + "\x70\x62\x68\x2d\x9c\x26\x4a\xb0\x6d\x6c\x58\x07\x20\x51\x30", + "\x8d\xf4\xb1\x5a\x88\x8c\x33\x28\x6a\x7b\x76\x51", + "\xba\x44\x6f\x6f\x9a\x0c\xed\x22\x45\x0f\xeb\x10\x73\x7d\x90\x07\xfd" + "\x69\xab\xc1\x9b\x1d\x4d\x90\x49\xa5\x55\x1e\x86\xec\x2b\x37", + "\xdc\x9e\x9e\xaf\x11\xe3\x14\x18\x2d\xf6\xa4\xeb\xa1\x7a\xec\x9c", + "\x2c\xed\x0d\x04\x32\x3d\xb2\x0a\xa8\xb8\xb9\x00\x0d\x2c\x33\xa5", + "\x48\xb8\x1c\xb7\x0d\x55\x5b\x7c\x38\x49\x2b\xa2\xc0\xa3\xb1\xb8", 1, + 256, 128 }, + { 96, 256, 128, 160, + "\x3f\x49\x4f\xd8\xf1\xb5\x06\x92\xde\x9c\xe3\x3f\x6d\x45\x1e\xf0\xc5" + "\x8c\x6b\x2c\x6f\xfb\xa3\xb4\x17\x10\xff\x63\xe6\x7e\xda\x68", + "\x9e\x62\x8e\x58\x03\x51\x92\x90\xe6\xb2\xc6\x8a", + "\xd0\x2e\xc8\x92\xd3\xb0\x3e\xac\xb2\xf1\xd8\xa6\x0a\x02\x84\x85\x77" + "\x6a\xf0\xfd\x66\x65\xcb\x6f\x74\xfb\xa5\xef\x89\x7e\x2c\xf5\x4b\x32" + "\xe8\x0b\xdd\xee\xc9\x38\xab\x53\x0b\x45\xed\x97\x12\x34\x80\x4f\xa8" + "\x4a\x19\x1d\xc1\x1a\xe6\x60\xf5\xa8\x66\x2a\x46\x51\xe9", + "\xf2\xc4\x1a\x26\xa4\x38\xe9\xff\x73\x3b\x78\x28\xf2\x4a\x24\x49", + "\x1d\x9f\x42\x56\xe0\x84\x66\x56\x0a\xb2\x71\xde\x36\x21\xa0\x3f", + "\x81\xdb\xe3\x8a\xc7\x51\x12\x7e\xfc\x11\xca\xed\xec\xa9\xc9\x3b", 1, + 520, 128 }, + { 96, 256, 128, 161, + "\xc1\x80\xc1\x2e\x6a\xf8\xcc\x07\x19\x04\x9e\xfe\x99\xd4\xdf\x2d\xe2" + "\x41\xef\xec\x5a\x01\x31\x45\xb2\xb7\x5e\x15\xdb\xa1\x6f\xe5", + "\x12\x6f\xbb\xd6\x99\xbe\xb3\x74\xf6\x7b\xaa\x7b", "", "", "", + "\x15\xdc\x4b\x81\x22\xb0\xe5\xad\x13\xdb\xea\x70\x96\xe8\x18\x68", 1, + 0, 0 }, + { 96, 256, 128, 162, + "\x80\x03\xe6\x54\x7a\x96\x4c\xd5\xc2\x84\x41\xc9\xb1\xa3\xc0\x83\xcc" + "\xb9\x6c\x7e\x13\x83\x85\xa5\x29\x4a\x1c\x30\x6b\x05\xf9\xf4", + "\xf2\x51\xc4\xa2\x62\x5c\x61\x2f\x86\xec\x16\x50", + "\x8c\x6e\xea\x97\x56\x15\x5d\x6e\xa9\x59\x5c\xc4\x9e\x8a\x74\xe1", + "", "", + "\xe6\xe9\x1b\xfe\x55\x18\xb7\x6e\x0a\x2d\xca\x79\xc6\x4d\x23\x2b", 1, + 128, 0 }, + { 96, 256, 128, 163, + "\xa4\x20\xe4\xf9\xa6\x16\xbf\x63\x1a\x94\x9b\x32\x4a\xe9\x01\x6a\x77" + "\xd9\x43\xa0\xfd\x1d\xa2\xb3\xe9\x74\x2e\x2e\xe5\x0f\xa0\x97", + "\x77\xf6\x71\x99\xab\x7b\x96\xf6\xf4\x83\x2c\x01", "", + "\xfc\xaf\x04\xe3\x33\xd8\x76\xae\x34\xfc\xab\x93\xaf\xd7\xba\xa7", + "\x13\x55\xcd\x0e\x1b\x42\xb1\x78\x81\x83\x00\x9a\x11\xca\xd5\x38", + "\x59\xd3\x5f\x9a\x12\xbe\xd5\xb5\xca\x87\x0b\x28\xba\xdb\x27\xf4", 1, + 0, 128 }, + { 96, 256, 128, 164, + "\x69\x05\xf2\xc4\xd6\x3a\x7b\xa7\xe9\x36\x6b\xa6\xc3\xbb\xf6\xe3\x55" + "\x2d\x56\x9e\xb5\x53\x21\xbe\xba\x3f\x8d\x98\xa1\x82\xb9\x7d", + "\x49\xe0\x8c\x06\xdb\xae\x3f\xf5\xcb\x7d\x9e\xcd", + "\xab\xa0\xc4\x4e\x9f\x93\x99\x74\x8f\x4a\x7e\x91\x9c\xeb\x8a\x62", + "\xb5\x62\x54\xe2\x21\xcf\x55\x8e\xa7\xd9\x19\x4e\xec\xd6\x3e\xd5", + "\x68\xe6\x6f\x43\x3c\x3c\xf9\xe4\x24\x72\x25\x68\x2f\x4c\x73\xa4", + "\x07\xf5\x3a\x76\xb8\x44\x60\xec\xf4\xaa\x18\x13\xe5\x17\x0b\x2f", 1, + 128, 128 }, + { 96, 256, 128, 165, + "\x77\x0b\x39\x74\x1c\x56\xd4\x67\x00\xa9\xf3\xcc\x23\x1d\x1a\xcb\x17" + "\x44\x98\x70\x2c\x0f\x2d\x0e\xec\x20\xdb\x57\x49\x4b\xb4\x9e", + "\x93\x84\xa1\xec\xbe\xe1\xde\x2b\x5a\xe7\x06\x84", "", + "\xea\x0b\x32\x28\xb8\x3c\xa6\x61\x50\xa7\x9a\xba\x15\x9e\x50\x6b" + "\x75", + "\x40\x52\xc0\xfc\x81\x63\x46\xc8\x69\x21\xdb\x57\x64\x6f\xeb\x29" + "\x43", + "\xa1\x9a\x58\x53\x10\xce\xb2\xdf\x76\x7e\xc1\x72\x4d\x52\xe3\x9d", 1, + 0, 136 }, + { 96, 256, 128, 166, + "\x41\xd6\xc6\xba\xbb\x72\x41\x53\x9a\xc1\x66\x47\x48\xdd\x1c\xf2\x9c" + "\xe7\x94\x0e\x29\x15\x3c\xd8\x18\x0e\xd1\x97\xda\xb5\xc7\x3f", + "\x00\x05\xde\xa1\x2e\xb6\x98\x50\x64\x7c\x7a\xd9", + "\x67\x5f\x31\xd7\x6b\xf4\x83\xd2\xd2\xab\x57\xcb\xe9\x3c\xf2\xf1", + "\xe5\xc4\x44\xa0\x45\x8d\xca\xf7\x89\xc8\xf3\x56\x66\xf1\x5b\xcc" + "\xb4", + "\x99\xa4\x9b\xde\x03\x72\x8c\x47\x9d\xaf\x4c\x67\xd3\x07\xf1\x28" + "\x5e", + "\xf0\xf3\x85\x9d\x12\xcd\x41\x48\xb9\xe8\x4d\x22\xba\x7a\xd9\x66", 1, + 128, 136 }, + { 96, 256, 128, 167, + "\xbc\x53\x6d\x8d\x9b\x43\x40\xcd\x14\x14\x7f\xca\x7c\xa3\x65\x73\xba" + "\x45\xbf\xf5\xb0\xa7\xcb\x80\x91\xa5\x50\xcf\x2b\x4b\xb9\x45", + "\x17\x86\x94\xeb\x62\xd7\x77\x3b\x0f\x0f\xbe\x8b", "", + "\xfb\x82\x20\x33\xc4\x43\x76\x80\x30\x1f\x72\xff\xc7\x4b\xa3\xbd\x46" + "\x7a\x9a\xb7\x46\x5a\xe4\x5e\xc8\x7a\xb4\xbe\xfd\x7c\xc9", + "\xb6\xce\x6d\x86\xab\x7e\xee\x1f\xdd\x7d\xcb\x95\x5b\x83\x24\x36\x08" + "\x39\xeb\x2b\xdd\xd8\x21\xec\xc1\xef\xe1\x29\x18\x26\x89", + "\x72\xfd\x99\x7b\xf0\x58\x9a\xa2\xd6\x07\x21\x65\x4f\x55\x2e\x3a", 1, + 0, 248 }, + { 96, 256, 128, 168, + "\xf5\x9a\xbc\xbf\x42\x18\xbd\x5c\x76\x01\xf0\x80\xb5\xfb\xd3\xae\x08" + "\x87\x33\x70\x2c\x8f\xbe\xf0\xc5\x29\x6a\x40\x6f\x56\x38\x27", + "\xa5\xeb\x0e\x6f\xe6\x69\xe6\x82\x39\xac\xe5\x50", + "\xd6\x03\x49\x1f\xbf\x09\x50\xd3\x64\x89\xab\xb4\x0d\xd8\xd4\x2b", + "\x97\xdc\xba\xcd\x70\xa6\x78\xcf\xae\xd1\x3c\x94\x2c\xf9\x20\xe8\x51" + "\xec\x3e\x6f\xb1\xf6\xc6\xeb\x95\xf1\xc9\x65\xfb\x1a\x13", + "\xc0\xb2\x7e\xdd\x65\x33\xcf\xba\x81\x32\x3a\xc7\x8d\x0a\xeb\x03\x71" + "\xb1\xd7\xb8\x99\x38\xe0\x4c\x31\x91\x48\x96\x15\x13\xfb", + "\x56\xaa\xbb\xde\x47\xab\x2c\x53\xdb\x48\x70\x30\x33\xf8\xca\x68", 1, + 128, 248 }, + { 96, 256, 128, 169, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8a\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 170, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x89\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 171, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x0b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 172, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc3\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 173, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\xf7\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 174, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0c\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 175, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0f\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 176, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x8b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 177, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x93\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 178, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x12\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 179, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\xa0\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 180, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2e\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 181, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb8\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 182, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xbb\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 183, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\x39\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 184, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x11", 0, + 0, 128 }, + { 96, 256, 128, 185, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x12", 0, + 0, 128 }, + { 96, 256, 128, 186, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x50", 0, + 0, 128 }, + { 96, 256, 128, 187, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x0b\x92\x80\x2f\xdc\xb9\xd7\xd4\x90", 0, + 0, 128 }, + { 96, 256, 128, 188, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8a\xc2\xdd\x77\x0d\xe5\x8f\x0b\x93\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 189, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\xf7\x0d\xe5\x8f\x8b\x92\x80\x2f\xdc\xb9\xd7\xd4\x10", 0, + 0, 128 }, + { 96, 256, 128, 190, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8b\xc2\xdd\x77\x0d\xe5\x8f\x8b\x92\x80\x2f\xdc\xb9\xd7\xd4\x90", 0, + 0, 128 }, + { 96, 256, 128, 191, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x74\x3d\x22\x88\xf2\x1a\x70\xf4\x6d\x7f\xd0\x23\x46\x28\x2b\xef", 0, + 0, 128 }, + { 96, 256, 128, 192, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 256, 128, 193, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 256, 128, 194, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x0b\x42\x5d\xf7\x8d\x65\x0f\x8b\x12\x00\xaf\x5c\x39\x57\x54\x90", 0, + 0, 128 }, + { 96, 256, 128, 195, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x9d\x2a\x30\xab\xc5\xe1\x78\xf7\xc6\x31\x7e\xc9\x49\x8d\xac\x39", + "\x8a\xc3\xdc\x76\x0c\xe4\x8e\x0a\x93\x81\x2e\xdd\xb8\xd6\xd5\x11", 0, + 0, 128 }, + { 56, 128, 128, 196, + "\x19\xd5\x32\xdb\xcc\x93\x4a\x00\x9c\xe1\xb9\x4a\x0b\x31\xdd\xc7", + "\xf0\xbf\x6a\x9b\xcf\x6c\x0d", "", "", "", + "\x7b\x12\x26\x3a\xaf\x1e\x4c\xb6\xe4\xb4\x06\xe0\x26\x69\x82\x09", 1, + 0, 0 }, + { 56, 128, 128, 197, + "\x8c\xdb\x7f\x67\x89\x27\x1a\x6e\xf3\xe0\x64\x61\xe9\x0e\xaa\x0e", + "\x7c\x0d\x6b\xce\xba\x28\x2e", "\xfb\xc4\xf4\xa5\x2e\xcb\x4c\xaa", + "", "", + "\x50\xb1\x2c\x1f\xa4\xdc\x4b\x2d\xc4\xdd\x0e\xb1\x52\xdb\x41\x9e", 1, + 64, 0 }, + { 56, 128, 128, 198, + "\x6b\xd7\x36\x3b\xe8\x1b\x3f\x80\x3c\x7f\xae\xe6\x07\x05\x02\x74", + "\x30\x3d\xa6\x78\xd1\x67\x9e", "", + "\x53\x9c\x7d\x6f\xcc\x0a\x69\x1b\xd3\x9b\xc4\x34\x22\xd4\xe1\x3c", + "\x38\x33\x8e\x92\x4b\xf2\xec\xc3\xae\x0f\x5f\x75\xf2\xaf\x2d\x30", + "\xe4\x0b\xbb\xa6\x73\x49\x55\x22\x3f\xab\x6d\xdb\x3c\x7b\xba\x83", 1, + 0, 128 }, + { 56, 128, 128, 199, + "\x5b\x49\xd6\x7b\x0a\x74\xe3\xf3\x9e\x8d\x0b\xac\x6a\x00\x50\x40", + "\xb9\xbb\xb9\xae\x00\x3b\x08", "\xaa\xdc\xed\xa4\x4e\x5d\x23\x23", + "\xfe\x66\xe3\x59\xd3\x40\xec\x00\x24\x17\x36\xc2\xa6\x78\x90\x02", + "\xa4\x41\xe2\xeb\x45\x8f\x8a\x6f\x2a\xc7\x56\x27\xab\x10\x85\xef", + "\x5c\x54\x51\x81\x68\x52\x79\x65\x21\x87\x81\x9a\xa6\xd1\x5b\x86", 1, + 64, 128 }, + { 56, 128, 128, 200, + "\x6a\x5b\x3b\x57\xf8\x3c\xac\x23\xeb\xbb\x97\xa6\x0f\x9c\x13\xc3", + "\x64\xc0\x18\x42\xe7\x3e\x74", "", + "\x7d\xab\x0c\x47\x34\x73\xdf\x8d\x30\x12\xc3\xfd\xf0\x93\xf0\x07" + "\x09", + "\xfb\x8f\x96\x40\x65\x71\x8f\x93\x90\x10\xea\x5e\x5d\xa3\x27\xcd" + "\xdb", + "\xed\xd3\x49\xc3\x74\xc3\xd0\xdb\x1a\xc3\x6f\x11\xb1\x50\x6d\x2c", 1, + 0, 136 }, + { 56, 128, 128, 201, + "\xeb\x26\x3b\x3a\x87\xfc\xf2\x32\x32\x7a\x05\xb2\x07\x92\x92\xab", + "\x95\x54\x02\x3b\xad\xf3\xe2", "\xbe\x0d\xd7\x00\x2e\x2f\xe3\x58", + "\x0c\xac\x1a\xfd\x57\x08\xab\x03\xc8\xd3\xfe\x1d\x7c\xc8\x3b\x26" + "\xff", + "\x81\x86\x0e\x45\xcb\x00\x9f\x57\x28\xf8\x0f\xd1\xdf\x21\x4f\x84" + "\x49", + "\xa2\xcb\x64\x54\xa2\xa4\x9e\x96\xed\xca\x89\xb9\x4e\x49\xc5\x0c", 1, + 64, 136 }, + { 64, 128, 128, 202, + "\xf3\x43\x47\x25\xc8\x2a\x7f\x8b\xb0\x7d\xf1\xf8\x12\x2f\xb6\xc9", + "\x28\xe9\xb7\x85\x17\x24\xba\xe3", "", "", "", + "\x1c\x64\x58\x30\xe6\xee\x05\x58\x9b\x70\xf0\x23\x47\xe1\x1c\x93", 1, + 0, 0 }, + { 64, 128, 128, 203, + "\xea\xf5\xc7\xe3\x5b\x61\xc6\x4f\xd8\x99\xbf\x26\x50\x6c\xb8\x3c", + "\x07\x1f\xfe\xd7\x58\x5e\xb0\xb7", + "\xf0\xaf\x44\x31\xf3\x3e\x7e\x15", "", "", + "\xe5\xe1\x54\xd4\x3f\x32\x98\x89\x6b\x34\xbb\x4f\x76\xb7\x39\x9f", 1, + 64, 0 }, + { 64, 128, 128, 204, + "\xde\xb6\x22\x33\x55\x9b\x57\x47\x66\x02\xb5\xad\xac\x57\xc7\x7f", + "\xd0\x84\x54\x7d\xe5\x5b\xbc\x15", "", + "\xd8\x98\x6d\xf0\x24\x1e\xd3\x29\x75\x82\xc0\xc2\x39\xc7\x24\xcb", + "\x97\xbc\x3c\x09\xd5\xe3\x71\x78\xe7\xfd\xd3\x5d\x53\x23\x91\x80", + "\x3a\x2d\xc0\xbf\xde\x10\x24\x70\x29\xf5\xc4\x89\xe3\x06\xa3\x96", 1, + 0, 128 }, + { 64, 128, 128, 205, + "\x9e\xbe\x10\xef\x15\xeb\xcc\x60\x00\xed\x72\xd9\x74\x21\x9b\x97", + "\x28\xa8\x40\x39\xf2\xda\xe6\x51", + "\x1b\xab\x91\x6d\x21\xbc\xbb\x35", + "\xd0\x73\xa8\x8d\x45\x36\x41\x51\x40\x87\x18\x78\x69\x30\xed\xfb", + "\x6a\xdf\x7f\x96\xa3\x20\x22\x71\xb0\x37\x87\x37\x2f\x4c\xf3\xce", + "\x50\xbc\xdd\x15\x2a\xe3\x31\x55\x48\x78\xf9\xa2\xb8\x14\x0e\x72", 1, + 64, 128 }, + { 64, 128, 128, 206, + "\x74\xdf\xd2\x96\x3b\xc8\x14\x83\x38\x09\x44\x14\xe3\xfc\x2b\x8d", + "\xd1\xef\x66\xef\x2e\xb7\x65\xb8", "", + "\x17\x9c\x18\x65\xe2\xbc\x0f\x70\x24\x87\xc4\xe5\x4f\x83\x74\x45" + "\x7e", + "\xa4\xb7\xb9\xc9\xc9\x02\xed\xdb\x02\xfb\x64\x87\x31\x40\x25\x6e" + "\x94", + "\x4b\xc1\xe9\x91\x48\xf6\xa4\x72\x2d\x4b\x92\x73\x4b\x08\x8d\x43", 1, + 0, 136 }, + { 64, 128, 128, 207, + "\x5b\xf0\x96\x5f\x14\x39\xed\x83\xee\xde\xaa\xd9\x46\x7f\x5f\x60", + "\xb9\x26\x8f\xec\xa7\x29\x68\x0f", + "\x31\xfb\x02\xa7\xf4\xeb\xe9\xaa", + "\xc8\x44\xb6\xc4\x57\xe1\xe5\xf4\x3f\x82\xb4\xe4\x84\xb4\x70\x9e" + "\xf1", + "\x61\x1e\xe7\xdf\x91\xb0\x62\xb7\x5d\xf8\x6b\x10\xa4\xce\xeb\x01" + "\x34", + "\x9a\x56\x7c\x20\x65\xd7\x83\x2c\x35\xf6\x14\x3a\x41\x41\x46\x62", 1, + 64, 136 }, + { 72, 128, 128, 208, + "\x2e\xc7\xa4\x68\xe3\x64\x91\x86\xe1\xf9\xde\xcc\xdf\x95\xa2\x29", + "\x99\xfd\xb1\x58\xfb\x8e\xbc\xce\x64", "", "", "", + "\x65\x8f\x01\xf9\x0c\x35\x33\x1c\xdd\x69\x86\xf7\x36\xcc\x37\xda", 1, + 0, 0 }, + { 72, 128, 128, 209, + "\xc6\xbc\x09\x87\xb5\xdc\x94\x32\xda\x66\xbd\xb1\x53\x85\x9f\xda", + "\xd7\x68\xd1\xb8\x0a\x09\x45\x06\xb4", + "\x47\x13\xf8\x6a\x53\xce\xdd\x50", "", "", + "\x4e\xc3\x1d\xfe\xbf\xf4\xa9\x0d\x5e\xfb\xdb\x85\x04\xcd\x8c\x79", 1, + 64, 0 }, + { 72, 128, 128, 210, + "\x6a\x1c\x98\xe4\xd2\x0b\xc6\xad\x59\x48\x33\xd6\xe9\xaa\x47\x94", + "\x3d\xe2\x9d\x92\xd3\x01\x8e\xaa\xfc", "", + "\xf6\x23\x32\x2f\xef\x6d\x49\xcf\x7a\xbf\xa1\x6b\x5f\xd8\x39\x51", + "\xaf\xcb\xf0\x38\x5f\x26\x89\x5b\xcd\x61\x26\x60\x06\xdc\x1d\x98", + "\x79\x14\xe3\x2d\x18\x10\x43\x32\x12\x34\xdc\x16\xd7\x9b\x35\x76", 1, + 0, 128 }, + { 72, 128, 128, 211, + "\x02\x2b\x66\x9b\x7d\x39\x1f\x0f\xf5\xfa\xb1\x23\xc2\xba\x88\x17", + "\x2e\x02\xb0\x4c\xe6\xb3\x48\xef\x86", + "\xf9\xc8\x62\x29\x17\x05\x51\x9b", + "\xab\xb1\x8f\x46\x62\xc1\xbf\xa7\x98\x45\x60\xde\xac\x4a\x41\x5a", + "\x45\xc3\xab\x70\xb1\xc8\x83\xc9\x8b\x53\xfc\x9c\x0b\xe7\x7e\xcb", + "\x2e\xce\xd5\xeb\x7d\x60\x43\x1c\x09\xd3\xba\x49\x23\x0b\x23\xd0", 1, + 64, 128 }, + { 72, 128, 128, 212, + "\x89\x65\xe6\x41\xf4\x6d\xbb\xb1\x6a\xa8\x3f\x94\x59\x37\x0d\xc9", + "\x31\x0c\x4e\xe0\x82\xc4\x87\x0f\xc2", "", + "\x5d\x22\x78\xc8\xb4\xed\x8a\x37\xc9\x54\x88\x93\x5c\x1d\xb0\x6e" + "\x68", + "\xdb\x91\xb9\x8d\xa5\x78\xd8\xe4\xeb\xf1\x8f\x65\xf2\x44\x57\x9e" + "\xb7", + "\x21\x25\xef\x21\xfc\xba\x47\x10\x39\xc1\x31\x78\x6c\x99\xe7\x4e", 1, + 0, 136 }, + { 72, 128, 128, 213, + "\x22\xc9\x42\xb1\x21\x2a\x3c\xfa\x19\x6e\x9a\xd0\x6b\x03\xfb\x2b", + "\x71\xa2\xb8\x75\x40\xb1\x1d\x76\xa6", + "\xc0\xe0\x59\x60\xb8\x31\xe8\x75", + "\x4e\x3b\x6e\x3e\x2b\xa5\x66\x3e\xeb\xc5\xd8\x3d\xc2\x49\xea\xbc" + "\xe1", + "\x6f\xfa\x87\x5b\x85\x3c\xc4\xf6\xa4\x1a\xfd\x43\x0b\xd7\xc3\xce" + "\xee", + "\x0b\x49\x64\x85\x6b\xb2\xb2\x7e\xda\x38\x26\xae\x12\x8f\x67\xb3", 1, + 64, 136 }, + { 80, 128, 128, 214, + "\x1e\x6c\x62\x14\xa6\xa5\xdd\x5b\x62\x8c\x71\xde\x07\x78\x81\x37", + "\x40\xbc\xc3\x15\xde\xc8\x8b\xf3\x26\xcc", "", "", "", + "\xdf\xd7\x0e\x3e\x5a\x13\x16\x6b\x46\x06\x13\xab\xab\x92\x8f\x26", 1, + 0, 0 }, + { 80, 128, 128, 215, + "\xe4\x13\x43\xe5\xff\xe2\x0f\xe4\x8f\xf0\x10\xb1\x46\xce\xae\xad", + "\x5f\x17\x2f\xbe\x9f\x8e\xec\x0f\xbf\x79", + "\x9b\x46\x67\x59\x01\xa4\xbe\x0f", "", "", + "\x21\x04\xbc\x9e\xcb\x79\xb7\x1f\x32\xf2\x7c\x9e\xe4\xfe\xc6\x40", 1, + 64, 0 }, + { 80, 128, 128, 216, + "\xfc\x93\x58\x2f\xa1\xf8\xb5\x8c\xc9\xe8\x0d\xd5\x83\xe9\xbf\x8b", + "\x5d\x4b\xf5\x87\x98\xfa\xc3\x51\xa3\x99", "", + "\x86\x6d\x5e\x1b\x0a\xa2\x90\x04\xe5\x1e\xa8\x7d\xe8\x6e\x3c\x05", + "\xb6\x46\x50\xb6\x93\x5f\xb0\x4b\x97\x42\xf5\x72\x9f\x28\x6e\x03", + "\xb9\xd3\x94\x7c\x16\x05\xa2\xd5\x8e\xc3\xf3\x22\x18\x46\xc7\xda", 1, + 0, 128 }, + { 80, 128, 128, 217, + "\xdf\x2d\xb4\x8b\x19\x44\xfd\x9e\x24\x58\x9d\x14\x35\x7d\x0f\x80", + "\xf9\xa1\xbb\x32\xf5\x79\xb5\xf0\x27\x28", + "\x03\x92\x26\x00\xd7\xd0\x33\xdd", + "\xea\xfb\x69\xe4\x02\x38\xa3\x4e\x39\x85\x23\xfb\x35\xbd\x66\x12", + "\x9d\xe6\xac\x37\xf5\x2b\x81\x35\x04\x7a\xf8\xd5\xe5\x7f\xb3\x6e", + "\x05\x43\x35\x1a\xa8\x6f\xf8\xca\x58\x25\xbd\x7b\xb7\xc0\x25\x4e", 1, + 64, 128 }, + { 80, 128, 128, 218, + "\x3b\xf8\xc3\xc3\x01\x19\x0d\x23\xf7\x1a\xc8\x2c\x0c\x5b\x0f\x9b", + "\xd5\x27\x44\x06\xa4\xf5\xa2\xe2\xd1\x01", "", + "\x03\xca\x74\xe5\x8b\x8b\x38\x50\x0e\x1e\x65\xb8\x33\x2f\x41\xf0" + "\x6c", + "\x99\x9f\x47\x6b\x51\xce\x68\x6a\xf5\x9b\x0b\xbe\x22\x1c\xe4\xe8" + "\xa4", + "\x2a\xc4\x83\x77\xd2\x39\xfa\x7f\xfb\xe3\xc5\x03\xe0\x27\x8f\x98", 1, + 0, 136 }, + { 80, 128, 128, 219, + "\x16\x94\x2e\xb8\x9d\x4f\x7f\xe6\x5b\xf9\xb4\x9c\x16\xf8\x30\xab", + "\x2b\xa7\x6d\x03\x99\x5c\x62\xdc\x7e\xd2", + "\x32\xc7\xc6\x07\x2d\xbd\x73\x5f", + "\xb2\x38\x0e\x9e\xb5\x96\xd5\xaf\x69\x7c\x0b\xa1\xd3\x01\xa8\x33" + "\xd9", + "\xf5\x63\xca\xb1\xf5\xe5\x6f\x23\x7a\x60\xc2\xc2\x95\x0a\xb7\xa5" + "\xc2", + "\x25\xc8\x9e\x41\x0c\x0a\x53\x5b\x94\xa3\x83\xad\x3c\x01\x44\x64", 1, + 64, 136 }, + { 88, 128, 128, 220, + "\x9d\x2f\xa7\x59\x29\x61\x2e\x12\x13\x46\x0f\x99\x89\x46\xdc\xec", + "\x0e\x94\x8a\x03\xdb\xfa\x10\x81\x7e\x88\x26", "", "", "", + "\x2f\x1f\xa3\x58\x5b\x9a\x11\xff\x47\xbd\x48\x6f\x95\x57\x22\x46", 1, + 0, 0 }, + { 88, 128, 128, 221, + "\x7a\x97\x04\x06\xa7\x47\xc2\x32\x7e\xcb\xc8\xb1\x07\xa1\x90\xcd", + "\x99\xb0\x79\xde\x95\x2d\x60\xda\x0d\x03\x4b", + "\x2f\x08\x0b\x2d\xeb\x66\x44\xef", "", "", + "\x9d\xca\x9b\xa4\x41\x01\x3c\xe9\xfc\x0a\xc3\xdb\xf4\x14\x62\x6d", 1, + 64, 0 }, + { 88, 128, 128, 222, + "\xac\xab\x31\x48\x3d\x9d\xe4\xad\x77\xf4\xe6\x3f\xe4\x1b\x57\xae", + "\x5c\x09\x39\xe7\x1b\xae\x1a\x9d\xe1\x67\xd4", "", + "\xe4\xd7\x2b\x2f\x7c\xb6\x9b\xc5\x4a\x49\xf4\xd5\xce\xa4\xf2\x3a", + "\x0c\x50\x68\x43\xba\x7b\xd2\xdc\x45\x78\xe6\xbb\x83\xfd\x0c\x8d", + "\xc2\x75\x37\x30\x37\x63\x5b\xae\x29\x4f\x99\xc8\xe4\x69\x64\xb3", 1, + 0, 128 }, + { 88, 128, 128, 223, + "\x82\x84\x1e\xf7\xfb\xae\x35\x54\x65\x25\xfb\xbe\xbf\x47\x18\xfb", + "\xb5\xcd\x81\x8f\x73\xa3\x6e\xd0\x25\xb6\xcf", + "\x44\xf4\x8c\x2a\x20\x45\x63\x58", + "\x8c\x2c\x82\x3b\xb3\x99\x41\xb1\xc6\xb7\x5b\xbc\x82\xf0\x5b\xa4", + "\xb2\x87\xc6\x37\xa7\x55\x43\x62\xc8\x0d\x6b\x24\xd5\x0d\xdf\xb3", + "\x39\x67\x27\x7d\xa0\xf8\x56\xf8\xf0\xad\x49\x28\x28\x94\xd2\xbb", 1, + 64, 128 }, + { 88, 128, 128, 224, + "\xf9\xc6\xd9\x62\x7f\xd2\xe7\x31\xe2\xf1\x15\xb3\xd0\xa5\x3b\xfd", + "\x08\x45\x58\xeb\xfb\x65\x82\xf3\xd1\x87\x9a", "", + "\x74\x63\xaf\x94\x62\x62\x79\xce\x01\x12\xf6\x70\xc3\x11\x50\x99" + "\xfd", + "\xd4\x6b\x6e\x96\x2b\x6b\x7a\x23\x52\xfc\x43\x79\x14\xd9\xc0\xd9" + "\xc0", + "\xeb\xe3\x36\x17\x6e\x27\x04\x1a\xe7\x87\x91\xac\xa3\x45\x48\xc8", 1, + 0, 136 }, + { 88, 128, 128, 225, + "\x1d\x28\x6e\x52\x5e\xc2\x86\x4d\x9e\xa6\xe7\xad\xfb\xdc\x49\x70", + "\x1a\xd4\x48\x79\xf1\x94\x7a\xbd\x50\x3d\xce", + "\x1d\xb0\xea\xaa\xa1\xe2\xc8\x48", + "\x7b\x40\xe6\xc9\x87\x69\x2d\x02\x02\xcb\x6f\x44\xb4\x23\xc2\x67" + "\xdd", + "\x84\xf9\x1f\x9d\x35\xb9\x98\x59\x8e\x50\xfe\xb3\x4d\xca\xa0\xd2" + "\x60", + "\xae\x4e\x38\xcc\x4b\x73\x09\x44\xbc\x45\x9d\xf8\x5a\x53\x6f\x4e", 1, + 64, 136 }, + { 104, 128, 128, 226, + "\x69\x4a\x2a\xe9\x4c\xc2\xfc\x6c\x82\xdc\xd1\x6c\x58\xa3\x41\x95", + "\x8b\x4d\xe9\x49\x7e\x78\xd9\xc7\x3b\xdc\xb3\x74\xde", "", "", "", + "\x36\xb2\x0b\xcb\x06\x46\x09\xcb\xc0\x3a\xe3\x27\x86\xf7\x2e\xb5", 1, + 0, 0 }, + { 104, 128, 128, 227, + "\x26\x8c\x96\x1a\x4f\xa5\x4c\x21\x4a\x2a\xf8\xfe\x76\xa2\x77\xcc", + "\xf1\x34\x43\xda\x0e\x41\x2f\x1c\xc7\xa9\x01\x65\xc6", + "\x4f\xad\x12\xb4\x02\xc5\x80\x29", "", "", + "\xbe\x58\x07\x13\x42\xc9\x34\x8b\xaf\x78\x10\x4e\x22\x58\xe6\x16", 1, + 64, 0 }, + { 104, 128, 128, 228, + "\x7c\x5a\xa1\xcd\x2f\xc1\x71\xd5\xbb\x91\xee\x74\xf3\x1e\x1a\x63", + "\x70\xa2\xa5\x30\x37\x34\x51\x78\x27\xfa\xa7\xee\x78", "", + "\x4b\xd1\xd3\x42\x7e\x07\x35\xa0\x8f\x47\x5b\xc7\x3e\xc3\x64\x8c", + "\x9a\xb6\x12\x14\xea\x66\x11\x51\x07\x06\x31\x5f\x0f\xc9\xbd\x65", + "\xa2\xb3\xfc\x3c\x13\x49\xcb\xfd\x9d\xed\x7c\xc3\x2e\xff\xb3\x89", 1, + 0, 128 }, + { 104, 128, 128, 229, + "\xe7\x4b\x73\xc2\xad\x93\xd3\x8d\xd4\x43\x2d\x6e\x51\xd3\xe3\xec", + "\x06\xee\x28\xea\x53\x2f\xf5\xaa\xe6\xb0\xf6\xa2\x8a", + "\x10\xbc\x98\x64\xf1\x33\x2e\x41", + "\xaa\xd5\xd7\x58\x04\x1e\x54\x43\xed\xe7\xe9\xbb\xac\x1d\xb4\x90", + "\xd3\xed\x6b\xb5\x5d\x98\xb0\x0e\x1b\x76\x93\x8a\x1c\x6b\xd5\xed", + "\x22\x20\x1e\x4e\xb2\xa4\x22\x91\xa7\xd5\x7e\x35\x70\x82\xd7\x7e", 1, + 64, 128 }, + { 104, 128, 128, 230, + "\xbe\x70\x77\x74\xd9\xea\xb3\x70\xdb\x4e\x82\x50\x29\x7a\x74\x37", + "\x79\x44\xb4\x87\xd5\x9b\x6f\xfc\xc9\x6c\x9d\xf6\x2d", "", + "\x28\xea\xb5\x68\x85\xe1\xe1\x2b\xd7\x2d\xef\x11\x38\x23\x7f\x0d" + "\xbc", + "\xee\x05\xff\xea\xab\x17\xbb\x4d\xe9\x45\x27\xe0\x29\x70\x58\xc4" + "\x8b", + "\xff\x89\x9f\x92\x9b\x37\x16\x39\xd4\x8c\x3c\x44\x7f\x51\xc5\x77", 1, + 0, 136 }, + { 104, 128, 128, 231, + "\x30\x82\x94\x16\xd8\x0d\x4b\x6d\xd9\x1a\x16\xc4\x69\x4c\x5a\xcd", + "\x8b\xc7\xec\xd5\xbd\xe1\x96\xb7\x23\x19\xde\x6b\x77", + "\x97\x0f\xe6\x5b\x77\x89\xa5\x55", + "\x4a\x31\x22\xf8\x01\xd6\x63\x82\x28\xfa\x0e\x30\xaf\x3f\x36\x62" + "\x7a", + "\x51\x25\xed\x68\xaf\xbd\x34\xbc\x00\xc7\x31\x71\xad\xa3\x1e\xe8" + "\x4a", + "\x1d\xa3\x20\xc8\xba\xb5\x25\xe3\x75\xf3\x7a\x3b\xba\x3e\x0e\xb9", 1, + 64, 136 }, + { 56, 192, 128, 232, + "\xb4\x54\x4c\xae\x6b\x60\xb7\x72\x0f\x3a\xde\x71\xe9\x0e\x58\xc2\x1d" + "\x6e\x48\x71\x83\xd4\x66\x6a", + "\x2b\xa3\xda\x11\x2c\xf5\xe6", "", "", "", + "\x34\x15\x51\xf2\xc0\x5e\xe9\x31\x4f\x0e\xb5\x52\x93\x9e\x54\x86", 1, + 0, 0 }, + { 56, 192, 128, 233, + "\x26\xbd\xdb\x2e\xb2\xa7\x27\xe2\x91\x0d\xf9\x4a\xd3\xe1\x2a\xc1\x30" + "\xa4\x9a\x8f\x7f\x41\x95\x1c", + "\xc6\xc0\xef\x48\x15\x1b\x32", "\x74\x43\xb9\x1e\x73\x47\x5d\xe1", + "", "", + "\xfd\x7a\xab\x59\x5d\xfe\xb3\xc8\xa1\x66\x0e\xab\x04\x3b\x3d\x01", 1, + 64, 0 }, + { 56, 192, 128, 234, + "\x80\x6e\xd9\xcf\x33\xa1\xc2\xfa\x6a\x8b\xff\xad\x79\x37\xc3\xea\x22" + "\x64\x08\xeb\xf2\x48\xd1\x76", + "\x2b\xeb\xcc\x0a\xf6\x72\xbf", "", + "\xbe\x17\x02\x41\x48\x68\xc9\x4a\xeb\x99\xc1\xa0\x88\xba\x8c\x48", + "\x6e\x6f\x3d\x3b\x36\x48\x2c\x39\xa9\x9d\x59\x7e\xa5\x82\xf4\x30", + "\x7d\x11\xf5\x54\x9d\x87\xdd\xa7\xb0\x76\x22\x02\x27\x0a\x7e\x28", 1, + 0, 128 }, + { 56, 192, 128, 235, + "\x09\x5f\xa6\x78\xa1\x04\xe9\xc3\xd2\x46\x30\x4c\x5d\xdd\xee\x04\x5d" + "\xda\xb3\xd7\x9e\xa8\xa7\x26", + "\xfe\xbf\x6b\xf7\xdd\x16\xa7", "\x6a\x44\x90\xba\x9f\x61\xdb\x88", + "\xd2\x5e\xcf\xa8\x77\x89\x60\x30\x05\x8d\xca\xca\xb3\x15\x9c\xb3", + "\xf3\x57\x8a\xca\x6d\x3c\xcb\xc9\x16\xf5\xc1\xd7\x1a\x45\x87\x8e", + "\x8c\x73\x2f\x4a\x57\x1b\xf1\x05\xc6\xed\x1c\xef\x6f\xab\x28\x76", 1, + 64, 128 }, + { 56, 192, 128, 236, + "\x98\x98\x8d\xa4\x62\xa4\x6a\xb3\xdd\x61\x3b\xd3\x70\x69\xf4\xf4\x29" + "\xa9\xa8\x18\x41\xe7\x6d\xd3", + "\xeb\xad\x9a\xf5\xf8\x69\xf5", "", + "\xaa\x3f\xc0\x55\x74\xee\x10\x1e\xd7\x52\x7d\xe5\xda\x4a\xc3\x78" + "\x60", + "\x9b\x7b\xb3\x5d\xb7\x23\x71\x8c\x7f\x9f\xd8\xcd\x5c\x83\x12\x4e" + "\x78", + "\xa5\x95\x67\x30\x75\x77\xc8\xe8\x31\xc2\x3a\xc0\x9a\x92\xc6\xa1", 1, + 0, 136 }, + { 56, 192, 128, 237, + "\x71\xde\x00\x8c\xd8\x20\xfc\x03\x39\x74\xb6\xb1\x30\x8f\x66\x28\x74" + "\x25\x9b\x19\x56\x2e\x70\xf3", + "\xe7\x25\xd3\x1d\xbf\x5b\x99", "\xd7\x67\xf4\x0e\x91\xc4\xf1\x5c", + "\x83\x1a\x38\xcb\xea\xa9\xf2\x2e\xdf\x91\x8e\x97\x19\x56\xc1\x5f" + "\xa3", + "\xc8\xea\xf9\x54\x6a\xf7\x22\x61\x72\x3c\xeb\x3a\xe3\xbb\xb7\x30" + "\x3c", + "\x68\xd7\x28\x74\x4e\x59\x77\x34\x2d\x93\xaf\x81\x44\x58\x57\xed", 1, + 64, 136 }, + { 64, 192, 128, 238, + "\xd7\x45\x99\xb3\xd2\xdb\x81\x65\x3d\xe4\x3b\x52\xfc\x99\x4c\x50\xd0" + "\xbe\x75\x9f\xab\x87\xc3\x3a", + "\xd1\xc6\x1c\xf8\x53\x25\x31\xb5", "", "", "", + "\xbd\x78\xdf\xc8\x04\xa4\x20\xc1\x9f\xb1\x3b\x2f\x58\xd8\x2c\x5c", 1, + 0, 0 }, + { 64, 192, 128, 239, + "\x8f\xc2\x69\xef\x34\xd2\xc2\x12\x7c\x89\x49\x3c\x09\x60\xee\x08\x49" + "\xfa\xdf\x76\x66\x78\x85\xd5", + "\x62\x6b\xf0\x0a\xcb\x93\x04\x80", + "\x13\xaa\x17\x48\xae\xc4\x10\x42", "", "", + "\x9c\x45\x32\x75\xaf\xb0\x06\xc7\x8f\x6f\x29\x07\x9c\x7c\x3e\xf1", 1, + 64, 0 }, + { 64, 192, 128, 240, + "\x0b\x17\x71\x98\xc8\xb4\x19\xbf\x74\xac\xc3\xbc\x65\xb5\xfb\x3d\x09" + "\xa9\x15\xff\x71\xad\xd7\x54", + "\x8f\x07\x5c\xbc\xda\x98\x31\xc3", "", + "\xc4\xb1\xe0\x5c\xa3\xd5\x91\xf9\x54\x3e\x64\xde\x3f\xc6\x82\xac", + "\x01\x48\xcd\xf9\x0d\x56\x6a\x8e\xb6\x51\x40\x99\x56\xc3\x69\x5e", + "\xb1\x0d\x57\xdf\x83\xc4\xc7\x9b\x9f\x59\x0e\x3e\x5a\xa9\xe9\xb6", 1, + 0, 128 }, + { 64, 192, 128, 241, + "\xdb\x86\x9e\x55\x57\x6f\x57\xc8\xf9\x26\x49\x65\x9e\x3c\xb8\xbe\x10" + "\x65\x6b\xbf\xf4\xb6\x94\x60", + "\x67\xc0\x30\x53\x32\xe1\x31\x7b", + "\x93\xf5\x3a\xb3\x6f\x45\xcb\xa3", + "\x00\x2f\xbd\x2e\x0f\x39\xd4\x9f\x25\x8b\x3f\x73\x98\x39\x1e\x2c", + "\x0c\xe4\xb1\xf5\x39\x22\xea\x14\x8f\x26\xa6\x38\xd1\xc9\xe7\x85", + "\x9b\xe9\x41\x7b\x3f\xf9\xe8\xd5\xb2\x4e\x04\x14\x39\xb0\x2c\x86", 1, + 64, 128 }, + { 64, 192, 128, 242, + "\x2a\xf9\x6f\x8c\xca\x1b\x56\x3d\x17\xe7\x96\x9e\x01\x64\x5e\xe7\xb9" + "\xf5\x41\x3a\xc9\x3e\xa5\x70", + "\x95\x8d\x1f\xaf\x8c\x12\x67\xd8", "", + "\xc7\x1e\xd8\x02\x7c\x74\x56\x26\xea\x03\xbd\x25\x62\x8b\x99\xe1" + "\xdd", + "\xfd\x66\xcd\x92\x6b\xea\xdb\xee\x33\xcd\xae\x43\x82\x4f\xba\xcf" + "\xc0", + "\x8c\x8d\xaf\xb2\x02\x7f\x2b\x0c\x03\xf6\x2b\x5f\x9f\xcc\x0a\xd8", 1, + 0, 136 }, + { 64, 192, 128, 243, + "\x5e\xd7\x70\x86\xec\x0d\xa8\xf6\xac\x00\x56\x3a\xd6\xfc\xb8\x50\x05" + "\xac\x40\xf3\x92\x11\xb0\xe4", + "\x4e\x70\xdf\xd4\x9c\xc2\xac\x76", + "\x29\xbf\x75\x6a\x2f\x77\x06\x6b", + "\xc0\x49\xdc\xf1\x5a\xf3\xf9\x75\x98\x7d\x5f\x12\x50\xfe\xf5\x41" + "\x4f", + "\x36\x96\x0d\xa0\x29\xe6\x7a\xee\xb1\x45\xd5\x7d\xc0\xda\x68\xac" + "\x27", + "\x09\x50\x5b\xad\xdf\x28\xcd\x84\x2f\xd7\xfa\x7c\x54\x4d\x0c\x48", 1, + 64, 136 }, + { 72, 192, 128, 244, + "\xf1\x62\xa1\x09\x40\x12\xf6\xbf\xb1\x02\x70\xcd\x56\x09\xa2\x0d\xc2" + "\x4d\xec\x37\x27\xf8\xe5\x98", + "\x05\x62\xf0\x3f\x51\x24\x64\x2f\x40", "", "", "", + "\xa9\xee\xf8\xc9\xf9\x46\x00\x06\xb7\x3f\x2d\xa2\x31\x7c\x7b\x7d", 1, + 0, 0 }, + { 72, 192, 128, 245, + "\x0f\xa5\x86\x1e\xf4\x39\x18\x42\x65\x11\x2c\xa6\xea\x78\x5d\x21\x4a" + "\x5b\xb1\x2d\xd1\x08\xe4\x34", + "\x04\x1f\xfc\xd9\x55\xeb\x49\x39\xff", + "\xbe\xed\x0c\x76\x3b\x56\xc5\x82", "", "", + "\x39\xfd\x1a\x21\x07\x54\x0f\x9e\x6d\x33\xad\x23\xb4\x25\xdd\xef", 1, + 64, 0 }, + { 72, 192, 128, 246, + "\x4f\x58\x9a\xaf\x03\xe1\x21\x95\x85\xf4\x11\x63\x1a\x2b\x28\x7f\x20" + "\xe9\xcc\xa9\x33\x04\xd0\x04", + "\xf0\xbd\x78\x63\xd3\x4b\x6c\x96\x3f", "", + "\xca\x80\xf9\x13\x29\xf1\xcf\xd8\x78\x4b\xdb\x97\xdc\x0d\x5b\x01", + "\x0c\x3f\x7b\x1e\x05\x85\xde\xaa\x80\x0a\x71\x05\xfc\x14\x13\x64", + "\x94\x27\xfd\x74\x87\x0e\x29\xdb\x52\x7f\x7d\xf2\x47\x47\x79\x39", 1, + 0, 128 }, + { 72, 192, 128, 247, + "\x07\x87\x9e\x22\xe8\xc3\xcb\x5b\x5f\xc2\x05\x7c\x39\x85\x90\x6c\x39" + "\xaf\xf4\xe4\x0a\xae\x4e\x20", + "\x4e\xde\x0c\x3a\xf9\xc0\xde\xbb\x8a", + "\xcb\x33\x3d\x66\xbd\xe2\x47\x5d", + "\xb9\xac\x42\xc5\xd3\x16\x90\x87\xa7\x21\x87\x9c\x19\x86\x59\x08", + "\x6c\xf2\xcd\x3a\x10\x61\xd9\xb6\xfb\xe3\x62\x33\x77\xc6\xb4\x43", + "\xf7\x29\x7a\x72\x66\xd2\xf7\xf7\xaa\x7e\xc0\x5e\x0f\x9b\xf9\xa9", 1, + 64, 128 }, + { 72, 192, 128, 248, + "\x0b\x14\x4f\x06\x68\xff\xd1\xa9\x7f\xf2\xd2\xbf\x93\x44\xef\x0e\x28" + "\x48\x96\x4a\xec\xb2\x85\x0f", + "\x3b\x7f\x06\xb4\xba\x5b\x0b\x71\xec", "", + "\x60\x0b\x5c\xa3\xe8\xcf\x20\xa0\x9f\xf7\x52\xec\x2e\x73\x78\xac" + "\x1f", + "\x58\xd5\x76\x23\x17\xcf\x50\x24\x62\x71\x59\xac\xe6\xb4\x8f\x79" + "\x7f", + "\x16\xee\x7c\x4a\xad\xf2\x58\x45\x80\x30\xe5\xaf\x1b\xdb\xed\x2d", 1, + 0, 136 }, + { 72, 192, 128, 249, + "\xa3\xbd\xd0\x65\xfe\x64\x75\xdf\x94\xa2\x09\x2c\x3f\x72\xb1\xdc\xd3" + "\xd0\xf0\x41\x3b\x4f\x34\xdc", + "\xfe\xeb\x53\xf7\xcd\x16\xad\xc8\xe4", + "\x6b\xa7\x08\x2e\x39\x8b\xab\x61", + "\xd1\xe7\x61\x64\x72\xca\x17\x01\x5e\xea\xea\xc3\x0b\x5b\x22\xf0" + "\x07", + "\x95\x38\xe5\xa7\x0a\xc3\x3a\xd0\x92\x4f\x03\x8b\x34\xd1\x99\x5b" + "\x7b", + "\xcb\x45\x9d\x32\xbb\xfd\x20\x93\xeb\x4d\x79\x33\xd5\x0f\xfa\x27", 1, + 64, 136 }, + { 80, 192, 128, 250, + "\x17\x2f\x22\xf2\xe5\x93\x64\xdc\x41\x8c\xd7\x51\xdf\xa8\x44\x4a\xe1" + "\x86\x44\xc0\xf9\xa2\xbe\x84", + "\xbf\x90\x26\xd3\xdd\xaa\x37\xe7\xf1\x80", "", "", "", + "\xb0\x77\xff\x4f\xad\x9f\xf4\xa9\x4b\x6d\xe8\xa6\x6b\xa5\xb1\x6c", 1, + 0, 0 }, + { 80, 192, 128, 251, + "\x14\x3e\xfb\xf8\xe0\x29\x3d\xd4\xc1\x31\x59\xcf\x26\x0e\xc5\x91\xf5" + "\xf9\x2b\xb3\xaf\x8d\xd8\x63", + "\x11\x1a\x95\xbb\xb6\x0f\x9a\x3b\xba\x53", + "\x51\xc1\x46\x78\xc4\x54\x47\x77", "", "", + "\x09\x9a\x7c\x50\x90\x44\x3c\xd4\x00\x0f\x97\x0d\x42\xbc\xd1\xd5", 1, + 64, 0 }, + { 80, 192, 128, 252, + "\x4c\x41\x10\x4d\x3f\x40\x26\x5f\x9e\x35\xc3\x20\xa0\x1e\x78\x76\xc3" + "\x14\x00\xa0\xbd\x4d\x70\x92", + "\x85\xfd\xc8\x1a\xfd\x9f\x48\x28\x17\x7e", "", + "\xba\x7c\xd0\x7d\xfd\x8b\x5c\xf6\xff\xd3\xdd\xb7\x63\x56\x12\xc6", + "\x38\x6b\x63\x4a\x5d\xef\x89\xdc\x73\x02\x72\x4a\xd1\x19\x21\xfe", + "\x4d\x79\x22\x01\xa9\x98\x88\x94\x57\xb4\xc8\x3c\xab\x0e\x5c\x35", 1, + 0, 128 }, + { 80, 192, 128, 253, + "\x8b\xcd\xe5\x17\xdd\xc6\x3f\xed\xcc\xe8\xe3\x41\x81\xf2\x35\x30\xf4" + "\x71\xd6\x85\x8c\x48\xdb\xf9", + "\xbc\x6d\x51\xde\x0c\x0b\xe7\xc4\x59\x11", + "\xe0\xb3\xfb\x36\xc7\xb1\x63\x41", + "\x1b\x42\x19\x8b\x4a\xc0\x82\x24\xe1\xe7\x61\xa7\x72\x05\xe3\x92", + "\xe8\x73\x99\x72\xe4\x18\x0e\x2e\x52\x01\x21\xd8\xe9\xaa\xd7\xc5", + "\x6c\xee\xbc\x4d\x20\x29\x45\x38\x3e\x51\x1b\x7c\xad\xde\x56\x95", 1, + 64, 128 }, + { 80, 192, 128, 254, + "\x37\xf0\xb2\xd5\x3d\x52\x40\x7e\xb9\xff\x33\x53\x0e\x74\xb4\xed\xf5" + "\x82\x5a\x7b\xb3\x7c\x3d\xc5", + "\x30\x3c\x76\x67\x53\x01\x1b\x63\x55\x44", "", + "\x1e\xd6\xab\xb2\xf2\x83\xca\x7f\xde\x5d\xe6\x62\xbd\x70\x58\xa1" + "\xea", + "\x75\xa6\x0d\xf0\x77\x8e\xb9\x3a\x34\x07\x2e\x74\xfa\x3d\x6b\x02" + "\x24", + "\xf2\x7b\x8c\xfc\x5b\xf5\xd2\xb4\xb4\xd9\x3f\xb5\x84\xd7\x19\xff", 1, + 0, 136 }, + { 80, 192, 128, 255, + "\x10\xa7\x79\xb2\x45\x74\x1f\x1a\xb3\x12\x4e\x0e\x50\x4f\xdc\xd3\x15" + "\x78\x4c\x67\xd0\x13\x6f\xcb", + "\x75\xa4\x46\x16\xee\x96\xb3\x0c\x9e\xca", + "\x4c\x5d\x64\x71\xac\x20\xdf\x18", + "\x57\xd7\x58\xf9\x24\xa6\xea\xef\xe4\xd2\x62\x59\x31\xfc\x84\x71" + "\x07", + "\xab\x7b\x01\xba\x57\xed\xc0\xa4\x1b\x19\x0f\xb0\xf1\xd5\x18\x6c" + "\xd5", + "\x3e\x93\xa4\x5f\x5c\xc5\xea\xca\x21\x50\xdb\x35\x34\xa8\x90\x3e", 1, + 64, 136 }, + { 88, 192, 128, 256, + "\x87\xc5\x5b\x2f\x18\x5f\x17\x7f\xaa\xf4\xb1\x6d\x93\xaf\x6d\xad\x47" + "\x71\x46\x34\x5d\x0e\xa9\x92", + "\x09\x46\xc6\x99\x53\xf4\xb9\x52\xbc\x7c\x23", "", "", "", + "\x25\x7b\x3c\x59\x7e\xcd\x1d\x67\xc3\xdd\x35\xdc\x70\xc6\x8e\x48", 1, + 0, 0 }, + { 88, 192, 128, 257, + "\x82\x07\xe8\xd5\x7d\xcc\xdf\x54\x80\xf7\x02\xc1\xfa\x72\xd0\xc6\xd0" + "\x2f\x1b\xad\xc6\xfc\x08\xc5", + "\xc1\x8e\x46\xa7\x0c\x59\x29\x80\xa2\xcc\xc2", + "\xbd\x2e\x2a\x9d\xa3\x2a\x9d\x67", "", "", + "\x11\x45\xc2\x0b\x7f\x31\xd5\x7d\x45\x8a\xfc\x65\x0a\x6d\x45\x90", 1, + 64, 0 }, + { 88, 192, 128, 258, + "\x2c\x2f\x7e\x8b\xb7\x5b\xa9\x31\xa7\x11\xec\xa4\xd3\x19\xe1\x9a\xd8" + "\x97\x67\x24\x8f\xee\x53\x60", + "\x21\xa1\x04\x56\x47\x0d\x08\x3c\xa7\xbd\x7c", "", + "\x1e\x9f\x46\x74\x41\xe4\x87\xbf\x68\xd1\x0b\xe8\x53\xb2\x44\x79", + "\x18\x6b\x50\xf0\xed\xf7\xb5\x23\x02\x13\x84\xf5\xd8\xc0\x90\x49", + "\x8d\xb0\x12\x36\xb7\x15\xa7\x64\x32\xfc\xb0\x2c\xff\x2f\x6b\xa6", 1, + 0, 128 }, + { 88, 192, 128, 259, + "\x0a\x3b\xf0\xa9\x26\xfb\x14\xa3\xb7\x16\xbf\xa0\x21\xa2\x08\xda\x03" + "\x30\xe5\x7b\xed\x36\x82\x8a", + "\xa0\xba\x36\xed\xc4\x3d\x93\x5e\xe9\x42\x13", + "\x3f\xcd\x93\xcc\xb8\xe9\x79\x56", + "\x9a\xb6\xc1\x09\xc8\x06\x9d\x05\x4c\xcb\xb5\xc3\x3c\x6e\x70\xd0", + "\x27\xd1\xe3\x53\x23\x37\x55\xa2\xfe\x28\x23\x16\x37\x73\x9c\x46", + "\xb9\xb7\x47\x65\xed\x3d\x53\x03\x1b\xf3\xc7\x34\x9e\x74\x34\x0e", 1, + 64, 128 }, + { 88, 192, 128, 260, + "\xa6\x8d\xfe\x22\xce\xf2\xbf\xf0\xf2\x8d\x4b\x68\xc2\xa6\x93\x8b\x16" + "\xdc\x21\x09\xba\xb0\x9c\x38", + "\xfe\x76\xa0\x3b\x77\x0b\x43\x1d\xc6\x87\x2b", "", + "\xf5\x56\x91\x55\x30\x58\x00\xbc\x94\x18\x4b\x1e\xf1\xc1\x52\xe1" + "\x97", + "\x34\x28\x2b\x16\x48\x9e\x7b\xc7\x13\x6a\x24\x98\x32\x8b\xd2\x2e" + "\x76", + "\xe9\xb7\xae\x57\xe2\xb7\xf6\x0d\x09\xf5\x0b\xed\x23\xb9\x34\x38", 1, + 0, 136 }, + { 88, 192, 128, 261, + "\xff\xe5\xd2\xad\x71\xd4\x32\xd6\xcd\x5f\x10\x72\xec\x2a\xcb\x7d\x7c" + "\xde\x9c\x5c\x61\x5d\x0e\xb9", + "\xea\x21\x98\x30\x74\x02\xa1\x06\xea\x42\x93", + "\xf9\x3d\xb4\xf4\xae\xc8\xaf\xe8", + "\xc9\xdb\x4d\x10\xd4\x23\x40\xac\x73\x62\x71\xed\xf9\xf6\x58\x1c" + "\xe8", + "\xbd\x96\x0c\x7f\x60\x39\x2e\x8d\x0a\xfb\x28\xba\x16\xea\x63\xaf" + "\xd3", + "\xf2\xca\x88\xd5\x49\xdf\xf2\x07\xd9\x79\x75\x6d\x13\xe8\x65\xd9", 1, + 64, 136 }, + { 104, 192, 128, 262, + "\xd4\x65\x54\x4b\x5d\xb9\xeb\x1c\x49\x5c\xaf\xaf\x5d\x9a\xc7\xe1\x0f" + "\xaa\xe7\x45\x41\xa0\xa7\x18", + "\x95\x82\xaf\xc3\x05\x56\xca\x12\xd1\x54\xc4\x2f\x03", "", "", "", + "\x37\xf6\x18\xf8\xbb\x7f\xf8\x5e\xc6\x44\xb1\xcb\xcc\xa4\xc2\x8a", 1, + 0, 0 }, + { 104, 192, 128, 263, + "\x6f\x1c\xce\x6a\x35\x3a\xa4\x5f\x92\x6f\xac\xbb\x68\x65\xd3\x59\x82" + "\x60\xdb\x53\x90\xe9\x37\xad", + "\xae\x0f\xe0\x77\x39\x85\x87\x74\x7a\x64\x2e\x54\x22", + "\x32\x66\x99\xf5\x6a\xc2\x8d\xef", "", "", + "\x07\x52\x0b\x38\x4c\xcc\xbb\x80\xb9\x67\x9a\x0e\xf4\x8c\xb6\xa8", 1, + 64, 0 }, + { 104, 192, 128, 264, + "\x36\xf9\x7a\x97\xd1\xdd\x67\xe5\xf8\x3c\xcb\x52\x9d\xa2\x5a\x60\x4b" + "\x68\xb8\xda\x90\x4f\xe3\xf6", + "\x23\x93\xa0\xa0\xe0\xb8\xef\xdd\x59\xdb\x34\x36\xdc", "", + "\xc0\x2f\x28\x77\x32\x33\xff\xca\x81\x2e\xaf\x1c\x94\x6c\xd8\xd1", + "\xd4\x12\x86\xc4\x61\xfb\x65\xd4\x10\x66\xa1\x03\x88\xeb\x69\xc8", + "\x07\x36\x96\xdf\xfa\x20\x71\x44\x00\x14\xdf\xac\x4c\x6c\xad\xb2", 1, + 0, 128 }, + { 104, 192, 128, 265, + "\xf2\x58\xd3\x3f\x1f\x1f\x3a\xad\xe5\x10\x3d\x56\xc4\x35\x7b\x7a\x4f" + "\x8d\xd2\x05\xe4\x60\x65\x8e", + "\x14\xa9\xed\x95\x39\x52\x5f\x54\x0d\x9a\x46\xaf\x69", + "\xb6\x71\x96\xee\x87\x89\x0f\x55", + "\xf2\x22\xfd\xfd\x34\x3b\x57\xa7\x0d\x00\x2d\x14\xa3\x9c\xae\x59", + "\xd2\x7c\xda\x7a\xda\x56\x38\xdb\x59\x94\x5a\x31\xd9\x3e\xf2\x43", + "\xc2\xc6\x4d\xd8\xb0\x8e\x7b\x4b\x1c\xe6\x0d\x5b\x96\x83\x29\x89", 1, + 64, 128 }, + { 104, 192, 128, 266, + "\xee\x79\x41\x97\xf2\x0e\x64\x3c\x38\x77\xad\x08\x5f\x03\x1c\x75\x0f" + "\xf2\x32\x56\x8e\x53\xd7\xc3", + "\x31\xea\xac\xed\x4a\x01\x42\xb6\x45\x5c\xf7\x16\xe9", "", + "\x77\x2c\x05\xb2\x37\x7b\xe0\xb3\xbd\xfd\x9a\x35\x7c\x27\x66\x08" + "\xb0", + "\xe0\x6e\x98\xbf\x61\x2f\x13\x44\xfa\xc0\xc3\xef\x8d\x3a\x65\x6b" + "\xa0", + "\xfc\x26\xa2\x6f\x23\x14\xe5\xa2\x79\xc2\xc7\xd0\x7c\x04\x45\x85", 1, + 0, 136 }, + { 104, 192, 128, 267, + "\xb1\xbb\xea\xe5\xfb\xa3\x04\x41\xe1\x2b\x1a\xd2\xf7\x4e\x27\x2b\xc2" + "\x05\x22\x1f\xe3\x4a\x34\x95", + "\x99\x6c\xf4\xb0\xed\xed\x6a\xf6\x6c\xef\xfb\xe8\xb7", + "\xb3\xc3\x37\xb6\x58\x59\x6f\x4c", + "\x39\x11\x39\x00\xd2\x87\xd9\x0c\x54\x01\xd2\x19\xaa\x52\x82\xb9" + "\x1e", + "\xd1\x92\x8c\xe8\x58\x77\xf1\xd1\xfd\x56\x96\xe5\x6b\xb5\x05\x91" + "\xe7", + "\x7b\x85\x27\xe9\x81\x92\xd7\x11\x1d\xaf\xff\x55\x17\x82\xf7\x01", 1, + 64, 136 }, + { 56, 256, 128, 268, + "\x49\x18\x28\xf2\xdd\xdc\xff\x5f\x96\x6e\x66\x27\xf4\xb6\xa8\x5a\x2e" + "\xa7\x6f\xd1\xe0\xb6\x11\x7a\x13\xe9\x4d\x0e\x81\xc0\x63\xa5", + "\x4f\xee\xdf\x9d\x9c\x07\xe0", "", "", "", + "\x1c\xa0\xa4\x18\xf3\x37\xa4\xc0\x4f\x21\x23\xfe\xfd\x31\x79\x6d", 1, + 0, 0 }, + { 56, 256, 128, 269, + "\xa5\x79\x05\xb9\xeb\x31\xfb\xb1\xcc\x53\x96\x39\xe6\x70\xb2\xf1\xd1" + "\x2e\x27\x71\x39\xb5\x1a\x09\x8c\xfe\xbc\x18\x20\xfb\xa1\xa4", + "\x27\xcf\xf7\x6e\x28\xc6\x13", "\xdc\xd2\xf8\x4e\xd0\xea\xfa\xd0", + "", "", + "\x0b\x21\xf5\x0e\x20\x6c\x07\x21\xc6\xc0\x59\xf9\x20\x7e\x6d\x3a", 1, + 64, 0 }, + { 56, 256, 128, 270, + "\x55\xfa\x96\xeb\x3c\x94\x5c\xab\x67\x6c\x42\xb8\xca\xac\x34\xa1\x71" + "\x7d\x43\x37\xf4\xc9\x08\x06\xb2\x26\xd5\x68\x12\x1e\xc5\xe0", + "\x20\x97\x98\x00\x6d\x01\x2c", "", + "\x34\xef\x60\x3e\x3c\x8f\x93\xa0\xe4\xa4\x77\x3f\x7b\x57\xac\xea", + "\x79\x7c\xa0\x5b\x20\xa1\x49\xd4\x2e\x5a\xb3\x38\x35\x85\x5b\x5c", + "\x81\x91\xbd\x25\x4a\x6e\x98\x6e\x3c\x22\xe8\x10\x68\x94\xd6\x4f", 1, + 0, 128 }, + { 56, 256, 128, 271, + "\xe0\xd8\x2f\x60\x88\xec\x67\x5d\x92\xec\x6b\x44\xa6\x7d\xc6\xeb\x66" + "\x00\xf1\xb7\x42\xbd\xd5\xa8\x51\xb0\x36\xaf\x02\xee\xf8\x25", + "\x06\xed\xf6\xab\x0c\x7a\x92", "\xe9\x8f\xdd\x29\x22\x91\xdd\x01", + "\x5b\xb3\x63\x92\x65\xc8\x56\x3e\x6f\xb7\x38\xbe\xd8\xc8\x53\x2c", + "\xcb\x25\x13\x41\x7f\x9c\xb5\x46\xd7\x38\x30\xb9\x19\xb2\xcb\x33", + "\xd3\xc0\x6c\x16\x14\xf7\xca\x3b\x09\x52\xd6\x7a\x5b\xd0\xd0\x17", 1, + 64, 128 }, + { 56, 256, 128, 272, + "\x12\x36\x80\xa3\x5c\x43\xcf\x61\x8c\x69\xf2\x81\x29\x81\x99\xe5\x4e" + "\x40\x08\x0e\x16\x57\x7f\x31\x0f\x09\x6e\x36\x7e\xe3\xcd\x40", + "\xfe\xea\x3f\x0f\x2d\x0e\xca", "", + "\x33\xee\x63\x0f\x34\x58\x8d\xc6\x8f\x8f\x43\x9f\xa3\x19\xf4\xef" + "\x1e", + "\x0b\x9f\x38\xde\xa9\xdd\x82\x65\x6f\xc1\xc2\xe1\x65\x1b\x12\xe1" + "\xc1", + "\x03\x79\x8b\x2b\xaa\xf5\xaf\x45\xe6\x7a\x71\x6c\x7b\x2a\x2a\x17", 1, + 0, 136 }, + { 56, 256, 128, 273, + "\x7b\xeb\x1e\x06\xb5\x85\xfa\xda\x87\x5f\xc6\x10\xc3\xcb\xfb\x97\x88" + "\xfe\xa2\x91\x43\x64\x10\x48\x7d\x8a\x84\x4c\x21\x7d\xfb\xb7", + "\x37\xaf\x09\x74\xce\x28\x51", "\x23\x30\x13\xd9\x17\xf3\xad\x76", + "\xe0\x39\x63\x76\xc6\xe7\x4a\xaf\x27\xf9\x33\xb6\xd5\x9f\x1b\xcf" + "\x8c", + "\x81\xf8\x49\x9c\x64\xed\x65\xe4\xd9\x96\xf8\xb2\xc6\x48\x4d\xe1" + "\xe2", + "\x6d\xb4\xb7\xc5\x5f\xcc\xa5\xfe\xde\xe9\x71\xa4\xa1\x22\xbb\xfe", 1, + 64, 136 }, + { 64, 256, 128, 274, + "\x61\xba\x69\x48\x97\x92\x5d\x1b\x41\x74\xd4\x04\x01\x46\x9c\x3e\xf2" + "\x67\xcd\xb9\xf8\x29\xed\xb1\xa1\x06\x18\xc1\x6d\x66\x60\x59", + "\x0d\x10\xc5\xc8\x4b\x88\xd6\x88", "", "", "", + "\x8e\xc7\x1f\x7d\xaf\x93\x5e\xdf\xac\x9d\xe9\x68\xf1\xd7\x64\x77", 1, + 0, 0 }, + { 64, 256, 128, 275, + "\x30\xb7\x84\x51\x11\x93\x55\x5f\x16\x11\x23\xac\xad\x2f\x18\xae\x3b" + "\xde\x91\x2e\xa9\xcc\x4a\x9e\x55\x31\x6d\x82\x2e\xce\x96\x52", + "\x0f\x1d\x38\xc6\xf3\x0b\x44\x75", + "\xd2\xcd\xd6\x22\x80\x88\x8f\xe5", "", "", + "\x02\xfc\xa4\x1f\x06\xb8\xc5\x43\x80\x84\x44\x0f\xf4\xce\xa5\xc8", 1, + 64, 0 }, + { 64, 256, 128, 276, + "\x11\x58\x84\xf6\x93\xb1\x55\x56\x3e\x9b\xfb\x3b\x07\xca\xcb\x2f\x7f" + "\x7c\xaa\x9b\xfe\x51\xf8\x9e\x23\xfe\xb5\xa9\x46\x8b\xfd\xd0", + "\x04\x10\x21\x99\xef\x21\xe1\xdf", "", + "\x82\xe3\xe6\x04\xd2\xbe\x8f\xca\xb7\x4f\x63\x8d\x1e\x70\xf2\x4c", + "\x74\xc3\xb0\x03\x22\xc0\x91\x60\x80\x37\xd4\xa8\xeb\x5a\xfb\xec", + "\xa0\x98\xb6\x7a\x2c\x79\xdd\x93\x94\x72\xa1\x85\x02\x63\x27\x01", 1, + 0, 128 }, + { 64, 256, 128, 277, + "\xa6\x74\x2d\xd3\x38\x7b\x1e\x11\xdc\x00\x48\x34\x71\x20\xf9\x17\x6d" + "\xff\x30\x29\x5c\x03\x41\xd6\x9b\xc2\xde\xac\xe1\x93\x3f\xd8", + "\xbd\x3a\xbd\x10\x1a\x6c\x62\x5e", + "\x61\x51\x54\x63\xb6\x84\x95\xbd", + "\xb9\xbe\x89\xba\x08\xc5\x5a\xc0\x44\xb6\x10\x9b\xc4\xa1\xeb\x6b", + "\x2f\xbb\xa1\xef\x88\x55\x54\x5c\x67\xcf\xc5\x3e\xd4\x9b\x37\x24", + "\x1a\x82\xcc\x39\x05\x01\xd2\x99\x15\xc2\xc1\x9a\xf0\xb5\xae\x53", 1, + 64, 128 }, + { 64, 256, 128, 278, + "\xc4\x3a\x2f\xa6\xd3\x71\x17\xc1\xad\x70\xcf\x07\xcd\x5d\x60\x7c\x91" + "\x3c\xa8\xfa\x55\x84\x80\xaa\x0a\x24\x13\xe3\xd6\xe8\xb1\xaf", + "\x83\x9a\xe2\x4f\x13\xf2\xea\xba", "", + "\x2c\xf8\x40\x59\x46\xbb\x72\x3d\x40\x66\x62\xa3\x1d\xff\xfd\x51" + "\x41", + "\xdb\x8c\x02\xd3\x79\x87\x60\xbb\x40\x38\xd3\x70\xab\x6a\x93\xe4" + "\x51", + "\x3d\xd4\x24\xa6\x17\x50\x2b\x64\x48\x4a\x88\x95\x7f\xf0\x94\xa7", 1, + 0, 136 }, + { 64, 256, 128, 279, + "\xf9\x48\x9d\xda\x8a\x08\xab\x83\x3f\x2a\x65\x8f\x3e\x42\x5a\xd6\x77" + "\x07\xb0\xa5\x29\x11\x08\x16\x22\xe0\xe7\xef\x90\xa3\x3e\x84", + "\x6a\x1b\x55\x7a\x0f\x47\x08\x22", + "\xec\x6c\x76\xbc\xee\x1e\xbc\x6b", + "\xa5\xd3\x97\xbe\xbe\x7a\xc5\x70\xd2\x39\x93\x90\xe8\xf0\xec\xb2" + "\xb6", + "\xa4\xe3\xe0\x95\xa2\x00\x41\xae\x21\x7a\xcf\xfd\x45\x5a\x74\x2d" + "\xb5", + "\x31\x7b\xc9\xf1\xb5\x20\xe9\x8e\xd8\x82\x0d\xd2\x40\x29\xab\x52", 1, + 64, 136 }, + { 72, 256, 128, 280, + "\x66\xf6\xd7\x9b\x72\x3c\xcd\x31\x36\xd2\xcf\x78\x8f\xc5\xb1\xc2\xf4" + "\xb9\x84\x63\xa5\x7a\xe4\xdd\x29\xf3\x88\x8a\xba\x37\xd0\x86", + "\x0a\x0a\xab\x42\x30\xfc\x3e\xe8\xca", "", "", "", + "\x2e\x59\xd7\xb1\xd1\xad\xa4\xc5\xf4\xc7\x4b\x35\x39\x66\x87\x99", 1, + 0, 0 }, + { 72, 256, 128, 281, + "\xdf\x89\xe9\x4e\x19\x79\x57\x6e\xb8\x6b\x48\x19\xc9\x02\xaa\x5c\xdd" + "\xfd\x14\xe0\x22\x45\x48\xc0\x35\x31\xea\xa7\x9e\x9a\x22\x64", + "\x39\xd9\x12\xcc\x1c\xd3\xde\x7f\x18", + "\xfb\x13\x08\xe9\x08\x2d\xba\x57", "", "", + "\xdd\x8b\x28\x4b\x1b\xa7\x18\xff\x14\x9b\x29\xc0\xbe\x62\xe7\x08", 1, + 64, 0 }, + { 72, 256, 128, 282, + "\x3e\x67\x83\x07\x50\x9e\xa3\x1e\xd5\xf3\xbe\x53\x2b\xa6\x1a\x4f\x03" + "\xbc\x8e\x13\x75\x11\x36\x41\xd1\x09\x98\xb5\x0d\x1e\x42\xe3", + "\xc0\x63\x66\x67\xb3\x31\xa0\x81\x13", "", + "\x32\x0d\x03\x28\xd2\x16\x4a\xfc\xfb\x89\x92\x65\x93\x8b\xae\x67", + "\x7c\x56\x07\x16\x00\xb1\xa3\xc6\xd8\x7a\x4e\xd8\xbe\x56\x18\x7a", + "\x58\xc8\x51\x26\xd5\xa5\x29\x1b\x48\xa9\x39\x55\x6c\xa8\xf3\xd6", 1, + 0, 128 }, + { 72, 256, 128, 283, + "\x7c\x9b\x18\x43\x5f\x55\x63\xe0\x35\x05\xa6\xf5\xed\xfc\xb1\x04\xde" + "\xda\x40\xec\x89\x99\x8f\x68\x16\xe1\x08\xda\x97\x04\xcd\xda", + "\x98\x63\xce\x13\x79\xa0\x6a\x5d\xef", + "\xf0\x0d\xc0\x5b\xd0\x00\xfc\x70", + "\xb0\x37\x68\x45\xc0\x26\x97\x93\x5f\x91\x43\x98\x55\x5e\xc4\x27", + "\x49\xd7\xd0\xb7\x96\xd1\x6d\xff\x6d\x0f\x7a\xab\x8c\x02\x27\x76", + "\x6a\x25\x90\xae\xc1\x3a\x76\x5e\xd7\x73\xf4\xde\xd0\xf1\x21\x86", 1, + 64, 128 }, + { 72, 256, 128, 284, + "\x94\x33\x11\xd4\xa1\xf7\xd2\x11\x08\xcc\xce\xe9\x40\x35\xdd\x71\x7f" + "\xc3\xab\x41\xd7\x3c\x36\xc2\xff\xbc\x01\x7f\x82\x22\xe8\x57", + "\x82\x33\x9e\x77\x61\x51\x3c\x74\xa7", "", + "\xb6\x8f\x03\x3c\x45\xc6\x72\xb6\x96\xc0\x32\x07\x67\x4b\x39\x5b" + "\x89", + "\xc7\xa2\xfe\x3a\x9f\xcf\x3b\x0b\x5a\xd8\xdc\xc3\x00\xc4\x9b\xa4" + "\x85", + "\x26\xa6\xc3\x71\x43\x86\xa9\x70\x56\x02\x0d\x4b\xf2\x4f\x7a\xa4", 1, + 0, 136 }, + { 72, 256, 128, 285, + "\xb3\x6a\x33\x80\xf9\xbf\xce\x99\x2d\x15\x5e\x18\x47\x3e\xef\x8c\x7e" + "\xee\xd8\xc4\xfe\x8f\x54\x47\xa5\x5f\xfe\x88\xdd\xf3\xbb\x9e", + "\xa2\x15\x98\x49\xb3\x9d\x86\x28\x52", + "\xbd\x75\x19\x2f\xba\xcb\xef\xe6", + "\x3e\x27\x32\x60\x92\x43\x55\xf5\x94\x89\x64\x60\x80\x87\x0f\x19" + "\xda", + "\x19\x5b\xcb\x64\xad\x94\x74\xf8\x3d\xd1\x65\x9d\x47\xc2\x2a\x02" + "\x82", + "\x80\xa8\x3e\x5a\x8c\xc7\x44\xbc\xb3\x22\xab\x07\x17\x39\x52\x93", 1, + 64, 136 }, + { 80, 256, 128, 286, + "\x44\xab\x20\x4d\x15\x0a\xdb\x17\xf8\x3d\x1e\x52\x05\xb6\xe1\x41\x96" + "\x73\xfa\xde\xe6\x10\xfb\x9a\x38\x18\x5a\x96\x74\x10\x21\xeb", + "\xff\x39\x14\x98\x2b\xe3\x0b\x3b\x21\x12", "", "", "", + "\xf5\x00\xce\xf3\x10\x41\x0d\x89\x40\xcf\x34\x90\xf5\xf3\xb5\xd7", 1, + 0, 0 }, + { 80, 256, 128, 287, + "\xdd\xee\x6a\x7b\x13\x1d\x31\x27\x5e\xc1\xcb\x35\x65\x4f\x9d\x25\xc3" + "\x94\x98\x0a\x1d\xda\x37\xf7\x0a\xf0\xfb\x62\xdd\x77\xa9\xde", + "\x06\xd8\x4b\xae\x11\x70\x8c\x42\x80\x23", + "\x04\xc1\x27\x1e\xf5\x2c\x04\x1b", "", "", + "\xba\xc4\x36\xba\x98\x5f\xdf\x3f\x14\x44\x6b\x92\xdd\xf3\x5d\xd1", 1, + 64, 0 }, + { 80, 256, 128, 288, + "\xd3\xb4\x4b\x8d\xfc\x35\x30\x40\x4a\x63\xb3\xca\x04\xcc\x71\xcf\xc7" + "\x1a\x55\x38\x44\x8b\x26\x25\xc9\x81\x85\x6c\xb7\xda\xed\x0f", + "\x7c\x3c\x42\xfa\x17\x34\x7e\x1d\xf7\x97", "", + "\x1d\x17\x75\x57\x96\x56\xf7\xf6\xc6\x89\x14\x01\xd7\x33\xe2\xab", + "\x61\xd6\xde\xba\x72\xb4\x1e\x9d\xa6\x25\x9f\xa8\x05\xd7\x7e\xb9", + "\x51\x3f\x57\x31\xbd\x00\x0c\xe6\x8a\x6e\xaa\xdf\x3c\x92\x53\x5a", 1, + 0, 128 }, + { 80, 256, 128, 289, + "\xfe\xec\xec\x22\x5f\xcf\x20\x09\x38\x18\x88\x09\x94\xfe\xda\xd5\x3d" + "\xc0\xc1\x74\x3a\xa9\x96\x71\xce\xfe\x29\x29\xa5\x03\xe0\xc6", + "\xab\x8d\x23\x83\x0b\x91\xdc\x6a\x89\x8e", + "\x24\x88\x9b\x2e\xf1\x2a\x31\x8a", + "\x7b\x70\xe1\xbf\xe1\xa7\x76\xe8\xf4\x4c\xa4\x32\xdd\x9e\xf9\x99", + "\x46\xf6\x07\x30\xdb\x04\x13\x36\xcc\x05\x1d\x4e\xf4\xde\x02\x9d", + "\x77\x17\xad\xcf\x7d\x3e\xd0\xf7\x87\x8e\x1a\xc3\x3a\x35\xcb\x47", 1, + 64, 128 }, + { 80, 256, 128, 290, + "\xcf\xb7\x33\x08\xa8\x30\x90\x16\x1f\xed\x74\x33\x68\xf5\x48\x08\x72" + "\xea\xa6\x2d\xf5\xa8\xea\x07\x7d\xee\x54\x0f\xd5\xa2\xef\x15", + "\x1d\x37\x5b\x8e\x07\xc3\xc4\xde\x82\xf8", "", + "\xfa\x0b\xc3\xe2\xcb\x70\x18\x3c\xd5\x6f\x47\xfa\x12\x91\x30\x1f" + "\x47", + "\xd9\x2c\x03\x75\xa4\xdc\xb1\x84\xbc\x90\x25\x15\x85\x06\x1d\xb0" + "\x36", + "\xa0\xdd\xef\x0c\x6b\xd9\x4f\xcd\xec\x39\xa7\xf0\x7f\x0a\x2c\x13", 1, + 0, 136 }, + { 80, 256, 128, 291, + "\x81\x5b\xda\xab\x3c\x18\x7a\xd7\x3b\x12\x7c\x8d\x39\xa1\x33\xb4\x1b" + "\x66\xe2\x99\xba\x24\xfb\x44\x6e\x35\xe3\xb1\x12\xdb\x8e\x66", + "\x0c\xd0\x55\x1f\x0c\x74\x17\x60\xb7\x47", + "\xc1\xd0\x1e\x47\x31\xf3\x60\x66", + "\x55\x87\x69\xc6\xd4\xf5\x0a\x0d\xb6\x20\xc2\x3f\xe1\x07\xa7\xfb" + "\xe4", + "\xc1\xce\x19\x2f\xb6\x71\x89\x2b\xb8\x3b\xd2\x2d\xbc\x82\xd6\x40" + "\x82", + "\x4e\x0a\x0b\xe1\xaa\x0e\x75\xcd\x7b\xb1\xdc\xb2\x7a\x01\x09\x10", 1, + 64, 136 }, + { 88, 256, 128, 292, + "\x46\x94\x8c\x6d\x69\x84\x54\x99\x10\x4d\x5d\xc2\xfa\xe4\x48\x80\xcf" + "\xe7\xd0\xbc\xbc\xee\x57\xef\xc0\x13\x3c\x26\x6b\x6d\x26\x21", + "\x05\xf7\x15\xfd\x0a\x56\x03\xdd\x84\xaf\x76", "", "", "", + "\x11\xe6\x78\xc0\xb2\x60\xc5\xa3\xea\x70\xf6\xa4\x6e\x4e\xf4\x36", 1, + 0, 0 }, + { 88, 256, 128, 293, + "\xe6\x89\xeb\x77\xa5\x78\x39\x9f\xa1\x7a\x75\x08\x3d\x25\x01\x8f\xfb" + "\xb6\x8f\x24\xd7\x7a\x02\x97\x57\x54\x1d\x65\x39\xbf\xfb\xdf", + "\x4a\x41\x96\x18\xb2\x83\x2d\xe7\xa4\xf9\x9b", + "\xec\xf4\x6e\xac\xa8\x41\x76\x9f", "", "", + "\x4a\x26\x15\xd2\xd5\xf8\xe9\x7b\x92\x74\x3b\x0e\xf2\xf4\x86\xab", 1, + 64, 0 }, + { 88, 256, 128, 294, + "\xaa\xcb\x13\x36\xd6\xee\x4d\x96\xa9\xa1\x2e\x5b\x8f\x25\xf0\x48\x00" + "\xd4\xaa\xba\x55\xf3\x79\x21\x8d\x64\xed\xb3\x46\x0f\xe2\x15", + "\x99\x4c\x19\x1e\x7a\x29\xc0\xef\xc1\xeb\x4d", "", + "\x7b\xea\xd5\xab\xcc\xc8\x76\xef\xb0\x10\x9e\x41\x2f\x06\xc7\x51", + "\x5a\xa3\x1a\xa5\xeb\x10\x36\x55\xb7\x8c\x4f\x7b\xcf\x08\xc9\x17", + "\x64\xe4\x1d\x31\xeb\x0d\xf8\x0a\xdc\xee\x32\x8f\x08\x1c\x4a\xeb", 1, + 0, 128 }, + { 88, 256, 128, 295, + "\x60\xd6\x84\x1e\x9e\x62\x18\xa2\xc8\x60\x5a\x77\x94\xe7\x4f\xb2\x15" + "\xdc\xf3\xa7\x0a\x00\x15\xd4\x97\xed\x16\x56\x4f\x2a\x83\xa0", + "\x4c\x93\xf5\x91\xaf\x92\xf1\x65\x96\x55\x4e", + "\x7c\xb0\xeb\x9a\xa2\x1f\xe8\x59", + "\xc0\xd1\xe6\x35\x58\x6b\x0e\xf8\x35\xc0\x14\x79\xa3\x21\x75\xa3", + "\x0c\x26\x61\x13\x54\x4d\x7a\x90\x1c\xe7\x21\xe1\xea\xd6\xd8\xf9", + "\x8a\x14\x9e\xaa\x05\xc8\x72\x2b\x26\x63\xc3\x45\xa6\xa5\x41\x8c", 1, + 64, 128 }, + { 88, 256, 128, 296, + "\x9d\xcd\xe5\x7c\xec\x27\xde\x6b\x58\x4d\xb4\xbd\x81\x09\x35\xbd\x3b" + "\x3c\x4e\xa8\xf2\x2a\x16\xa7\xb2\xa6\x2e\xf5\x16\x79\xa1\x3b", + "\x5c\x34\x39\xbd\xca\x45\x7b\x02\xb0\x49\x25", "", + "\x40\x2c\xb1\xfd\x3b\x07\x96\x20\x0d\x88\x8f\x7b\x39\x92\x35\xf5" + "\xb5", + "\x43\xf7\xd9\x9c\x1a\x0e\x50\x4a\xad\xd8\xf8\xb2\x98\x1b\x4a\xea" + "\x52", + "\x95\x8e\x19\xb5\xc1\x4e\xeb\xd2\xd2\x55\x09\x33\x6a\xa6\xd4\xc2", 1, + 0, 136 }, + { 88, 256, 128, 297, + "\xb3\xa0\x6e\x00\x10\x0f\xfc\x42\xdb\xdd\x53\x17\xf4\x3d\x2b\x48\xf8" + "\xc1\x1e\x4b\xd6\xe9\xe3\xed\xab\x58\xe9\x94\x4c\x55\x92\x78", + "\xf3\xd2\x4b\x36\x2b\xf1\x2b\x84\xb8\xa6\x6a", + "\x02\x42\xb5\xe8\x04\xa7\x91\x88", + "\x03\xb0\x3b\x45\xf6\xf3\x20\xb9\x9d\x81\x58\xff\x8b\x00\xf0\xad" + "\x92", + "\xd1\x27\xfd\x42\xb2\xca\x4b\xec\xa9\xf9\xac\x86\xb6\x3a\x16\x22" + "\xef", + "\x8a\x49\x9a\xab\x9e\x8f\x40\x96\xda\x60\x3c\x6c\xcf\xb9\xac\x4a", 1, + 64, 136 }, + { 104, 256, 128, 298, + "\xa4\x99\x4b\x65\x14\x35\x36\x70\x7b\x15\x1e\xe6\xe7\x9e\x69\xab\x9c" + "\x6d\x73\x00\x08\x19\xfd\x29\x91\xdd\x28\xab\xed\xe6\xb3\xe8", + "\xac\x64\x44\x49\x72\xd7\x78\xd5\x2f\x55\x31\xae\x88", "", "", "", + "\x8f\x35\x30\xc8\xad\xf8\x6e\xbc\x6c\x44\x97\xce\xde\x15\xcc\xd9", 1, + 0, 0 }, + { 104, 256, 128, 299, + "\xf9\x9f\x27\x20\xf0\x3e\xc3\xa9\xd0\xda\xd3\x7e\x3a\x91\x5b\xea\x3a" + "\x11\xcb\xa4\xbb\x0f\x60\xcd\x8f\x54\x2b\x33\x01\x63\xbc\xd0", + "\x7e\x0f\x99\xa0\x48\xb6\xe2\x87\x97\x20\xfe\x43\x18", + "\x94\xba\x97\x7e\x74\x45\x5e\xd8", "", "", + "\xcd\xc3\xbe\xf3\x9e\xa5\x3a\xf6\x80\x19\x9e\x36\x26\x09\xdc\x29", 1, + 64, 0 }, + { 104, 256, 128, 300, + "\x50\x5b\x26\xd1\x66\xa6\xeb\xd3\xdb\x69\xcf\x12\xbe\xe2\x5b\x73\x65" + "\x1d\x0d\x33\x2d\x0f\xb2\x48\xb5\x0c\xe9\xa1\xfb\x3a\x13\xf1", + "\xfb\x04\xf7\xbb\x3c\xd3\x82\xcb\xc0\x89\x37\x19\xaa", "", + "\xae\x8b\xb1\xcb\xc9\x2c\x73\xe7\x3e\x59\xa0\xd7\xa9\xd7\xf5\x28", + "\x43\x35\xb7\x08\xf2\x7d\x1c\x1b\x4d\x6e\x98\x5f\x18\xab\xa7\xc5", + "\x06\x9e\xe6\xf5\x27\x9d\xab\x73\x59\x3e\x11\x44\x0d\x23\x9e\xb8", 1, + 0, 128 }, + { 104, 256, 128, 301, + "\xa6\x93\x8b\x2e\x56\xd5\xdc\x55\x66\x59\x56\x84\x0a\xc6\x90\xc8\xac" + "\x9f\xf4\x21\xcc\x06\x2f\xc3\x42\x09\xf7\x71\x5f\x2d\x52\x6e", + "\xad\x37\xde\x72\xd3\x52\x15\x46\xd5\xff\x51\x46\x2b", + "\x0a\x38\x09\xbc\x56\x3c\x66\x75", + "\x9d\x28\x6b\xcc\x11\x5f\x10\xb2\xca\xa8\xc5\xd8\xda\xa9\x1e\xc7", + "\x4e\xd4\xdb\xc8\xaa\x8c\xf6\x37\x50\x21\xd1\x5e\x43\xc1\xf6\xc3", + "\xbf\xba\x9c\x41\xec\x63\xaa\x29\x6b\x14\x46\xb8\x88\xb6\x25\x1c", 1, + 64, 128 }, + { 104, 256, 128, 302, + "\x45\xe5\x9f\x44\x29\x08\x73\x60\xb1\xb2\x40\xfb\x9b\x59\x1d\x86\x1a" + "\xd4\x93\x68\x8b\x0a\x5e\x8f\x85\xff\xea\x2a\xcf\xf8\x39\x3a", + "\x20\xe8\x93\xf4\x56\x2b\xc1\xc5\x6c\x32\xc0\x0c\xc3", "", + "\x3f\xbc\x33\x8a\xd7\xbb\xd6\x77\x8c\xab\xe1\x34\xa0\x2c\x68\xe5" + "\x3c", + "\x67\x63\x7f\x9c\x9f\x7d\xa4\x1d\x4b\x36\x37\xe3\xa0\x54\x36\x2b" + "\x30", + "\xb3\x46\xd7\x00\xcf\xec\x0f\x2d\x8e\x17\x6d\x3e\xb9\xc5\xbe\xc9", 1, + 0, 136 }, + { 104, 256, 128, 303, + "\x00\x4c\x6c\xa0\x40\x78\xbd\xfd\x55\x7f\x91\x50\x25\xa8\xec\x93\xb5" + "\x36\x8b\x86\xca\xf3\xd6\x57\x43\x2a\x5e\x1d\x1c\xef\x99\x17", + "\x99\x9d\x95\xed\xb9\x25\xe7\x74\x4e\x32\x87\x40\x09", + "\x08\x2c\x14\x33\xbb\x64\xe1\x10", + "\x05\x52\xfc\xb5\x2c\x49\x8d\x91\xb8\x98\x97\xae\x6f\x64\x0e\x1f" + "\x5a", + "\x5b\x47\x08\xb7\x2b\x68\x46\x66\x39\xe4\xb5\xf3\xd3\xda\x1d\x1e" + "\x84", + "\xf7\xf0\xc0\xe0\xc0\x1b\xf7\x72\xcb\xd9\x0c\xb9\x80\x93\xdf\xb8", 1, + 64, 136 }, + { 96, 128, 32, 304, + "\x10\x3e\x85\x9d\x3a\x23\x87\x24\xbf\x85\xb2\x10\x0f\x44\x2f\x1d", + "\xf9\x1d\x64\x78\x41\x61\xfa\xbd\x6c\x96\x2e\x50", "", "", "", + "\x2b\x31\xc2\x1b", 1, 0, 0 }, + { 96, 128, 32, 305, + "\x71\x46\x6d\xc3\x04\x6b\x1e\x6c\x08\x38\xba\x6c\x9e\xf4\x1e\x79", + "\x29\x28\x09\x5b\xd7\x96\x2e\x9e\x60\x24\xa2\xb9", + "\xa6\x17\xcc\xe7\x4d\x04\x39\x90\x05\x97\xcb\x3d\xdc\xfc\x25\xfb", + "", "", "\x38\x3f\x8a\xbc", 1, 128, 0 }, + { 96, 128, 32, 306, + "\x3c\xc9\x38\x04\xe2\xd6\x99\x61\x92\x78\xa9\x41\x38\x9c\xec\x3c", + "\xc7\x75\xdd\xa3\x14\xaf\x64\xc3\x10\xa7\xc1\xd3", "", + "\x12\x4c\xe7\x1e\x08\xc1\x32\x4f\x91\x65\x70\xd5\x33\x03\x29\x19", + "\xf2\x46\x75\x4c\xd3\x2a\x99\x60\xd3\xd5\xe5\x35\x2f\x1d\x73\xc7", + "\x60\xdb\xd6\x76", 1, 0, 128 }, + { 96, 128, 32, 307, + "\x90\x89\xe1\x78\xf3\xf9\x0b\xfc\x0f\x68\xe5\x59\xd3\x38\xc3\x9d", + "\x8b\x2b\xfc\xa6\x47\x75\xb5\x09\x35\xb4\x82\x21", + "\xca\x9b\x40\x50\xb6\xbd\x0f\x0e\xba\xef\xfb\x78\xf2\x4a\x41\x1f", + "\x33\xd9\x02\x09\x3b\xa5\x21\x69\x33\x23\x6c\x08\xfa\x5c\x0c\xb2", + "\xe0\x57\x18\x08\xbf\x38\x9c\x1a\x07\xca\x7e\x5b\xbf\x49\xa1\xff", + "\xcc\x34\x6e\x6d", 1, 128, 128 }, + { 96, 128, 32, 308, + "\x33\xcf\xf6\x80\x61\xe3\xf5\xf9\x41\xc8\xc2\x0c\x89\x60\x8b\x77", + "\xeb\x58\x1f\x66\xcc\xc7\xf1\xda\xa2\x35\xbf\x27", "", + "\x24\xa9\xd8\x95\xf6\x04\x6b\x93\x68\xb0\xb6\xb0\xfb\x39\x6c\xd1" + "\x0e", + "\x12\x1f\x88\xe8\x1d\x27\xda\x5c\x5d\x5c\x9a\xb3\x97\xc7\xb2\x05" + "\xf7", + "\x2f\xa4\x51\x6d", 1, 0, 136 }, + { 96, 128, 32, 309, + "\xc0\x1b\x91\x5d\x2d\x21\x12\x28\x8e\xd0\x4d\x2c\xdd\x38\x9b\xd7", + "\x98\xac\xed\x46\x74\xfa\xeb\xe3\xfd\x48\x81\xcc", + "\x20\x90\xbd\x59\x34\xb2\x0e\x26\xc7\x04\xaf\x9f\x85\xc9\xc4\x10", + "\x63\xda\xb2\xe2\xa2\x2a\x63\xa7\xe5\x50\x46\x67\x63\x45\x55\x93" + "\x4d", + "\x13\xef\xab\xa6\x39\x13\xe7\xa9\xd8\xa4\xfd\x89\xe3\x49\xc4\xa0" + "\xa0", + "\x65\x53\xc6\x47", 1, 128, 136 }, + { 96, 128, 32, 310, + "\xa9\x68\xcf\xd7\xf6\x3f\x3a\x27\x68\x71\xe3\x03\x83\x07\x7d\xe8", + "\x54\xbf\xc8\x2e\x1c\xb5\xc0\xb9\xa6\x5b\x25\x2c", "", + "\x03\x0f\xa6\xb0\xed\xde\x45\xcb\x65\x88\x13\xd0\xd7\x97\xae\xdf\x1b" + "\xa2\x7f\x43\x5d\xf4\xf4\x43\xa3\x46\x9e\xa0\xe4\x1e\x63", + "\x60\xd5\x64\x55\x60\xb0\xfa\x0b\x29\x57\x0b\xaf\xd2\xb6\xe1\x8d\x83" + "\x9c\xea\xf8\x82\x42\xcb\x6c\x7c\x60\x8d\x30\x01\xb7\xcd", + "\xa9\x2b\x60\xf6", 1, 0, 248 }, + { 96, 128, 32, 311, + "\x76\x45\x87\x4c\xc7\xa9\xf0\xe7\x44\x32\x03\xab\xec\x23\x45\x5b", + "\xce\x56\x6b\x86\x6e\xf0\xfd\x3b\x09\x6f\x3e\x9e", + "\x1e\x00\x46\x9a\xd4\x5b\x2c\x24\xcd\xdb\xa5\x29\x85\x16\x9a\xab", + "\xb6\xb0\x50\x21\xae\x99\xe4\xaf\xe0\xec\x92\xc0\x09\xd0\x6c\x42\x86" + "\x02\x0f\xab\xca\x1c\x1a\xc7\x68\xfa\xf1\x84\x50\x61\x91", + "\xea\x62\xf4\x28\x31\x70\x94\x24\xf8\xa8\x13\x83\x02\x47\x7d\x51\x6b" + "\x05\xe3\x1f\x23\xb4\x5e\xf3\x77\x03\x3b\x79\x23\x29\x2f", + "\x77\x4a\xae\x96", 1, 128, 248 }, + { 96, 128, 48, 312, + "\x1a\x85\x2b\x34\x56\x35\x3c\xfd\x21\x72\x6d\x11\x22\x10\x9f\x1d", + "\xbd\xe9\x16\x5d\x65\xf3\x01\xa2\xe4\xff\x1d\x4a", "", "", "", + "\xbd\x22\xf7\x19\x5c\x49", 1, 0, 0 }, + { 96, 128, 48, 313, + "\x44\xb5\x29\x8a\x67\x7b\xaf\xf5\xc3\xa6\x5d\x51\x2a\x65\x19\x92", + "\xcb\xb2\x50\x28\x3f\x75\xa6\x60\x82\xf1\xa7\x85", + "\x19\x03\x1c\x68\x8c\xee\xe8\x4e\x2d\x25\x25\x3a\xcc\xba\xe6\x8e", + "", "", "\x93\x10\xab\x0e\x0d\x1c", 1, 128, 0 }, + { 96, 128, 48, 314, + "\x63\xc7\x47\xbe\x2f\x30\x69\xd5\x00\x15\xf6\x9d\xba\xe0\x98\x76", + "\xbc\x2c\x94\x05\x25\xe5\x14\x40\x98\x15\xab\x19", "", + "\xad\x5c\xa7\x0a\x32\x53\x63\xc3\x4b\x2f\x3d\x5a\x85\x76\xb9\x64", + "\xac\xb6\x2f\x8c\x47\x81\x27\x9d\x5c\x81\xcc\xae\xe4\xf6\x1e\xbe", + "\xcb\xbc\xa0\x32\x69\x50", 1, 0, 128 }, + { 96, 128, 48, 315, + "\xa4\x65\xbe\x21\xf5\xb4\x20\xcd\x39\x00\x9b\x0e\xf8\x9d\xbe\xc2", + "\xd8\xb2\x87\xca\xee\x5a\xf6\x9b\xc8\x95\x45\xe9", + "\xb4\x52\xe6\xc1\x12\x64\x7d\xe6\x74\x24\x9d\x1e\xec\x10\x9f\xfc", + "\xc1\x90\xd1\x27\x03\x34\x01\x6d\xae\xeb\x12\xf0\xdd\xd5\x59\x05", + "\xcd\x9e\x9c\xb0\x1e\x77\x37\xcf\x87\xa7\x36\xa5\x0a\x40\x69\x4b", + "\xae\x86\xc7\x00\x51\x83", 1, 128, 128 }, + { 96, 128, 48, 316, + "\x1c\xb1\x73\xba\x47\x85\xbc\x6b\x72\x8c\x86\x29\x29\xda\xea\x5f", + "\xd7\x6c\xdf\xb5\x95\xc3\xfe\x3d\x7c\xc0\x65\x4f", "", + "\xee\xf6\x69\x1b\xa8\xe2\x28\xb7\xbd\xa4\xb2\x6f\xd3\x53\x95\x07" + "\x57", + "\xe8\xad\x83\x31\x15\x84\xd4\xb4\xbd\xb2\x1b\xa0\xf6\x2f\xbb\x13" + "\xd2", + "\x0f\x8f\x63\x95\x41\x3d", 1, 0, 136 }, + { 96, 128, 48, 317, + "\xa4\xa1\x07\xff\x7a\x4c\x09\x78\xaa\x17\xc5\x51\xff\x9f\x8a\x04", + "\x50\x2d\x4b\xc4\x40\xc3\xf6\x6d\xb3\x9a\x09\xf4", + "\x20\xd6\x43\xe0\xaf\x5c\x67\x3b\xe4\x54\xe5\x31\xd9\x29\x95\xc1", + "\x51\x0d\x64\x55\x1a\x78\xca\x2c\xd8\xd3\x22\xf8\x2f\x6e\x2c\xd6" + "\x17", + "\x57\x85\x87\x6e\x6f\xd0\x45\xa0\xce\xa1\x85\xec\xb0\x75\x10\x2f" + "\x97", + "\x6c\x21\x04\x66\x57\xd9", 1, 128, 136 }, + { 96, 128, 48, 318, + "\xb7\x8d\xd7\x5d\x16\xec\xe4\x9b\xfd\x01\xe8\xf4\xbc\xd0\xd5\x2e", + "\x1f\xff\x0b\x5a\x56\x6f\x3d\x1b\x25\x2e\x51\x66", "", + "\xf4\x38\x00\x03\x59\x44\x8e\xd5\xd7\x91\xbe\xab\x63\x72\x99\xa1\x8c" + "\x9d\xf4\x5e\x6a\x03\x04\x28\xcc\xa6\xcc\x05\xb2\xc2\x5e", + "\x65\xc3\x63\xa0\xcf\x88\xf9\xea\x74\xc4\x7f\x46\x98\x1f\xc9\xa8\x45" + "\x40\x2c\x52\x05\xb1\xd0\xc1\xbd\xb4\x24\x9c\x78\x87\xfb", + "\x39\xa8\x32\xac\x3b\x9f", 1, 0, 248 }, + { 96, 128, 48, 319, + "\x22\x8b\xf7\x86\xa9\x37\x1d\x98\x75\x18\x96\x78\xa4\x0f\x55\xa0", + "\x41\x48\xce\x9b\x64\x72\x28\x75\x1f\x31\x3c\x2d", + "\x33\x90\xa6\x5d\x1a\xce\x02\xbf\x67\x26\x52\x54\xbe\x9c\x34\xd7", + "\xa6\x0a\x55\x5b\xa4\x8a\x06\x5d\xa2\x99\x9a\x45\x26\xce\xce\x66\xe8" + "\x60\x0f\xe1\x20\x96\xdb\x76\x67\x71\xe4\x0f\xcf\x40\xd9", + "\xfd\x06\x73\x7f\x69\x5a\xd8\x7d\x70\x35\x4b\x67\xc2\x40\xcc\x80\xe4" + "\x1e\xea\x60\xf3\x58\x34\xfa\x1c\x86\x43\x9a\x3a\x26\x93", + "\x22\x28\x51\xc9\x6f\xc9", 1, 128, 248 }, + { 96, 128, 64, 320, + "\x5a\xfb\x73\xf3\x7d\x05\x14\x75\x66\xa7\xac\x97\x34\xeb\xa3\xff", + "\x02\x6d\xd1\x25\xc9\x8e\xf1\x50\x7f\x6d\x1d\x15", "", "", "", + "\xa4\xc4\xb1\x36\x62\x5f\x02\x43", 1, 0, 0 }, + { 96, 128, 64, 321, + "\x3c\xf9\x38\x73\x3c\xb7\x6e\x43\x3a\x5b\x5c\xcb\x06\xbe\x34\x21", + "\xa5\xbc\x5b\xd3\x83\xce\x11\x08\x10\x2c\x3c\x7b", + "\xbe\xfd\xff\x73\x13\xd3\x3c\xa6\x39\x8f\x84\xb3\x2e\xf7\x7c\x65", + "", "", "\xd6\x65\xa6\xea\x1a\xc4\x64\x9a", 1, 128, 0 }, + { 96, 128, 64, 322, + "\x22\xed\x64\xb5\xb9\x4a\x3c\x41\x16\xd0\x2b\x4f\xbd\x4e\x58\x81", + "\xf4\x98\xfd\x65\xda\xb2\x34\x52\x0d\xe5\x29\x20", "", + "\x94\xb0\x3b\x07\x77\x2b\x70\x56\x2b\xc7\x29\x50\x5b\x4a\xd4\x26", + "\x4c\x4d\xfe\x97\x11\xb3\x20\x26\x4f\x3a\x57\xec\xdc\xd5\x98\x50", + "\xb1\x3a\xea\x29\x80\x76\x7f\xd7", 1, 0, 128 }, + { 96, 128, 64, 323, + "\xea\x5a\x91\x5f\xd7\xbe\x0a\xaf\x14\xb8\x8f\x5d\xc4\xfd\x71\x9a", + "\xae\xec\xf1\x9f\x7d\x33\x79\xee\x55\xba\x64\x68", + "\x13\x79\x1a\xad\x58\x12\xa3\x62\x29\x1a\x4f\x6d\x63\x68\x7d\x33", + "\xd3\x13\xe0\x9c\xd4\x8b\x06\xf1\x6e\xf9\x17\x8e\x42\x62\x4b\xd0", + "\xf9\xbc\x9a\x66\x18\x6b\x6a\x60\x03\x5d\x14\x4d\xfb\x34\xc4\xaf", + "\x2f\xb6\x37\xff\x91\xd6\xfd\x9e", 1, 128, 128 }, + { 96, 128, 64, 324, + "\x89\x12\x11\x03\xc3\x50\xe2\x9f\x7c\xd5\x80\xf0\x5b\xbf\xea\xac", + "\xf6\xd6\xe8\x02\xab\xdf\x43\x23\x00\x30\xa8\x96", "", + "\x63\x68\x40\xff\xbc\x66\x19\x1b\xc3\x7b\xf2\xe6\xbd\xdf\x28\xbd" + "\xa9", + "\xc6\x91\x20\x62\x54\x8d\xba\x55\xe6\x18\x4e\x8f\x50\x7d\x7f\x9c" + "\x7d", + "\x1b\x30\x0d\xe3\x55\x38\xc2\x52", 1, 0, 136 }, + { 96, 128, 64, 325, + "\x03\xad\x5f\x47\x2b\x97\x8c\x5f\x72\xb7\xb1\xc2\x90\x80\x37\x4c", + "\x77\x0d\xcc\x2e\xa1\xc2\xd9\xf6\xc9\x04\x94\x7d", + "\x97\x2c\x90\xe3\x87\xf0\xaf\x93\x6b\x1c\x9d\xb0\xeb\xfe\xbb\xe9", + "\x78\x47\x05\x11\xca\xf1\x2c\xb8\x82\x62\x80\x92\xbb\x57\x3b\xde" + "\x8c", + "\x3f\xb2\x2c\x2c\x36\x6c\x0a\x46\xba\x16\x40\xec\xcb\x54\x4d\xbd" + "\xd2", + "\x3e\xc7\xc4\x88\x8a\x12\x88\xfc", 1, 128, 136 }, + { 96, 128, 64, 326, + "\x48\x56\xb1\x07\xdb\xbc\xe7\x02\xc7\xcd\xaa\x7e\xc1\x74\x0f\x35", + "\x6f\x41\xac\xab\xda\x1e\x03\x48\xc4\x29\x0f\x0f", "", + "\xd3\x2d\xec\xc5\x5d\xbd\x0c\x08\x91\x6c\x9a\x9e\x3d\x08\x46\xae\x2c" + "\xac\xae\xb1\xba\x0e\x04\xeb\x02\x77\x2c\xf6\xa5\x0e\x46", + "\x2f\x3f\x13\x3c\xa5\x44\xea\xa5\x15\xa1\x6f\x8b\x1c\xf1\x2e\x17\x4a" + "\xa8\x0d\xb6\x08\x26\x8e\xad\x25\xac\xe1\xca\x4e\xef\xed", + "\x2f\xfa\x78\x6a\xdc\x94\xae\x2a", 1, 0, 248 }, + { 96, 128, 64, 327, + "\xc0\x83\x39\xa6\xf8\x0b\x84\xe2\x01\xe3\xd6\x03\x0c\xdb\x3f\x02", + "\x1c\xbf\x2c\xa3\x13\x30\xab\xe7\x49\xdb\x58\x8b", + "\xb5\x35\xa8\x47\xdf\xc9\x62\x01\x2d\x91\x3a\x40\x76\xf5\x8f\x9f", + "\x4f\x9f\xd6\xad\x16\x56\xcc\xe9\x9a\xf7\x46\x99\x60\x07\x3a\x24\x15" + "\x69\xce\x32\xda\xd5\x58\x11\x1b\x50\x30\x60\x53\xa0\xb6", + "\xc9\x1d\x4c\x8b\xf7\xfd\xba\x49\xb8\x70\x01\xfc\x3e\xc9\x5f\x45\x5b" + "\xa3\x2b\xc0\x5b\xa3\x36\xbc\x3d\x58\xf4\xad\x08\xb5\xbc", + "\x34\xd6\x22\xfe\x4b\xa3\xca\xc5", 1, 128, 248 }, + { 96, 128, 80, 328, + "\x1f\xaf\x80\x05\xf7\x75\x53\xf5\xee\x26\x86\x5e\x31\xf5\x08\x7b", + "\x40\xdf\x77\xe5\x37\xc8\x95\xab\x71\x46\x4a\xcc", "", "", "", + "\x22\xff\xed\x9c\x2d\xca\x19\xfa\x32\xef", 1, 0, 0 }, + { 96, 128, 80, 329, + "\xf2\x0c\x79\xf8\x45\xbe\xd4\x06\x46\x9c\xf1\xcd\x3f\x7d\xaa\xc5", + "\xf7\xbc\xa6\x6e\xcc\xd7\xd4\x94\xde\xc7\x58\xf5", + "\x6e\x45\x36\x57\x58\x83\x92\x5a\x92\x9c\xed\x31\xad\x8f\xb6\xdd", + "", "", "\x2f\x25\xa1\xd0\x02\x61\x58\x9f\x3f\x00", 1, 128, 0 }, + { 96, 128, 80, 330, + "\xb8\xba\xe0\x12\x60\xce\xd6\x19\x4e\xf8\xdf\x72\x2d\x65\x9b\xe6", + "\x71\xd1\x0b\x7c\xbb\xbe\xcb\x84\x3e\x67\x8a\xb5", "", + "\x38\x7c\x03\x24\xcd\x47\xd3\xf2\x2c\xc9\xd9\x68\xa7\x2e\x43\x4d", + "\x0c\x36\xe3\x03\xe2\x95\xa2\x89\xbb\x13\x47\x40\xe2\x1a\x66\x64", + "\xd3\x58\x7e\x21\x86\x55\x3f\xd9\xd4\x09", 1, 0, 128 }, + { 96, 128, 80, 331, + "\xb8\x00\x30\xb8\x3c\x4b\xca\xfd\x1b\x7e\xc9\xc7\x0a\xb9\x22\x4c", + "\xa7\x89\x45\x7f\x80\xbd\xc5\xb8\xf1\x5f\xea\x91", + "\x12\x30\x97\x7b\x9a\x5b\x12\xc8\xee\x10\xa3\xb4\xab\xb4\xf0\x6e", + "\x19\x7a\x27\xed\xfc\x49\x95\x3b\x6d\xad\xfb\xe7\x17\x0f\xc7\x50", + "\x2b\xc4\x76\x3b\xa5\xb4\x24\xa1\xf2\x6b\xb6\x25\xd9\xf6\xd5\x15", + "\xd5\xbd\x4f\xa2\x3a\x45\x39\x5c\x71\x6f", 1, 128, 128 }, + { 96, 128, 80, 332, + "\x95\xe5\x17\x9e\xb1\x97\xf1\xa5\x15\xe1\x99\xbd\x93\x79\x48\xcd", + "\x49\xd4\x07\x7a\xd5\xd8\xbb\x84\xee\xcc\xf7\x11", "", + "\x45\xd9\x09\x5c\xf3\x20\xc5\x82\xc8\x97\xf0\xab\xb5\x3e\x3a\xed" + "\xec", + "\x36\xb3\xc9\xe7\xc2\x54\x39\xf2\x05\xff\x0e\x38\xff\x46\x79\x61" + "\xb8", + "\xc6\xb8\x39\xab\x6e\xe9\x97\x8e\xae\xdb", 1, 0, 136 }, + { 96, 128, 80, 333, + "\xfa\xc8\xd9\x8a\x8a\xf9\x32\x39\xb0\xd9\x55\x16\x57\xc5\x95\x1d", + "\xff\x1f\x41\x9b\xed\x64\xbf\x4a\x02\xc3\x57\xe5", + "\x2e\x3f\x10\x2d\xe4\x45\xb4\xec\x11\x7b\x63\xfb\xa7\x08\x9d\xe8", + "\x11\x97\xd7\x6a\x46\x9c\x17\x43\x82\x01\xef\x40\x00\xfa\x05\xf0" + "\xa9", + "\x96\xe2\x39\x10\xda\xa8\x64\xeb\x12\x68\xdb\xf2\x33\x9e\xd4\xbb" + "\x62", + "\xeb\xd1\x52\xd5\xf2\xd0\x0c\x60\xec\xd4", 1, 128, 136 }, + { 96, 128, 80, 334, + "\xde\x54\x50\x44\xb8\x14\xf3\x13\xc2\x3b\x5c\xb8\x54\xf7\x39\xa2", + "\x99\xfc\xac\xe8\xe5\x9b\xdd\x6b\x88\xdd\x96\x0c", "", + "\x83\x52\x5d\xbb\xb5\x4d\xe0\xfc\x1d\x24\x87\x49\xa7\x16\xd9\xde\xbc" + "\x65\xfe\x44\xc7\x9b\x16\x3b\x36\x14\xfb\x8d\x62\xee\x2e", + "\xa0\xe6\xe3\xd5\x31\xb8\x63\xb9\xe6\xf3\x8c\xf0\x3d\x60\xf1\xd6\x93" + "\x0c\xb1\x7a\xa4\x1a\x78\xa6\x6d\x5b\x94\x9c\x5f\x7e\xc7", + "\x01\x38\xc4\x33\x9b\xfe\xd8\x18\x96\x4b", 1, 0, 248 }, + { 96, 128, 80, 335, + "\xbb\xbf\xa9\x44\x44\x93\xdd\x2f\xbf\x72\xba\xf3\x87\xa4\x09\x00", + "\x29\x35\x56\x7a\xa5\x72\x90\x8e\x49\x91\x71\x30", + "\xc0\xdb\x66\x6f\x38\x14\xfd\xc2\xcf\x7c\xb3\xd4\xce\xfa\xf2\xd3", + "\x59\xfc\x37\x65\x4b\x0a\x5e\x3b\x86\x87\xa3\xd8\x5b\x32\x64\x4d\xc7" + "\xa1\x56\xb6\x0d\xd7\xa6\x4d\x22\x98\x37\x3e\x15\x8f\x21", + "\x46\x4f\x1d\x04\x17\x28\x0f\x22\xf0\x60\x53\xcb\xea\x16\xe2\x8e\xb0" + "\xf7\x90\x82\xa6\x82\xb5\x8c\xb7\x19\x42\x36\x93\xe6\x6c", + "\x19\x87\x01\x52\x03\x23\xf4\x61\x3b\x59", 1, 128, 248 }, + { 96, 128, 96, 336, + "\x0a\xd9\xda\x99\x4d\xb2\xed\x7b\x9e\x35\xe9\x88\x95\x19\x4c\x4b", + "\x57\xe3\x64\xc1\x6b\x36\x89\xbc\x15\x6b\x31\x15", "", "", "", + "\x73\x58\x6e\xab\x8c\xed\x75\x40\x62\x0f\xea\x72", 1, 0, 0 }, + { 96, 128, 96, 337, + "\x97\x04\x95\xf7\x0d\xc6\x4f\x0f\xe4\xe8\xc1\x09\x46\xdf\x2e\xd1", + "\x69\xb1\xa3\x19\x5c\x16\x55\x17\xfe\xd6\x65\x95", + "\xbf\xe8\x73\x6a\x21\x13\xf7\x74\xc6\x82\x8e\x5b\x93\x0f\x1c\xb9", + "", "", "\x18\xd7\xc5\x4f\x8f\xcb\xec\x44\x2b\x31\x39\x87", 1, 128, + 0 }, + { 96, 128, 96, 338, + "\xf3\x63\xf1\xa7\xd3\x3c\x96\x94\x9f\xd0\x8f\x44\x0c\xfb\xa0\x00", + "\x67\xb9\x20\x07\xf5\x7b\x83\xfd\x9f\x3e\xe6\xfa", "", + "\xa6\x51\xd2\xca\x4b\x16\x98\x0b\x0e\x4a\x7a\x10\xc7\x5c\x47\xed", + "\x20\xc2\xa2\xf1\x8d\x07\x53\xac\xd3\x6e\x20\x49\x85\x14\x95\x28", + "\x4a\x44\x22\xd3\xb9\x9c\x8d\x77\xdb\xde\x2a\xb2", 1, 0, 128 }, + { 96, 128, 96, 339, + "\x6b\x74\x89\xd1\x58\xf3\x77\xe6\x69\x2d\x84\xa9\x77\x27\xff\x41", + "\x9a\xc0\x91\xef\x05\xd0\xce\x74\x28\x82\x7e\xd3", + "\x38\x46\x9f\x10\x49\xa7\xea\x3d\xa0\x55\x1c\xfb\x34\x01\x0b\xf6", + "\x35\x2d\xbd\x5b\xb8\xab\xf0\xa0\x97\xb9\x29\x16\x0b\x8f\x8c\xec", + "\x00\xd3\x94\x8b\xac\x85\x72\xed\x1e\xd5\x9c\x26\x55\xb7\x69\xf7", + "\x32\x3b\xa1\xc8\x06\xf3\xea\xc6\x73\x01\x5a\x88", 1, 128, 128 }, + { 96, 128, 96, 340, + "\x38\xd4\xd2\x7c\x30\x83\x49\x68\xb5\x28\x5b\x99\xac\x18\x73\x4a", + "\x18\x3a\xe3\x52\xd9\xc3\x40\xee\x61\x67\xc3\xb6", "", + "\xf1\x53\xd0\x1c\x5b\x9a\xb2\x02\x45\x56\x87\x53\x7e\x83\x52\xd2" + "\x94", + "\x3c\xa2\x62\xd9\x2d\xb8\x40\x4d\x5d\xb0\xe5\x5c\xcc\xdd\xff\x06" + "\x5b", + "\xd2\x7a\xd6\x86\x6e\xa9\x2b\xa2\x68\x0d\xce\xef", 1, 0, 136 }, + { 96, 128, 96, 341, + "\x74\xb9\x75\x6c\xb2\xac\x63\x61\xce\x9d\x68\x44\x77\xb8\xd0\xc9", + "\xc2\x60\x6f\xc9\x64\xb6\x13\xa0\xb1\x53\xfc\x0f", + "\xdc\xe9\x30\x9c\xf7\x1c\xed\x35\xeb\x22\x0c\x70\x9f\xdd\xc4\x14", + "\x66\xf3\x21\x69\x11\x74\x80\x38\xf9\x14\x32\x34\x49\x14\xee\x8a" + "\x35", + "\xc8\xae\xe8\x33\x0a\x37\xca\x70\x6f\x47\x6f\x77\x4f\xf3\x57\x00" + "\xca", + "\xb3\x7e\x7e\x62\xd0\xb1\xae\xab\x26\x78\xbc\xf3", 1, 128, 136 }, + { 96, 128, 96, 342, + "\xe1\xe4\x5f\x35\x00\xa4\x05\xdf\x5a\xbd\xcb\x3b\x86\xbe\xa1\x4d", + "\x59\x28\x9d\xa2\xd5\xf1\x3e\xea\x49\x95\x61\x1f", "", + "\x5c\xda\x60\x60\xa7\xe1\x05\xcc\x57\xc7\x75\xa0\x2a\xf9\x21\x75\x73" + "\x50\xc9\x69\x2b\xc4\xfa\x40\x4a\xce\x98\xeb\x1e\x61\x71", + "\xf2\x67\x01\x4a\xc2\x64\x66\x05\x8b\x80\xd2\x8c\x0d\x82\x52\x1d\x69" + "\xb2\x30\x2c\x36\x56\x74\x0c\x23\x78\x31\x85\x9a\x0f\x24", + "\x9e\xdc\x28\x31\x74\x36\xd6\x6d\x75\x2a\xd9\xb7", 1, 0, 248 }, + { 96, 128, 96, 343, + "\xbc\xe5\x99\xcb\x75\xa2\x27\x10\x70\xe6\x19\x9c\xb0\x96\x65\x6b", + "\xf9\xa8\x33\x3f\x46\x73\x68\x9e\x39\x59\xc9\xe0", + "\xf6\x1e\x10\x35\x17\x1c\x92\xb0\x22\xae\x55\x9e\x86\x57\x93\x0e", + "\x41\x89\x4a\xcc\x83\x8d\x4a\x8f\x62\xe6\xcc\x92\x71\xf1\xd6\x5d\xf7" + "\xf3\x65\xa3\x8e\x9a\x94\x11\x0f\x4c\x8d\x57\xb8\xbe\x18", + "\x0e\xb5\xc0\x3d\x69\x15\x3d\xbe\x79\x4c\x53\xcb\x29\x3b\x25\xd3\x8c" + "\xaf\xa1\x36\x72\xc9\x15\x60\x68\xa4\x02\x6d\xb0\xd7\x08", + "\xd4\x06\x38\x9c\xe2\x22\x8e\xbb\xbc\x40\x0b\xbf", 1, 128, 248 }, + { 96, 128, 112, 344, + "\xf2\x1f\x07\x00\xf1\x6a\xa0\x98\xd6\x61\x7c\xc3\x68\x30\x12\xc1", + "\x42\x39\x4a\x30\xfc\xd2\x52\x55\x6b\xf2\xcb\x36", "", "", "", + "\x39\xd0\x70\x3d\xcf\x7d\x0c\x31\x62\x22\xd7\x16\xaf\xec", 1, 0, 0 }, + { 96, 128, 112, 345, + "\x7d\x28\x27\x73\x9a\xd3\xce\x2d\xc7\xf2\x7e\x35\xf6\xcd\x83\x7f", + "\x09\xa2\x49\x07\x7d\xb1\xf8\x4e\x98\x4a\x98\x29", + "\xb5\xe5\x9d\x8c\x3f\x81\xda\xe7\x78\x9a\x82\x6a\x0d\x32\x00\xf9", + "", "", "\x45\x55\x3d\x58\x83\x9d\x45\xa3\x77\xbe\x85\xe9\x5a\x41", 1, + 128, 0 }, + { 96, 128, 112, 346, + "\x1e\x70\xde\x0c\xba\x8f\x88\x48\xdb\xc8\xdd\x9c\xfa\x53\xc1\x61", + "\xd4\xe6\x77\xbd\xb0\x4b\xf9\x35\xd1\x30\xce\x15", "", + "\x71\x02\xb7\x71\x0b\x1d\xb1\xa0\x74\x84\x74\xf8\xe3\x7b\x6d\xd8", + "\x55\xdf\xe0\xe8\x8c\x81\xbf\xc5\x61\x97\x5d\xfa\xba\xa2\x1a\x12", + "\x02\x4e\x3b\xf1\x98\x5a\x7f\x7e\xcc\xda\xa0\xee\x2a\x18", 1, 0, + 128 }, + { 96, 128, 112, 347, + "\x0b\xb3\x80\xf7\x25\x73\xe6\xd4\x97\x17\x23\x81\xf5\xf4\xeb\x6a", + "\xe7\x9f\x20\x84\x0e\x41\x82\xac\x6b\xf0\xf8\x48", + "\x44\x9d\xce\xa2\x7c\xd6\x10\x31\xf9\xbf\xaf\x87\xd3\xbc\xf9\xc4", + "\x4a\xbf\xe6\x0e\xd6\xbf\x24\x19\x0e\x41\x6e\x68\x09\x71\x8f\xa0", + "\x12\xe6\x3d\x6d\x51\xc1\x4a\x2e\xe7\x81\x0a\x24\x0f\xfe\xbc\x13", + "\x5c\xfc\x7d\xf5\xf7\x0a\x81\xf9\x3a\x8f\x47\x14\xe1\x43", 1, 128, + 128 }, + { 96, 128, 112, 348, + "\xc4\x8d\xe4\xc9\x1e\xfd\xf7\xb2\x4c\x8f\x80\x07\x3f\x6f\x17\xd2", + "\xdd\xa7\x45\x36\x01\xd5\x16\xe0\x87\x32\x0e\xbf", "", + "\x01\xbe\x64\x0c\xaf\xc9\xeb\x72\x88\x27\xfc\xf1\xc9\xcb\xa5\xe0" + "\xcc", + "\x23\x5b\x7e\xdd\x4b\x3d\xf0\x3c\x4a\xd8\xff\x11\x12\xcc\xb3\xd9" + "\x28", + "\x7b\x95\xbc\x04\x20\xce\x86\xac\x2f\x2a\x37\x5d\x5f\xe0", 1, 0, + 136 }, + { 96, 128, 112, 349, + "\x21\xf8\x4c\x57\xb6\xd4\xa2\xd2\xd3\x0d\x4c\x37\xec\xf1\x10\x30", + "\xd3\xce\x63\xa1\xaf\x0b\xef\x6c\x9e\x0b\xbd\x81", + "\x0f\x78\xdd\x2b\x4e\x56\x6c\x9d\x15\xd0\x52\xb0\x1f\x6f\x85\xba", + "\x33\x58\x54\x3c\x39\xc1\x00\x25\xe0\x1c\xe8\x90\x06\xba\x00\x43" + "\x26", + "\xcb\xb9\x21\x40\x6c\xe9\xe4\x4c\x19\xb0\x19\xca\x26\x9a\x6b\x7b" + "\x13", + "\xf6\xba\x3a\x6a\x43\x3a\x50\xc4\xab\xe0\x0d\xa2\x41\x1f", 1, 128, + 136 }, + { 96, 128, 112, 350, + "\xa0\x8b\xe6\x8d\x04\x4e\x76\xe4\x7d\x04\xd0\x93\xee\x54\x8e\x59", + "\xf1\x7b\x6f\x94\xa8\x86\x42\x05\xc7\x57\xa6\x35", "", + "\x05\xe4\x54\x20\x55\xfa\x84\xe2\xd3\x49\x83\x7d\xef\x40\x23\x53\xfe" + "\xa7\xec\x56\xd3\xfd\x81\xa4\x48\x31\x40\x3f\x1f\x7f\x72", + "\x60\x19\x8b\xdf\x8b\x37\x8d\xbe\x55\x32\xdb\x73\x29\xaa\xcc\x58\xba" + "\x32\x5b\x82\x79\x65\xd4\x69\xb4\xec\x7d\x06\x98\xb9\xdf", + "\x05\x99\xec\x3c\x8c\x56\xae\x5d\x1e\xe6\xea\xc6\xca\x05", 1, 0, + 248 }, + { 96, 128, 112, 351, + "\xd9\x5f\x7f\x6a\x07\x53\x0e\xe3\xf9\x67\xfd\x1f\xf9\x90\x8a\xfe", + "\xba\xbe\x02\x81\xf3\xcb\xa8\x06\x67\xc3\x65\xaf", + "\xf4\x9c\x18\xd6\x5a\x19\x7b\x97\x3d\x26\xfd\x29\xa1\x43\x74\x60", + "\xe0\x2f\xec\x4c\x11\x8b\xcc\x96\x67\x01\x58\x72\xd8\x96\xc8\x86\x8c" + "\x15\x90\xf8\x47\x34\xce\xc6\x5c\xe9\x0b\x3d\xc0\x76\xd3", + "\xbc\xd3\x79\xde\x4b\xd6\xea\x47\xf2\x93\xa2\x5c\x41\x1d\x14\xd0\x1f" + "\x5d\xe7\x3a\x99\xc5\xe2\xe2\xe5\x93\x0f\xd3\xbb\x3b\x23", + "\x43\xb9\x4d\xf5\x7d\xf5\xf7\x6a\xc5\xa0\x1f\x52\x51\x38", 1, 128, + 248 }, + { 96, 192, 32, 352, + "\x85\xe0\x17\xfb\xc8\x60\x56\xc8\xc1\x89\x15\xb3\x69\xc0\xc9\x2d\xd3" + "\xaf\x3f\xc6\x77\x78\x2f\x8c", + "\x3d\x57\x51\x1e\xaf\xfb\xe4\xe9\xe9\x0d\x6e\xc3", "", "", "", + "\x4f\xc4\x19\x2c", 1, 0, 0 }, + { 96, 192, 32, 353, + "\x0d\xf9\x7b\xa4\x0e\x5f\x24\xcd\x5f\x1b\xd0\xec\xf4\x74\xac\x9a\x4a" + "\x8b\x4c\xf1\x38\x80\x65\x49", + "\x70\x5b\xc1\xa6\xcb\x54\xc1\x43\xd4\xfa\x10\x02", + "\xdf\xf5\xca\xd8\xf5\xb6\xcc\x65\xdf\x4e\x4e\x12\x80\x2b\xd0\xe6", + "", "", "\x48\xde\xd0\x1f", 1, 128, 0 }, + { 96, 192, 32, 354, + "\x79\x50\x63\x24\x8c\x61\x9c\x9c\xe6\x1b\x56\xc1\x7d\xb6\xc0\x23\xfa" + "\xb1\x2c\xa6\x10\x31\x92\x5c", + "\xd3\xaa\x2d\xfb\x01\x9b\x56\x46\x7f\xdb\x36\x8b", "", + "\x01\x7e\x69\x0c\x00\x69\xbf\x92\xd6\x9f\x27\x0d\x32\xaf\x15\xef", + "\x3b\xf2\x4b\x2a\xda\x60\x4a\xd0\xad\x9f\xa5\x38\xe3\xb4\xe3\x8a", + "\x84\xd2\xcf\x30", 1, 0, 128 }, + { 96, 192, 32, 355, + "\xa7\xac\xdc\x89\xa8\x6a\xda\x31\x90\xda\x95\x4e\x02\x9b\xd3\xd5\x8e" + "\x05\xbb\xee\x02\x72\xcc\x94", + "\x4a\xca\x59\x38\xa8\x8a\x69\x8e\xc7\x45\xb4\x43", + "\x9c\xfc\xfd\x28\x4e\xd3\x5f\xa1\x11\x04\xe5\x28\x56\xfa\x3d\x08", + "\x64\x18\x73\x87\xc7\xcf\x3b\x56\x20\x63\xab\x35\x45\xca\x71\xaa", + "\x97\x02\xe5\x11\x9a\x1b\x3a\x09\xe7\xc8\x0e\x65\xe8\x2b\xb8\xc8", + "\x46\x12\xd0\xbf", 1, 128, 128 }, + { 96, 192, 32, 356, + "\x63\x69\xc4\x28\x16\x2c\xd7\xc8\x61\xaa\xf2\x8c\x4c\x36\xb8\xe5\x38" + "\x89\x5e\x46\x9e\x0d\x1f\x48", + "\xc5\xf0\x9c\xb9\xd0\x30\x8a\x13\xfc\x73\x19\x12", "", + "\xb1\x98\xa9\xce\x48\x23\xd7\x47\x79\x36\xf5\xcf\x9c\x73\x9a\x83" + "\x0c", + "\x1a\x39\xea\x1b\xec\xc6\x94\x11\x65\x66\x98\x7a\x67\xd8\xff\x3e" + "\xd7", + "\x3c\x31\x93\xc5", 1, 0, 136 }, + { 96, 192, 32, 357, + "\x12\x40\xf2\x45\x5c\x1e\xf9\xa7\xe7\x6f\xe9\x3f\x61\x79\xd9\xbd\x6a" + "\x24\x9b\x66\xba\x26\xbb\x0a", + "\xc4\xf3\xa0\x72\xf2\x0f\xc2\x2a\x9f\xeb\x74\xc8", + "\x41\xda\x67\xad\x57\x37\xcd\x4d\x60\x1b\x37\x8d\x31\x2f\x87\x40", + "\xb4\xf3\xa0\x29\x07\x6a\x1b\xce\x99\xe8\x36\x5b\x1b\x12\x70\x5f" + "\x17", + "\xaa\xd2\xfa\xc7\x98\xf0\xf4\x3c\x70\x30\xaa\xa3\xae\xe1\xfa\x50" + "\x8f", + "\x09\x73\x36\x97", 1, 128, 136 }, + { 96, 192, 32, 358, + "\xc3\x35\xb7\x6e\xa5\x97\xd4\x44\xa1\xe6\x63\x50\xbd\xce\xfe\x5c\x96" + "\x84\xaf\x17\xee\x9e\x39\xa4", + "\xab\xd4\xd1\x25\x1f\x48\x1d\xfe\xe3\x5c\xfc\x71", "", + "\xa8\xca\xad\x10\xe1\x60\x20\x41\xa0\xa2\x92\x76\x3c\xe5\xf9\x03\x23" + "\xec\xfd\x3c\x93\x17\x05\x33\x3f\x3b\x00\xe6\xfb\xe2\x62", + "\x6e\xaf\xde\xed\xbe\x5b\x36\x7b\xaf\x14\x06\x43\x68\xe6\xf3\x2e\x2e" + "\xd0\x7b\x0e\xa6\x22\x18\xcf\x50\x56\x9f\x79\x6b\xb6\xb4", + "\x45\x16\x64\x97", 1, 0, 248 }, + { 96, 192, 32, 359, + "\x6a\xcc\xd4\xbd\x6b\x18\xb9\xf6\x59\x36\xd8\x7f\x5b\x3f\x83\x39\xd8" + "\xae\x08\xa2\xa8\x6b\x67\x05", + "\xa3\x9d\xd2\xe2\xc2\xe2\x15\xce\xbd\xa0\x0e\x13", + "\xdf\x01\x44\xcb\x65\xec\x35\x29\x9d\x30\x45\x8b\xd6\x1a\x60\xcd", + "\x97\xe6\xba\x8a\x7e\x71\x7f\x8c\x16\x0b\x9b\x4b\xf5\x2e\x5b\xa0\x39" + "\x89\xd1\xfb\x17\xe0\x80\x78\xd7\x7f\x7c\x26\xa6\x53\x00", + "\x3a\x66\x3f\x78\x97\xc8\xf7\x74\xac\x8d\x74\xbb\xf0\x53\x04\xe4\xd7" + "\xfe\xf9\x2b\xb5\x96\x1d\x0c\x88\xe4\x13\xae\xb4\x7f\x36", + "\x04\xd5\x07\x14", 1, 128, 248 }, + { 96, 192, 48, 360, + "\x21\xac\xb0\x9b\xb4\x8d\xc6\x41\x7e\x4d\x87\xa3\x16\x8f\xcb\x84\xe3" + "\x19\x50\x51\x93\x31\xdb\x93", + "\x5d\x3e\x03\x63\x37\x46\xd3\x72\x9b\x60\x9d\xd1", "", "", "", + "\x40\xd0\x42\x96\xcf\x7e", 1, 0, 0 }, + { 96, 192, 48, 361, + "\xeb\x47\x0c\x2b\xe6\xb3\x25\x75\xc4\x2c\xf9\x08\x53\xbd\xbe\xd1\xe6" + "\x41\x2c\xae\x16\x15\x33\x0f", + "\x9e\xd7\x46\x01\x5a\xd7\x2f\x1f\x4a\x86\x88\x37", + "\xe8\x15\x4f\xb5\x03\xcb\xa6\x64\x91\xa7\xa9\xad\x2f\x31\x02\x82", + "", "", "\x8d\x8f\x8d\x39\x46\xb8", 1, 128, 0 }, + { 96, 192, 48, 362, + "\xb7\x85\x8c\x86\xb3\x55\x19\xe9\xc4\x28\x62\x4c\x41\xf4\xda\x83\x79" + "\xd0\xf8\x55\xb3\xdc\xd6\x22", + "\xd6\xcb\xa3\x5f\x02\x78\xd6\x67\xd9\x3d\x43\xe3", "", + "\x77\xae\x65\x5c\x3d\x5f\x9a\x6e\xc0\x6c\xce\xc7\x14\x82\x7d\x87", + "\xd6\x74\xd8\x3e\x11\x21\xbe\x22\x6b\xd7\x33\x55\xdd\x33\x65\x7e", + "\x57\x82\x10\x5b\x45\xbd", 1, 0, 128 }, + { 96, 192, 48, 363, + "\xa9\x98\x8d\xf0\xc0\x01\x13\x2d\xee\x87\x30\x6d\xaa\x4a\x48\x06\x2c" + "\xa0\xa7\x3a\x61\xd3\x8b\x2e", + "\xd0\x2d\x4c\x03\x6d\x75\xb4\xc2\xbd\x03\x86\x05", + "\x6d\x9d\xf5\x3d\xc7\x1e\x44\x76\x61\xb5\xd6\x4b\x31\xc2\xa6\x6d", + "\x66\xda\x05\xe7\xd6\xdd\x8f\xb9\x99\x82\x7f\xd5\xcc\xe8\xa1\xe8", + "\x65\x70\x4e\x76\x07\x60\xfd\xcb\xed\x42\x8a\x29\xef\x60\x48\x84", + "\xe8\x9d\x18\xb4\x39\xc6", 1, 128, 128 }, + { 96, 192, 48, 364, + "\x50\xa0\x22\xe8\xd8\x56\x41\x33\x7e\x86\xd1\x4d\x75\xf5\x37\x74\x78" + "\xaf\x29\x7d\x20\x91\xf5\xcc", + "\xad\x59\x6a\xfb\x65\x49\x09\x81\x62\xca\x53\xed", "", + "\x0d\x7d\xe7\x6a\xf7\x7e\x8d\x11\x8e\x97\x19\xd5\x42\x9b\x3b\xe4" + "\x5b", + "\xf2\x0d\x2d\xa7\x45\xeb\x30\xc5\x16\x63\xa8\x4e\x9e\x1e\x00\x27" + "\x84", + "\x90\xe8\x41\x8f\x11\x3d", 1, 0, 136 }, + { 96, 192, 48, 365, + "\x97\xc5\xaf\x8e\x0f\x0c\xa6\x9d\x77\x13\x7f\xec\x21\xdb\x36\xc9\xde" + "\xa6\xc8\x36\xa9\x2a\x29\xd2", + "\x7b\x4e\x8a\x9a\x66\xf6\x82\xbd\x9a\x2f\xd5\xe1", + "\xbd\xa1\xce\xb6\x3c\x2c\x5f\x54\xee\x92\x6a\x83\x20\x94\xe8\x87", + "\xcf\x14\xe1\xfd\x8c\x85\x7c\x3b\xa2\x74\xaf\xd4\x23\xec\xc1\xd8" + "\xd9", + "\x67\x21\x31\x0f\xca\x31\x2d\x96\x14\xe4\xe2\x9d\xff\xc7\x3f\xdf" + "\xdb", + "\x80\xab\x89\x81\x90\xdd", 1, 128, 136 }, + { 96, 192, 48, 366, + "\x44\x33\xa4\x40\x38\x99\x57\x49\xfc\x7b\x84\xf3\x76\x4f\x9f\xc9\xf8" + "\xf9\x1d\x20\xed\x19\x00\xa6", + "\x6a\xac\x37\x28\x40\x93\xec\x85\x9b\x3c\x0a\xf4", "", + "\x33\xd5\x96\x7e\x20\xad\x5c\x2d\x98\x8c\x6c\xd2\x6a\x92\x15\xb5\x2e" + "\x0f\xb4\xdf\xbb\x37\x53\x0a\xd4\x4f\x4b\x0e\xc4\x1e\x12", + "\xcb\xe3\x7d\x72\xb0\xc3\xea\xf2\x81\xab\x34\xac\x47\xb6\x39\xf4\x40" + "\xd2\x18\xe0\x9b\x14\x80\x8b\xc3\xa8\xe2\xf6\x48\x4b\xa5", + "\x69\xe1\x9b\xaa\x18\xe3", 1, 0, 248 }, + { 96, 192, 48, 367, + "\x2b\x05\x07\xb1\x6f\xa4\x9e\x08\x8b\x18\x4e\xb0\xd3\xe1\xd3\xf0\x53" + "\xea\x3f\x8e\xaa\xf3\xe5\x3d", + "\xb0\xac\xcf\x6f\x28\x26\x2e\x0e\x1f\xce\x23\xfd", + "\xa4\xbd\x9d\xa3\xad\x1f\x44\xf5\xdc\x19\x71\x8f\x67\x8d\xe5\xbf", + "\x3e\x83\x20\xfe\x8a\xbf\xbc\xbf\x29\xd7\x24\xdc\x33\x07\x15\x6e\x6b" + "\xe5\x3b\x21\xe9\xbc\xc3\xcd\xa9\x1b\x38\x0a\xd5\x80\xda", + "\x44\x51\x76\x14\x12\x89\x25\xef\xdb\x7f\xb1\x3a\x6c\x4a\x75\x9b\x73" + "\x7c\x82\xd3\x09\x86\xd2\xaf\xde\x97\x3c\x88\xd6\x01\x3c", + "\xea\x19\xec\xad\x57\x16", 1, 128, 248 }, + { 96, 192, 64, 368, + "\x5c\x04\x9b\x3e\xdd\x2b\x92\x6f\xcd\x34\x34\xc4\x21\x53\x2b\x7e\x79" + "\x08\x71\x2a\x85\x05\x72\x26", + "\xb2\xb7\x93\x46\x9d\x4a\xc1\xdf\xc3\x75\x6c\x8f", "", "", "", + "\xc2\x78\x16\xbb\x97\xe9\x8d\x11", 1, 0, 0 }, + { 96, 192, 64, 369, + "\x5b\x04\xc3\x42\xef\xd5\xe8\x9a\xa5\xd3\x8e\xf3\x2e\xed\xea\xf2\xac" + "\x03\x5f\x43\xb9\xb4\x20\x1d", + "\x14\xd4\x78\x1e\x21\x59\x2e\xfc\x44\x09\xb9\x44", + "\x3f\xd3\xb6\x91\xd0\x51\x1d\x71\xf5\xdb\xec\x4f\x13\x20\xfc\x8c", + "", "", "\x2f\x84\xac\x2d\x50\xbe\xf7\x5e", 1, 128, 0 }, + { 96, 192, 64, 370, + "\x1a\x2f\x92\x13\xe6\x6c\x96\x93\x06\xb9\x8c\xe3\x3d\x32\xca\x91\x26" + "\xe7\x65\x78\x35\x5a\x67\xab", + "\x6e\xda\xe6\x28\x13\x3c\x51\x0f\x00\x96\x58\x5a", "", + "\xd4\xa9\x42\x70\x12\x40\x3f\x9c\x51\x8c\x7b\x23\x60\xce\x0a\xb3", + "\x92\xbd\x38\xe1\xfd\xfa\x11\x75\xdc\x23\x0b\xe5\xf5\x41\x76\x0b", + "\x99\x02\xe9\xda\x26\x90\x82\x95", 1, 0, 128 }, + { 96, 192, 64, 371, + "\xa5\x93\x74\x68\xdd\xde\x3c\x31\x2b\x6f\xce\xc7\xd5\xd1\x9a\x92\x85" + "\x3d\x2c\x66\xad\xa9\x7a\x18", + "\x5b\xf9\x54\x7b\x27\x53\xdd\x71\x2a\x5d\x8f\x95", + "\xe5\x6b\xc3\x35\x6c\xbd\xdb\x3e\xf0\x99\xca\xd5\x89\xbb\xe6\x84", + "\x58\x13\xc3\xc7\x56\xa8\xf2\x72\x1a\x08\xbe\x97\xc4\x43\x92\x69", + "\x1b\x77\xaa\x30\x34\x01\x89\xfe\x4f\xbb\xd7\xea\x1c\x96\xd5\xc4", + "\xee\x2f\x19\x5b\x66\x7a\xa2\x67", 1, 128, 128 }, + { 96, 192, 64, 372, + "\x4d\x85\x76\xff\x63\x5e\xc7\xd9\x9c\x47\xbe\x74\x12\xa2\x84\x6f\xc6" + "\x38\xc9\xf9\xfb\x0f\x55\x31", + "\x8a\x53\x40\xf4\xa8\x5e\x3a\x9c\xf7\x43\x0f\xeb", "", + "\x0b\x89\x63\x37\xa5\x9a\xf8\xe9\xca\x15\xf3\x3c\xd6\xda\xaa\xe0" + "\xac", + "\x5f\xdf\x4a\x0f\xce\x8b\xe9\xcf\x74\x0b\x61\xd1\x20\x88\x3b\xcc" + "\x1e", + "\x56\x63\x21\xb1\x2e\xce\xc6\x87", 1, 0, 136 }, + { 96, 192, 64, 373, + "\x9c\xa4\x67\xaf\x0c\x27\x29\xf4\x3f\xbd\x86\x63\x73\xef\x4b\x8f\x2b" + "\xca\xbe\x43\xf5\xa1\x0f\x97", + "\xe3\xec\x43\x9d\x33\x4b\x9f\xc0\x7d\x65\xdf\xf5", + "\x60\xd9\xbe\x32\xc5\x62\x66\x6a\x19\x01\x42\x84\x74\x04\xe8\x04", + "\xf0\x61\x11\x0e\x43\x63\x6e\xb5\x25\xcd\x2f\x94\xf6\x31\xf1\x28" + "\x2d", + "\x5f\x8d\xd3\x0c\xb7\x0f\x49\x5e\xb5\x77\x77\x30\xb0\xc7\xda\xde" + "\x30", + "\xb0\x1f\xee\xdd\x3e\xd3\x64\x0b", 1, 128, 136 }, + { 96, 192, 64, 374, + "\xe9\x23\xbb\xfb\xbd\xb8\x1c\xec\x86\x32\x63\x49\x40\xc9\x24\xbc\x9a" + "\x23\x0f\x15\x87\xf0\xed\x63", + "\x41\x90\x00\x4b\xf9\x66\xaf\x35\xe0\x49\x44\x5d", "", + "\xa3\x8f\x8e\x64\xa3\x91\xa0\x9b\x8a\x29\x8d\x4f\xeb\x01\x13\xe3\x08" + "\xcb\xfc\x6e\xdb\xc3\xcd\x59\xa2\x5a\x31\xa3\xf0\xd5\x34", + "\x01\xc7\x76\x5b\x13\x96\xfc\x6d\x36\x2c\x00\x77\xa3\xa1\xef\x9c\x3f" + "\xe5\x4b\x87\x68\x8b\x7a\x64\x12\x0d\x8a\x20\x2d\xe3\x9c", + "\x89\x90\xa6\xb1\xf3\x86\xcc\x7c", 1, 0, 248 }, + { 96, 192, 64, 375, + "\x7b\xcc\xcd\x49\x44\x60\xa7\x55\xcf\x82\xee\xdc\xca\xe6\xb1\x41\xb3" + "\xc5\xb8\x36\x0f\x09\xdc\xfd", + "\x3a\xb4\x79\x8a\xd9\xc0\x5c\xc7\x93\xf5\xc3\x3d", + "\x90\x55\x30\x0f\x9a\xf4\x4b\x8c\x4a\x7f\xdd\xbd\xd8\xe2\x49\x72", + "\xe1\xeb\x07\xd7\x97\xf5\xfe\x2a\x31\xc2\x8c\x03\x82\xb5\x21\x61\x2c" + "\xbb\x0a\x6f\xdc\x6e\x53\xe2\x7b\x29\x14\x08\x78\x82\xd0", + "\x6a\x7c\xb6\x7d\x39\x58\x97\xd5\xce\x8c\x59\x73\x09\xd5\x10\x20\x14" + "\x9b\x1f\xeb\x13\x13\x61\xdc\x1a\x23\x6e\xe9\x2b\x40\xa8", + "\x1f\xfc\x13\xf7\x4d\x3c\x07\x76", 1, 128, 248 }, + { 96, 192, 80, 376, + "\x89\xc4\xe7\x97\x06\x2e\x49\xad\x02\xd2\xbc\xf2\xeb\x0f\xf6\x5f\xc1" + "\x7c\xd2\x9c\xd5\x5c\x8b\xbf", + "\x68\xde\x74\x04\xe6\xd1\x37\xa5\x83\x89\x0b\x0f", "", "", "", + "\xba\x50\xe0\x4e\xf8\x87\x14\x55\x62\xf1", 1, 0, 0 }, + { 96, 192, 80, 377, + "\xc9\xb9\x28\x03\x80\x27\x6a\x99\xc7\xe8\xb0\x0b\x03\xa0\xac\x35\x93" + "\x66\xa9\x25\x53\x2c\x4b\x08", + "\x3d\x69\x75\x92\xa7\x80\x07\xee\x3f\xc9\xf8\x71", + "\x56\x6f\xce\xd9\xa2\x4e\x20\xae\x05\x5f\x03\x4d\xe8\x9f\x76\x2a", + "", "", "\x7e\x43\xc9\x45\xa8\x82\x6a\x9f\x71\x64", 1, 128, 0 }, + { 96, 192, 80, 378, + "\x19\x90\x4b\xd0\xb5\x44\xa2\x9e\x2c\x0a\x30\x5a\x12\x43\x23\xa1\xde" + "\x6f\xae\xb7\x1b\xdd\x0f\x87", + "\x1d\x15\xb8\x7d\xfe\x88\xc8\x31\xb1\x05\x45\xaa", "", + "\xa6\x89\x6e\x25\x78\x68\x9e\x31\xd3\x05\xf3\xce\x21\x41\x5f\xfd", + "\xbc\xbd\x19\x43\x82\xf5\x21\x49\x8c\x93\x0f\x05\x2f\x81\xf5\xc5", + "\xca\x50\xc7\xa2\xd0\xe3\x9a\x64\x2e\x92", 1, 0, 128 }, + { 96, 192, 80, 379, + "\xfd\x10\x95\x28\x5c\x2d\x1d\x6a\x65\x45\x00\x45\x3e\x12\x41\xf1\x3a" + "\xda\x13\x64\x23\x4d\x16\x6c", + "\x95\x48\x7c\xb5\x3c\x6f\xae\x13\x29\x00\x52\xdc", + "\x7d\x97\xf4\xb8\x61\xa8\x33\x6c\xe9\xb4\xc7\x25\x0c\xbd\x82\x5b", + "\xfe\x05\xd5\xfd\x3a\x3d\x4a\x70\x7b\x4a\x63\x09\x7c\x48\x3c\x9c", + "\x49\x85\x19\x00\x2f\xf3\x26\x6d\x85\x84\xe5\x64\x17\xa8\x55\x11", + "\x82\x4e\xfa\xaf\x7b\x11\x98\xbe\x4d\x3e", 1, 128, 128 }, + { 96, 192, 80, 380, + "\xb2\xfb\x99\x1a\x1a\xbc\xfd\xba\xfa\x87\xb4\x15\xc8\xfa\x0f\x03\x95" + "\xe3\x2f\x23\xd7\x8a\x1a\x88", + "\x0b\xbb\x61\x0c\x4f\xb7\x55\x18\x0e\xfe\xe9\xcb", "", + "\x17\x3a\xbd\xf0\xe8\x4a\x4d\x6b\xce\x7f\x84\x9c\x50\xee\x54\x80" + "\xc5", + "\x19\x5c\xe4\xaa\x74\xd9\x9f\xd8\xe5\x44\x4e\x29\x6e\x6a\x5d\x13" + "\x9e", + "\x29\x08\xe7\xb3\xa0\x07\x2a\xb8\x64\x6d", 1, 0, 136 }, + { 96, 192, 80, 381, + "\x00\x1c\xca\xa8\x5e\xd7\xda\x56\xfa\x3f\x1e\x9b\x47\xe3\xa2\x04\x2c" + "\x18\xf2\x1c\x19\xe6\xe9\x64", + "\x8c\x16\xa9\x44\xb8\x0e\xef\x9d\x32\x5e\x1b\x71", + "\x21\x8a\xd9\xdb\x9c\x23\x92\x14\x87\x58\xec\x3c\xc4\x8f\x9c\x12", + "\xed\x07\x99\xeb\xa5\x04\x59\x5e\x80\xa7\x32\x5d\x13\x4c\x5d\xe3" + "\x9a", + "\x2e\x3e\x78\xff\x00\xe5\x83\xa6\x39\x45\xdc\x2c\xb7\x28\xa2\x84" + "\xfb", + "\x3c\x89\x31\x64\xf1\x45\x6d\x54\xb9\xdd", 1, 128, 136 }, + { 96, 192, 80, 382, + "\x1d\x75\x7f\x84\xc6\x47\xa4\x2c\xe3\x95\xb5\x4d\xb5\xd9\x21\x79\x86" + "\x27\xba\x1b\xcf\xcc\x7f\x64", + "\xc0\xa2\xa0\xfc\xf5\xc2\x00\xe1\x7c\x32\xc3\x94", "", + "\x51\xa2\xa8\xd9\x95\x84\x4e\x4e\x78\xf9\xb2\x0b\x1a\xf6\x73\x20\xb1" + "\x80\x90\x3c\xbb\xf4\xef\xbc\x60\x1b\x99\xb4\x1f\x07\xf8", + "\x6b\x8b\xc9\x3c\xb3\x48\xd8\x42\xf3\x23\x6b\x66\x58\xde\x7e\xe3\xf5" + "\x57\xe9\x34\x69\x25\x3e\x8a\xfc\x7f\xee\xa8\x7f\x78\xf4", + "\x81\xd5\x96\xe3\x77\xa3\xa3\x01\x64\x0e", 1, 0, 248 }, + { 96, 192, 80, 383, + "\x54\xf1\x0b\xae\xb7\x56\x4e\x94\x7b\xb6\xe1\xe2\xa8\x37\xc1\x6d\xda" + "\xe0\x64\x6a\x8b\x7e\xb3\x38", + "\x8e\xc4\xd8\x54\x4f\xd2\x1e\x6a\x51\x32\xab\xc9", + "\x87\x96\xea\x33\x62\x18\xd2\xa0\x99\x1b\x4c\xb4\x23\x01\xf6\x5d", + "\x1d\x8a\xab\x01\x08\xd7\x29\x90\x92\x8b\x9d\x1a\x8a\x48\x0b\x93\xaf" + "\x27\x63\x4b\x16\x60\x77\xe3\x13\x4e\x2e\x87\x91\xca\x13", + "\x00\xf6\x49\xa1\xfb\x32\x1a\x48\xfc\x1d\xac\xd5\xb9\xfc\x19\x77\x9d" + "\x7f\xb4\x94\xca\xd6\x0e\x2c\x2d\x72\x77\x13\xcd\xe9\x3f", + "\x7f\xc9\x19\xa9\x2b\xf3\x57\x7b\xd0\xb0", 1, 128, 248 }, + { 96, 192, 96, 384, + "\x08\x71\x13\x72\xdb\xac\xac\xbb\x68\xef\x12\xe5\xef\x59\xb6\x9f\xd4" + "\x6c\x9b\xe4\xc2\xfb\x83\x24", + "\x83\x21\x74\x84\x12\x38\x0b\x0e\x7b\x14\xa7\xef", "", "", "", + "\x6c\x1e\xe8\xd9\x71\x8f\x72\x4c\xe8\xd9\x6b\xeb", 1, 0, 0 }, + { 96, 192, 96, 385, + "\x0a\x12\x32\x6b\x7e\xfa\xc1\x79\x42\x1f\xdb\xce\xd8\x0d\x52\xf2\x40" + "\x7e\x99\x3e\xf5\x04\x77\xf9", + "\x34\x47\x21\x31\x54\x78\x40\x26\x3d\x9e\x9f\xd3", + "\x17\xbe\x4b\xb1\xe4\xa4\x0f\xac\x70\x68\x79\x38\x1a\x2d\x6f\x47", + "", "", "\x6f\xb7\x2e\x37\x68\x83\x2a\x7b\xab\x90\x7a\x75", 1, 128, + 0 }, + { 96, 192, 96, 386, + "\x56\xd5\xf5\xeb\x69\x7d\x96\xfa\x9c\x11\x52\x8b\x19\x1e\xaf\xc4\x15" + "\x9c\x2c\xb2\x90\x2f\x06\x95", + "\x8e\x04\xdf\x79\x13\x42\x92\x99\xcf\x2f\x23\x37", "", + "\x9f\x6e\xda\xfc\x71\xef\x15\x61\xd7\x00\x5a\x53\x3a\x5c\xde\xb5", + "\x53\x61\x42\xd2\x7a\x03\x12\xb8\x0e\x53\xff\x32\xbe\x18\x9e\x29", + "\xf1\x6a\x36\x4c\x64\x22\x9f\xab\x13\x90\x87\x61", 1, 0, 128 }, + { 96, 192, 96, 387, + "\x25\x94\x30\x07\xa4\x49\x78\x39\xbc\x13\x38\x69\x45\xb4\xbc\x46\xbf" + "\x10\x5e\xaf\x6e\x6b\xec\x2c", + "\x56\xad\xa6\xe5\x59\xc2\x68\x75\x50\x92\xbd\x6f", + "\x25\xfe\x12\xa5\x28\xd1\x26\xb4\xcc\xbf\x68\x10\x17\x0d\xc2\x8c", + "\x20\x21\x1e\x52\xeb\xbd\x1b\xba\x78\x38\xd4\x02\xe8\xeb\xba\x93", + "\x4c\x41\x2a\xc4\x1e\xc5\x22\x82\x5a\x88\x44\xf7\xd5\xf8\xf6\x07", + "\x81\x77\x42\xf2\xa0\x1b\x9d\xbe\x9a\x7f\x39\x02", 1, 128, 128 }, + { 96, 192, 96, 388, + "\xeb\x3c\xa2\x96\x88\x8a\x76\x28\x98\xe5\x10\x3f\x0d\x54\xd5\x38\x74" + "\xfc\x7f\x4e\x4b\x9d\x21\x5d", + "\xf6\xa1\x64\xa5\x5c\xbe\x06\x44\x72\x39\x71\xb2", "", + "\x96\xda\x54\x41\xe8\x83\x12\x53\x6d\x28\x92\xb1\xe2\x7b\x41\xe6" + "\x41", + "\x35\x8c\x85\xd8\x3d\xce\x34\x5c\xc5\xa1\x56\x60\xad\xb0\x16\xa9" + "\xf8", + "\xf9\x3d\x30\x5c\x3c\xf6\x92\x93\x28\x9e\x09\xd6", 1, 0, 136 }, + { 96, 192, 96, 389, + "\xaf\xe1\x2b\xcd\x5e\xf3\x54\x90\x71\x3d\x20\xfe\xd4\x8f\x6b\x94\x2b" + "\x08\x1b\x9f\x24\x44\x41\x83", + "\x95\x3e\x94\x4b\xea\xa7\x6f\xd2\x46\x3c\x27\x8b", + "\xc9\x02\x81\xef\xb0\xb9\x48\x9b\x61\x72\x2f\x1f\xc7\xde\x5b\xa6", + "\x94\x9f\x81\xce\x36\xdb\xe8\x5e\xac\x14\xa7\x2b\x8b\x77\x58\xea" + "\x47", + "\x69\x7c\x99\xee\xe5\x05\x6d\xec\x98\x51\x60\xab\x2a\xfc\xbf\x0c" + "\x3f", + "\x03\xbb\x58\xfa\xcd\xe9\xaf\x29\x08\xb5\x2e\x55", 1, 128, 136 }, + { 96, 192, 96, 390, + "\x73\x8e\x95\xe8\x89\xdc\x79\x3e\x29\xf3\x3b\x9e\x35\xee\x3c\x10\x30" + "\xd7\x53\xe5\xe9\x9b\xdd\xbd", + "\xc3\x29\x71\x81\x6c\x7d\x84\x34\x2f\xf7\x64\x88", "", + "\x7f\x64\x2c\x9e\x9d\x91\x57\x1b\x87\x45\x0d\x59\xa9\xbe\x2b\xe6\xb4" + "\x5c\x5b\x8a\x0e\xeb\x32\x69\x32\xc3\xe8\x75\x11\x84\x85", + "\x4a\xef\xf5\xa7\xca\x46\xa8\x80\x4e\xab\x6f\x23\xcb\xeb\x24\x02\x72" + "\x40\x8a\xf0\x64\x47\xb7\xa6\x73\x8f\x91\xc4\xa9\x0f\x20", + "\xc7\xee\xdf\x4f\x03\x33\x29\xa5\x12\x50\x44\x55", 1, 0, 248 }, + { 96, 192, 96, 391, + "\x70\xbc\x9c\x8a\x60\xda\xc5\xf2\x53\xec\xf3\x2c\x7d\x1e\x6d\xe1\x31" + "\xea\xb7\x9f\xaa\x83\x1e\x76", + "\x45\xeb\xab\xe2\xb6\xa0\x3f\xbb\x15\x97\x85\x31", + "\xab\xd0\x9a\xe3\x17\x84\x91\xea\x28\x98\x2b\xc8\x39\xe3\x97\x21", + "\xee\x8c\x1a\x65\xe1\xc2\x49\x1e\xe7\x25\xa2\x85\xad\x1f\x3a\x22\x75" + "\xc2\xec\x4a\xf8\x2b\xa3\x2a\x66\xcd\x7e\x87\xdb\xff\xea", + "\xf4\x2c\x5b\xeb\x2a\xf7\xa9\x71\x5d\x53\x5c\xb7\x21\xba\xdd\x42\x1d" + "\x47\x2f\xab\xee\x43\x4c\x77\xf4\x2d\x0e\x4b\x16\x3e\x4c", + "\xb5\x0f\x8c\x07\x0e\x11\x70\xb1\x0b\x9a\x99\x32", 1, 128, 248 }, + { 96, 192, 112, 392, + "\xcd\x2a\xdc\x91\xb1\x9d\x56\x4b\xab\xc9\x7e\x12\x03\x7c\x8b\xc9\x1a" + "\xf6\x87\xf9\x59\xda\xe1\xd4", + "\x67\x8b\x89\xfd\xf4\xe1\x35\xdd\x67\xe3\xf2\x8f", "", "", "", + "\xa7\x54\x07\xb6\x29\x89\xf1\xbe\x04\x13\x1a\x43\xce\x16", 1, 0, 0 }, + { 96, 192, 112, 393, + "\x59\xb4\xb1\x81\x6f\x2a\x9f\xb6\x26\x6a\x39\xaf\x3e\x49\x7e\x2b\x89" + "\xb1\xfa\x51\xfc\xb9\x65\xef", + "\x99\x05\x97\x9f\x5b\x03\xa5\x0d\x54\x40\xaa\x08", + "\xe8\x2c\xb1\x13\x39\x72\x77\xe2\x20\x05\x2a\xc5\x53\x04\xd7\x93", + "", "", "\x67\xf6\xe8\xc4\x46\x5b\xb6\x47\xc0\x3e\xf4\xfc\x5f\x1d", 1, + 128, 0 }, + { 96, 192, 112, 394, + "\x2d\xe7\xb9\x83\x7d\x63\xf5\x31\xdb\x27\x05\xc5\xe2\xc8\x00\xaf\xbf" + "\x5c\xce\xf7\x3b\x80\xf7\x9d", + "\x1a\x0c\xe3\xa2\xe9\x28\x3d\x06\x92\x85\x41\x6c", "", + "\xc1\x26\x51\x83\xd4\x09\x5f\xbe\xa0\xbf\xa3\x5b\x27\x81\xd5\x2a", + "\x34\x0b\x2f\xd7\xc3\x9b\xba\x1a\x1a\x93\x91\xb6\x01\x0e\xf8\xb4", + "\x99\xb1\x9c\x91\xeb\x23\x12\xff\x5c\x42\xbd\x88\x90\x68", 1, 0, + 128 }, + { 96, 192, 112, 395, + "\x7f\xda\x00\x39\x31\xc5\xea\x09\xaf\xa3\xc9\x3b\xca\xa9\xcd\x33\xaf" + "\xfa\x55\x06\x1d\xf3\xc4\xda", + "\xc7\xa7\xaf\xe9\xd8\xd9\xda\x3a\xc8\x1c\x7b\x58", + "\x32\x09\x2c\x8d\xc6\x2f\xf2\x57\x0f\xae\x6e\xcc\xcf\xd9\x2b\xe2", + "\xc5\xbd\xa3\xc8\x6f\x31\x16\x0c\x96\x23\x98\x4d\xf8\x85\xf9\x2a", + "\x50\x22\x4b\xff\x26\xb6\xb9\x66\x69\x32\x50\x37\xf1\x29\x4a\x2a", + "\xf1\xfe\x37\x1a\x39\x74\xcd\x23\x80\xda\xb2\xc7\xdb\x62", 1, 128, + 128 }, + { 96, 192, 112, 396, + "\x0b\xb8\x9e\xe6\x66\xcc\x14\x3c\x89\x58\x3c\xe3\x05\x5d\x02\x80\xa3" + "\xff\x65\xdd\x5b\x0a\xc9\xa8", + "\xca\x9e\x52\xc9\xf7\x5a\xe6\x26\x25\x6d\x21\x0d", "", + "\x4c\x8c\xdd\x0f\x6c\x9e\x8a\x00\x91\xb7\x30\x70\x42\x98\xb9\x0e" + "\xaa", + "\x93\x18\x5d\xe9\x8b\x9b\x95\xa1\x18\x55\x09\x64\x40\x02\x7f\xf5" + "\xd5", + "\x8e\xdf\x2a\x34\x08\x3c\xd4\xfc\x82\xee\x34\x90\x4d\x55", 1, 0, + 136 }, + { 96, 192, 112, 397, + "\x44\xde\xf0\x2b\x19\xd8\xb7\x4b\x25\x80\x1e\xc2\x52\x73\xb6\x8f\x50" + "\xde\xa1\x26\xec\x4a\x36\x66", + "\x21\x5d\xbc\x07\x2f\x69\x8b\xa9\x6f\x85\x50\x48", + "\x20\x28\x29\x92\x7e\x08\xe4\x0a\xed\x36\x96\xff\xde\xdd\x10\x7d", + "\x49\x25\xd7\xf7\x0c\x12\xa6\xb8\x48\x7d\x0c\x9f\x16\xf4\x8e\x8e" + "\x8d", + "\xc5\x4f\xb9\xe5\x55\xfb\xcb\x5e\x1e\x70\xaa\xaa\xef\xbc\x12\x25" + "\x00", + "\x1b\x59\x84\x53\x8b\xea\xfa\x71\x14\x2f\x0c\x0e\xc4\x2a", 1, 128, + 136 }, + { 96, 192, 112, 398, + "\xa2\x9a\x0c\x3f\x1e\x2e\x3e\xe8\x8d\xfd\x3f\x01\x9b\x42\x5a\x5f\x64" + "\x75\x26\xd3\xf3\x68\xa1\xbe", + "\xf2\xba\x8a\x66\x20\xd4\xe5\x34\x87\xb8\xd6\x6d", "", + "\x23\x59\x87\x89\xd7\xa7\xa2\x48\xd1\x7e\xc0\xc6\xaa\x31\x32\xb4\x10" + "\x2c\x0d\xf2\xfd\xab\xa4\x3e\x4e\x45\x81\x43\x9b\xdf\xdb", + "\xb0\x80\x13\x0f\x95\xe4\x6d\x79\xed\x5f\x67\xf0\xdc\x88\xa2\x3c\x34" + "\xdc\xf7\x4f\x6e\x61\xb6\x56\x21\xb6\xef\x53\x75\x82\x79", + "\x67\x85\x6e\x34\x44\x53\xca\xb3\x35\xa8\xf0\x7b\x1f\x63", 1, 0, + 248 }, + { 96, 192, 112, 399, + "\x90\x95\x4f\xb2\x2a\xa8\x48\x60\xdd\x7c\x3f\xad\xb3\x19\xdd\x1d\x16" + "\x85\x77\x23\xbd\x1b\xad\xb1", + "\x7c\xa5\x17\x8d\x42\x79\xad\xb8\xd2\x2b\x48\x70", + "\xfc\x77\xe3\xc5\x01\x03\xc5\x86\x08\x82\xe8\xce\xd3\x40\x29\x33", + "\xea\x6e\x7a\xa0\x10\xe1\x9a\x7c\x76\xa4\xd7\xd3\x44\x0d\xc6\x1e\xed" + "\xa4\x4a\x5a\x6f\xb7\xfa\x82\x44\x17\x18\x5d\x4a\x55\xce", + "\x87\xb7\x53\xfa\x7c\xe3\xe7\xd1\x62\x92\x59\x46\xe8\x57\x19\x93\x3a" + "\x0d\xd1\x0e\xaf\x72\xb5\x25\x9c\xba\xce\x8c\x41\xc5\x3b", + "\x2b\x55\x64\x55\xa6\x57\xee\x61\x71\xf4\xed\xe6\x43\xb5", 1, 128, + 248 }, + { 96, 256, 32, 400, + "\x99\x48\xed\xf5\xcf\xb2\xf5\x33\x63\xed\x83\xbf\xb1\x5e\x7c\xb5\x02" + "\xf0\x62\x8d\xc9\xf2\xb8\x72\x23\xf2\x23\x34\xc4\x0b\x89\x23", + "\x44\xe1\x54\xe9\xb3\xf7\xfd\x47\xa9\x7f\xc7\xbe", "", "", "", + "\x92\xa5\x99\x22", 1, 0, 0 }, + { 96, 256, 32, 401, + "\x93\xb7\x5a\xc1\x29\xec\x19\x5f\x8c\x18\x0e\x9b\x91\xdf\xed\xaa\xe2" + "\xb2\xfa\xcd\xc1\x55\x93\xb3\xe4\x25\x8c\x78\xd2\xff\x94\xd7", + "\xdb\x53\x5f\xe7\x23\xba\x65\x0b\x66\xd2\x30\xfb", + "\x7b\x3d\xd4\x20\x60\x71\x39\xc1\x9c\x6d\xb7\xa4\xef\xe0\x9a\x0b", + "", "", "\x50\xe4\x2c\x1f", 1, 128, 0 }, + { 96, 256, 32, 402, + "\xc7\x67\x91\x45\xa1\x5e\x53\xdb\x5c\xd6\x16\x61\x43\xa9\xfe\xfe\x67" + "\x46\x71\x5f\x5d\x84\xd9\xdf\xa6\x04\xf1\xd3\xdc\x33\x7e\x6c", + "\x6a\xc0\xd6\xaa\x44\x6e\x86\xff\x32\xf8\xfc\x76", "", + "\x83\xb0\x83\x05\x52\x6f\xbc\xbc\xde\xeb\xb3\xd7\xa8\xac\x44\xf5", + "\xdc\xb5\x25\x05\x59\xa0\x3c\x8e\x70\xe5\xc0\x10\x71\x21\xcf\x58", + "\x00\x81\xc1\xfd", 1, 0, 128 }, + { 96, 256, 32, 403, + "\xbc\x5c\xaa\x30\x6c\x42\x3e\x6f\x85\x0c\xd5\x64\x4b\x09\xdd\x4b\xa7" + "\x76\xb3\x01\x71\xc7\x2e\x00\x50\xe5\xa6\x0a\xfe\x9c\xb7\x7c", + "\xf9\x56\xd4\x14\x22\xd8\xeb\x63\x23\x1b\x38\x26", + "\xfa\x6e\x34\x46\x33\x18\x23\x7e\x98\x5d\xd2\xf7\x2b\x0d\xd0\x14", + "\x46\xc9\xda\x60\x2a\x54\xfe\x80\x37\xcf\x0b\xee\x72\xaf\xfc\x72", + "\xdb\xb6\xb4\xec\x70\xf9\x32\x4f\x4b\xc2\x2b\x59\x24\x09\xd4\xac", + "\x80\x3a\x69\x48", 1, 128, 128 }, + { 96, 256, 32, 404, + "\xa9\x59\xc6\x10\xa8\xef\x46\x8b\xb8\xe8\x66\xa0\x9b\x26\x27\xa6\xc3" + "\x9e\xe2\xed\x51\x0d\x22\xe8\x72\xaf\xa6\x3e\xba\xb7\xcf\xb0", + "\xf6\x48\x00\x2f\xfd\x7c\xff\x0b\xd2\x6d\x1c\x45", "", + "\x34\x65\xe9\xb8\x35\xc2\x16\x95\xbf\xd9\xa5\x20\xa9\xe0\xf0\x79" + "\xd1", + "\xbc\xa3\x38\xdc\x06\xcc\xf0\x3c\xbf\x30\x25\x1c\xce\xac\x64\x8a" + "\xaa", + "\x97\x6e\xd7\x31", 1, 0, 136 }, + { 96, 256, 32, 405, + "\xe2\x61\x31\x4c\x54\x0a\xef\x81\x14\x5a\xc2\x23\xff\xc7\xbd\xe0\x11" + "\x65\x67\x93\x57\xe7\x86\xcd\x2f\x88\x15\xe2\x3f\x1d\x69\xdf", + "\xfd\x69\xf4\xb9\x39\xe3\xbb\x09\x00\x6f\x2d\x2b", + "\xae\x20\x5a\x7a\xcc\x94\x57\x16\xf7\x52\xf0\x95\x42\xb7\x8c\x5a", + "\x90\x64\x8b\x56\xd3\x5b\xf1\xca\x99\x0e\xa2\x59\x50\x35\x4f\xf1" + "\xa3", + "\x66\x6d\xe4\x14\xb3\x38\x90\x81\xd0\x70\x28\xd5\xa6\xa3\xf8\x5d" + "\x5b", + "\x49\x5a\x49\x8b", 1, 128, 136 }, + { 96, 256, 32, 406, + "\x04\x79\x81\x7a\xfb\x26\xc2\xce\x77\xb7\x15\xbb\xb0\xd6\x43\x02\xfb" + "\x09\xff\x92\x5d\x34\x98\x35\xcd\x1d\xd3\x27\x9f\xbb\x72\x38", + "\x4f\xa9\x0e\x2d\x99\xc7\xa6\xd2\x5d\x38\xbc\xb4", "", + "\xdd\xeb\xd4\xe2\xaf\x2e\xfe\x97\x20\xc9\xe2\x72\xe4\x01\xb9\x3a\xc1" + "\x1b\x0b\x8f\xf9\x76\xad\x2d\xea\x0c\xbb\x3e\x8c\x5a\x7f", + "\x6e\x20\x73\xfa\xa6\x80\xe0\x58\x85\xa5\x9b\x7a\x75\xdd\xe2\xd3\x0f" + "\xd6\x33\x32\x33\xfc\x9d\x03\xe9\x9c\x49\x0f\x8c\x94\xef", + "\xbe\x02\x37\xf8", 1, 0, 248 }, + { 96, 256, 32, 407, + "\x22\x8e\xdf\xfb\x10\x35\x24\x83\x59\x07\x72\x3f\x7a\xf9\xec\x18\x02" + "\x3c\xb8\x2c\x71\x96\x97\xb3\xa1\xc5\xdf\x0f\x2c\x30\xab\x18", + "\x44\xf7\xcb\xcb\xf2\x5c\x4c\x0f\xaf\xea\x93\xf2", + "\x2d\x70\x18\x20\x3f\x67\x83\x38\xef\xb6\xb3\x41\x14\x97\x94\x1f", + "\x10\x64\x7f\xb1\xe5\x04\x0f\xa0\x09\x09\xd3\xfe\x51\x71\xf0\x4c\x1c" + "\xe9\x45\x40\x83\x5e\x19\xe6\x25\x35\x5b\x81\x3d\x81\xe7", + "\x2e\x15\x81\xea\x47\x4d\x67\x07\xa6\x94\xbb\xab\xb2\x6e\xfb\xad\xe1" + "\xeb\x8d\x8e\x8c\x06\x3f\x7c\x05\x82\x09\xeb\x1b\x33\xb5", + "\xb0\x6b\x64\xb5", 1, 128, 248 }, + { 96, 256, 48, 408, + "\xf1\x62\xc3\x19\xc6\xcf\xf0\x93\xd5\x95\x6a\xee\xde\x37\x01\x18\x19" + "\x42\x88\x82\x11\x08\x78\x24\x81\x78\x27\xa4\x32\xf8\x6d\x9f", + "\x69\xd7\x42\xd9\x4f\xee\x25\x11\x40\xe6\xd7\x79", "", "", "", + "\xfc\xdc\x5a\xa3\x39\x14", 1, 0, 0 }, + { 96, 256, 48, 409, + "\xed\xdc\xcd\x5e\xda\x6e\xaf\x42\x1b\xbf\x87\xd9\x19\x54\x9c\x1f\x3a" + "\xc2\x04\x5c\x0a\xc2\xbc\xc2\xef\xa5\x0e\xc8\x40\x50\xb3\x6e", + "\x30\xf3\xdb\x31\x2e\x76\xd2\x93\x45\xed\xde\x59", + "\x07\x8e\x76\xef\x2d\xee\xbd\xd8\xf2\xd5\x49\x08\x9f\x4a\x93\xe3", + "", "", "\x0d\xf6\xd2\xdc\x83\xdf", 1, 128, 0 }, + { 96, 256, 48, 410, + "\x1c\x9b\x20\xe6\x4a\xd7\x83\xbf\x04\xf8\x01\xbe\x53\x2f\x6b\x08\x8e" + "\x00\x4d\x3a\xa2\xd7\x2d\x77\xf3\x9e\xc8\xfe\x9d\xdc\x51\x89", + "\x49\x7f\xa4\x1d\xf3\x08\x58\xe3\xfb\xb3\x6a\x68", "", + "\x03\xd3\x3c\x0a\x11\xa6\xcd\xa9\x9d\x76\xe9\x8f\x75\x05\x9f\xbf", + "\xde\xea\x99\x43\x7d\x38\x5b\x21\x1f\x3d\xeb\xda\x65\x86\x9d\xaa", + "\x81\x8e\xa9\x63\x04\x2c", 1, 0, 128 }, + { 96, 256, 48, 411, + "\x5c\x8b\x72\x87\x01\x76\x56\xc3\x10\x8d\x7e\xb6\x14\x37\x10\x4b\x41" + "\x1f\xd2\xd6\x15\x24\x5b\xf2\x3c\x82\x7d\x3d\xab\xe4\x30\xa5", + "\x99\x6a\x93\xe4\x7c\x2d\xab\x38\xc9\x35\x29\xae", + "\x23\x33\xe2\xc9\x58\x02\x88\x3f\xb3\xcf\x98\x73\x4d\xcf\x9c\x64", + "\xd1\xa2\x2a\x8d\xa2\x20\x07\x2c\x49\xd8\xaa\x1e\x28\x33\x34\xa6", + "\xd6\xc3\xe9\x29\x18\x13\xd3\x9a\xd9\x19\x48\x79\x03\xc6\xa7\xa5", + "\xe6\x39\x54\x04\x16\xfd", 1, 128, 128 }, + { 96, 256, 48, 412, + "\x96\x4c\x2d\x69\xf7\xb5\x3c\x40\x62\x88\x41\x01\xd5\x62\xf5\x23\x16" + "\xcc\xbc\x81\x4a\x29\xb0\xfe\x6e\xfe\x7f\x1e\xc7\xf7\xdd\xfe", + "\x63\xe4\xd4\x4f\x41\xf3\xce\x45\x14\xb7\x37\x00", "", + "\x52\x36\xea\x08\x20\xe8\x37\x45\x21\x2c\xdc\xd7\xc1\x0a\x5f\x35" + "\x29", + "\x05\x67\x5f\xa4\x2a\x07\xd4\x3f\xe9\x1b\x53\x39\x7f\x74\x60\x9c" + "\xff", + "\x54\x53\x41\x14\x41\x5d", 1, 0, 136 }, + { 96, 256, 48, 413, + "\xc0\xa9\xd3\x35\xf3\x29\x96\x6a\x5b\xc8\xcd\xef\x38\x60\x90\x80\xb8" + "\x5a\x2e\x6e\x96\xf6\xac\x82\x03\x67\x94\x96\x6e\x7c\x82\xc1", + "\x58\x29\xfb\x77\xbb\xdf\xb3\x87\x21\xa5\x91\x00", + "\x49\xa3\xed\x9c\xd3\x09\x68\xfd\xb7\xff\x73\xd1\x2d\x30\xe1\x55", + "\xff\x00\xf8\xc6\xcc\xbc\x90\xa8\x4f\x94\xfc\x98\x8c\xba\xb8\x2c" + "\xa1", + "\x29\x9b\x30\xe3\xd3\xc6\x06\x0b\xf5\xd2\x1f\x7f\xc0\x13\x89\x69" + "\x68", + "\xa5\x96\x5c\x20\xfd\xc0", 1, 128, 136 }, + { 96, 256, 48, 414, + "\x48\x71\xb9\x1e\x5e\x7f\x3c\xf9\xcc\x1b\x01\xd5\x0b\xc6\x20\x36\x10" + "\x75\xad\xa3\xed\xc4\x23\x39\x8d\x47\x40\xde\x72\x1f\x8c\xe1", + "\x7b\xd8\xcf\x2f\xc2\x4a\x3a\x83\x5c\xf9\x1b\xf7", "", + "\xe4\x39\x36\xc2\xd0\x5a\x3a\x35\xf7\xc2\xdd\xd1\x65\xd3\x97\xe5\xd3" + "\xc2\xec\x2b\x48\x23\x60\xd3\xf2\xe6\x21\x7c\xe0\x00\x37", + "\x43\xb6\x65\x99\xdd\x97\x82\xbe\xcf\x88\x4f\x04\x4f\x0c\x85\xb4\xae" + "\x6f\x7f\xd0\xf6\xce\x2a\xfb\xba\x84\x2e\x6b\x59\x4b\x3a", + "\x55\xd8\xf7\x6e\xa7\xe6", 1, 0, 248 }, + { 96, 256, 48, 415, + "\x71\x70\xed\x6d\xbf\x43\x4b\xfd\x0b\xcb\x6b\xd6\x92\xa3\x69\x36\x52" + "\x51\xfa\x31\x90\x9b\x4a\x2e\x3b\xee\x10\x66\x3a\x01\xe0\x0f", + "\xd1\x3e\xe3\x9b\x84\x2f\x86\x0a\x5f\x4d\x78\xe3", + "\xf8\x7d\x88\x71\xa8\x95\x1c\x39\x85\x73\x21\xe3\x20\xb8\xb8\x36", + "\x14\x5b\xe0\xa7\x8b\xdb\x38\x01\x4e\xe6\x19\x31\x45\x13\x1e\xc8\xa3" + "\xfd\x7c\x89\x79\x3a\x30\x05\x36\x4f\xf1\xe7\x93\xf6\x7b", + "\x6b\x91\xe6\x79\x92\xf8\x70\x30\x6f\x24\x24\x70\xc5\x11\x31\x87\x3e" + "\x2a\x6c\x07\x6c\xda\x25\x9c\x33\x49\xc9\x49\x4c\x39\x04", + "\xc8\xcb\x36\x0f\x80\xc5", 1, 128, 248 }, + { 96, 256, 64, 416, + "\xd2\xa4\x1c\xd9\xce\x5e\x91\x7d\x16\xb9\xab\x55\x81\x9e\xf8\x50\x1e" + "\x06\xaa\x78\xef\x13\x2f\xd3\xeb\xe6\xfe\xcd\x91\xbe\xb3\x9b", + "\xf7\x1b\xf6\xbc\x21\xc6\xd6\x35\x4e\x4b\x4c\xdf", "", "", "", + "\x23\x92\x8a\x00\x9d\x21\xa1\x0f", 1, 0, 0 }, + { 96, 256, 64, 417, + "\x88\x21\x07\xab\x29\x05\x3d\x4b\x44\xc8\x7b\x5b\xb9\x49\x37\x21\x1c" + "\x20\x52\x8d\xa9\xac\x49\x0f\x6c\x57\x4c\xae\xcd\xcd\x2f\x17", + "\xe6\xa1\x35\x37\xbb\x7f\x2a\xf7\x49\xb3\x18\x23", + "\xe9\xee\x32\xe6\xf1\x97\xe4\x02\x04\x68\x2d\xac\x42\xdd\x4c\x75", + "", "", "\x57\x73\xc7\x25\xf2\xf9\x46\x17", 1, 128, 0 }, + { 96, 256, 64, 418, + "\xb9\x67\x09\x1c\x98\xbb\x64\x92\x24\x30\x83\x3d\x1b\x55\x33\x26\xb8" + "\xe9\x1b\x6e\xf7\x14\x19\x71\xcc\x8e\x8c\xc5\xf6\xef\x61\x70", + "\xa5\xdd\x07\x6d\x8a\x9d\xc3\xd7\xec\x43\xd0\x4f", "", + "\xc8\xa3\x31\xb5\x54\xe6\xc7\xb0\x78\x3c\x53\xfe\xe6\xf1\x61\x8e", + "\x99\xb5\xc2\x22\x25\xe5\x32\x5f\x9a\xa9\x59\x9a\x34\xde\xec\x59", + "\xe9\xc9\x36\x19\xd3\x3d\x26\x8d", 1, 0, 128 }, + { 96, 256, 64, 419, + "\x71\x60\x43\x47\x20\x50\x4d\xce\x28\x84\x56\x25\xa3\x42\x31\x66\xd9" + "\xb5\x02\x5d\x97\x5c\x6e\xe4\x72\x99\xbb\x5b\xd6\x77\xdb\xeb", + "\x02\x96\xc9\x5b\x44\xc1\x74\x63\x43\x4c\x7e\x19", + "\x19\x08\x2b\xf5\x7b\x6c\x41\x30\xea\xc5\x8c\x05\x26\xa0\x44\xee", + "\xc6\x6a\x48\x61\x5b\x62\xd2\xd8\x5e\xa8\x2e\xe4\xd5\x28\xa0\x3a", + "\x89\x34\xea\x7a\xfb\x44\xfd\xca\x40\x27\xed\x9b\xbb\x24\x73\x58", + "\xb3\x33\xf0\xe1\x38\x3c\xf3\xe8", 1, 128, 128 }, + { 96, 256, 64, 420, + "\x1d\xd5\xa0\x92\x94\x9b\x67\x63\x5d\xb0\xc4\x8a\x03\x74\x0d\xa8\x06" + "\xdb\xe9\x7a\xad\x5b\x84\x12\x30\x0d\x68\x5c\xec\xfe\x84\x07", + "\x3f\x9e\xa3\x93\x62\xc8\xd8\xe4\x92\xea\x8b\x41", "", + "\x84\x40\x8d\x8b\xcc\xb4\x28\x8e\x62\x2b\xf7\xc6\x31\x40\x1d\x99" + "\x08", + "\x4f\x19\xd2\x7e\x0a\x40\xe4\x83\x5e\xbc\x3b\x89\xce\x8b\x51\x94" + "\x1b", + "\x44\xd9\xe5\x0d\xca\x91\x5c\x2e", 1, 0, 136 }, + { 96, 256, 64, 421, + "\x47\xf6\x64\xe6\x79\x0f\x3e\x25\xbc\x41\x0d\x84\x7f\x38\x66\x2f\x04" + "\x5f\x0a\xa3\x64\x14\x29\xed\xf8\x09\x9f\x4b\x4d\xf3\x2f\x06", + "\xf0\x92\xa3\x57\xb5\xef\x0c\x97\x5e\xe1\x69\xc4", + "\x33\x8b\x4c\xc6\x0e\xc1\x51\xfa\x28\x3c\x1c\xb1\x0e\x72\x2d\x9d", + "\xb0\x1d\xfe\x72\x41\x66\xa2\xbc\x98\xcb\xb9\x6c\xf5\x40\x02\x8a" + "\x0e", + "\xd7\x74\x6f\x18\x6a\xab\xfa\x36\x68\x54\x81\xec\x8a\x7f\x00\x22" + "\xe8", + "\x41\x15\x82\x92\xa1\xd8\x7c\xfd", 1, 128, 136 }, + { 96, 256, 64, 422, + "\xa4\x7a\xbe\x6e\x86\x7f\xbc\x16\xc4\x6a\x6f\xd7\xf1\x0b\x77\x92\x9b" + "\xaa\x12\x93\x69\xc8\x98\xd2\x52\x65\xb0\x17\x00\x56\xf9\xd0", + "\x65\x0b\x12\x68\x7c\xa8\x5a\x50\xe6\x50\x98\x84", "", + "\x20\x09\x14\x57\x1d\xd0\x38\x27\xf0\x7c\x2b\xd9\x38\x2e\x7d\x19\xd6" + "\x2f\x1e\xa4\xa7\xc7\x26\x9d\x86\x73\x3e\x43\xe4\x5a\x4d", + "\xbc\x23\x14\xa5\x89\xdb\xdd\x95\xb3\x58\xcd\xad\x30\xb1\x5e\x86\x7d" + "\xcd\x8d\xbd\xe4\x28\xb4\x7e\x39\x0a\xc4\x37\x62\xf6\x34", + "\x88\x1f\xa5\xfe\xcb\x51\x4c\xcf", 1, 0, 248 }, + { 96, 256, 64, 423, + "\x11\x04\x80\xea\x9c\x9f\x4c\x5e\x6b\x5b\xe0\x1a\x2a\xaf\xc8\x61\xd1" + "\x37\x0c\x24\x3a\xff\x9f\xaa\xfd\x0a\x92\xa9\xd1\x8e\x58\x45", + "\x0e\x5c\xf6\x83\xe1\x32\x04\xcf\x91\xa2\xd4\xb6", + "\xc4\x90\xa5\xfa\x19\xb9\x7c\x3e\x3a\xdf\x20\xbc\x4d\xf5\x11\x40", + "\xc9\x2e\xc3\xd6\xa2\xc2\xfa\x19\xc4\x5b\xe7\x10\x7a\x48\xa9\xea\x0f" + "\xe4\x6a\x92\x97\x8b\x5d\xab\xb3\xf9\x4b\x45\x7b\x5f\xbd", + "\xbb\x51\x10\xdd\x12\xbd\x3d\x12\x14\x4c\x8d\xe5\x5b\x3b\x26\x77\xfc" + "\x70\x84\xd5\x6a\xfc\xc6\xa7\x6a\x52\x28\xff\xf8\xdb\xd3", + "\xe3\x9b\x0d\x11\x74\xf7\x60\x9b", 1, 128, 248 }, + { 96, 256, 80, 424, + "\xa0\x91\x7e\xbe\x15\x17\x78\xcb\x88\xbb\x2e\x35\x61\x69\xad\x1a\x4b" + "\x9e\xbe\x2b\xcc\x2a\x35\x2b\xc7\x89\xa5\x0b\x4f\x31\x2d\x3e", + "\x32\x81\x13\x54\x38\x26\x08\xbd\x07\x6d\x8a\x87", "", "", "", + "\xb4\x1d\x1d\xaf\xd0\xd2\x59\x31\xd2\x85", 1, 0, 0 }, + { 96, 256, 80, 425, + "\xaa\x8a\x48\xf8\xb6\xd1\x86\x34\xec\x96\x33\x8e\x82\x0f\x7e\xb9\xf0" + "\xfe\xa8\x86\x4b\xb9\x27\xa5\x7c\x65\xf8\x34\x49\x90\x19\x9b", + "\xa0\xf7\x30\x46\x48\xf9\x7a\x30\x34\x91\x6d\x35", + "\xf4\xab\xe3\x08\x15\xce\x6a\xe9\xcf\x2f\x4e\xaa\x8b\xd0\x04\xcb", + "", "", "\xf7\x60\x5f\x52\x01\x93\x6d\xa1\x6d\x39", 1, 128, 0 }, + { 96, 256, 80, 426, + "\x8a\xb5\x05\x16\xb0\x53\xa3\xed\x51\xb9\xf8\x4f\x76\xdb\xf9\x30\xbd" + "\xe2\xb5\x5a\xa4\x99\xa0\x16\x19\x43\x50\x46\x1f\xf0\xc7\x08", + "\x14\x0a\x72\xad\x89\xb2\xfa\x23\xc3\x85\xe8\x04", "", + "\x53\x6b\x90\x06\xa4\x1f\xeb\xbe\x7a\x10\xd1\x6a\xe2\xb6\x44\x88", + "\xf1\xed\x66\x67\xa2\x18\x87\xa3\x94\xd8\x16\xa4\x5a\xe0\x6a\x5d", + "\x45\x55\xc7\x16\x14\xa7\x65\xc6\xa8\xfc", 1, 0, 128 }, + { 96, 256, 80, 427, + "\x14\x50\xb9\xd4\x36\x61\xc2\x7d\xbd\x08\x00\xd6\x61\x6c\xac\xf4\xe2" + "\x83\x10\x99\x0e\x74\x4f\x8a\x89\x66\x54\xae\x43\x87\x2b\xcb", + "\x91\xb8\xa7\x08\xbe\x02\xcb\x63\x35\xc2\x85\x83", + "\xa4\xa3\xe0\xca\x16\x5b\xff\xcc\x30\x52\x05\x66\x7c\x38\x68\x6b", + "\x6c\x03\x01\x32\x6a\x61\x33\xf5\xd5\xfa\x87\x17\xda\xe4\xe1\x90", + "\x37\xd8\xf3\x8e\x20\x4c\x36\xc0\x29\xcf\x15\xf7\xff\x3a\xc5\xda", + "\xfd\x94\x29\x18\xf7\xaa\xf3\x08\xe5\x6e", 1, 128, 128 }, + { 96, 256, 80, 428, + "\x1b\x05\xcb\xa5\x87\x24\x72\x13\xa0\xd9\x59\xd6\x4a\x29\xa5\x9e\xe2" + "\xd0\xee\xa2\xd9\x7e\xfa\x29\x68\x61\x43\x4d\xb8\xe5\x27\x54", + "\xb8\x3b\x0e\x7a\x52\xdd\x50\x7a\x8d\x67\x36\x61", "", + "\xd0\x76\x3c\x30\x60\xb7\xf9\xeb\x2d\x42\x75\x8c\xcb\x3e\xbb\x03" + "\x11", + "\x27\xb4\x4a\x64\xbc\xd0\xa0\x73\xd7\x70\xa7\x1d\x38\x2b\xd4\xec" + "\x99", + "\x9a\x1c\xff\x81\x02\x66\x69\xdb\x70\xd1", 1, 0, 136 }, + { 96, 256, 80, 429, + "\xe2\x54\x11\x83\x8a\x5a\x8d\xc7\xfa\x86\x6e\xa7\x43\x47\xab\x00\x3f" + "\x2a\x86\x62\x27\x5b\x69\x3a\xe8\xa6\xdd\xba\x97\x9e\x18\x87", + "\xbd\xad\x47\x51\x7b\xb9\xb6\x6b\x5e\x64\xc2\x19", + "\x9a\x1b\x3c\xda\xc0\x76\x7c\xb2\x34\xe5\xe4\x68\x78\x6c\xe3\x27", + "\x88\x98\x29\xa2\xb4\xa8\x86\xd3\x9f\x1f\x7f\x68\xc2\xcd\xb4\xf3" + "\x65", + "\x80\x66\xe4\xa8\xcd\xf7\xac\x2f\x70\xe2\xf5\xba\x51\x26\xd5\xe3" + "\x47", + "\x39\x29\x24\x8c\x35\x8b\xdc\xf3\x62\xf1", 1, 128, 136 }, + { 96, 256, 80, 430, + "\xde\x1c\x7d\x37\x84\xb9\x82\x50\x42\x2e\x6f\xff\xed\x88\x57\x71\x54" + "\xc1\x93\xf7\x2d\x4a\x97\x96\xd4\xff\x4d\xfc\x88\x23\x5a\x17", + "\x9d\x14\xa6\xb7\x93\x32\xee\x97\xc4\x8f\x07\xe1", "", + "\x55\xdc\x11\x79\xcd\xad\x38\xd4\x5e\xd4\x39\x39\x5c\x67\xa8\x72\x4d" + "\x75\x13\xa9\xa4\xc6\x2f\xb5\x9a\x78\x8b\x0a\xc6\x7b\x7d", + "\x94\x82\xb6\x00\x66\xc9\x99\xcc\x89\x5c\xf9\x80\xe8\x1a\x29\x23\x7f" + "\x80\x9e\x9b\x80\xb3\x24\x90\xe6\x0a\xc8\x57\x30\xca\xfc", + "\x67\x5e\xb8\x19\x7e\x60\x5b\xdd\xf2\xe5", 1, 0, 248 }, + { 96, 256, 80, 431, + "\xbd\x02\x55\x52\xc3\x4a\x55\x2f\x07\xde\x3a\x34\x8b\xf7\xdf\xb3\x08" + "\xbe\xc3\x6c\x47\x89\x3a\xd2\x9f\x3f\xe4\x41\xe2\x4f\xb2\x55", + "\x18\x51\x40\xaa\xc8\x3f\x26\x1a\x8c\x0d\xce\xa4", + "\xa7\x21\xa6\x9f\x3a\x24\xdd\xbc\x2e\x16\x01\x52\x28\xc8\x48\x3a", + "\x05\xba\xbe\x1d\x63\xf8\x12\x06\x9d\xfc\xd0\xf5\x92\x62\xfe\x05\xbd" + "\x45\xb3\xc1\x1a\x3d\x6b\xdf\xea\x5a\x0c\x80\xd1\x32\x20", + "\x74\xc9\x70\x0f\xb3\xc7\xbd\x4d\x65\xbd\xcd\x0d\xf8\xcc\x73\xa4\x14" + "\xad\x9c\xd7\x87\xb0\x5c\xc9\xff\xbf\xb6\x3c\x84\x8d\x1a", + "\x4d\xd4\x3d\xc3\x2b\x30\x16\x73\xf4\x04", 1, 128, 248 }, + { 96, 256, 96, 432, + "\x07\x37\x42\x4e\x0c\x2f\x40\x48\x63\x81\x33\xa1\x8d\x67\x6d\xc1\xd8" + "\x3a\x23\x38\x77\x61\x3a\xcc\x0e\xb5\xa6\x81\x30\x53\x66\xc0", + "\xf0\x28\xd0\xec\xf2\x6c\x31\x2b\x9f\x62\x33\x95", "", "", "", + "\xd3\x7f\x07\xc4\xec\xef\x1f\xca\xf0\xfe\x44\x4a", 1, 0, 0 }, + { 96, 256, 96, 433, + "\x8e\x4c\x13\xc9\x82\xa0\x6f\x3a\x98\x29\x59\xeb\x7c\x2e\x9f\x0e\x41" + "\xa8\xe0\x54\x36\x0e\x5b\x93\x11\x1b\xc6\xd9\x39\x70\xee\x8d", + "\x8c\x08\x1e\xb6\x0f\xa0\x90\x35\x95\x71\x3a\x73", + "\x24\xf1\xed\x7c\xad\x53\x54\x68\x02\xe2\xe5\xf5\xed\x51\x62\x47", + "", "", "\x97\xfe\xd4\x10\xc9\xfd\xb0\x6b\xcd\xb3\x85\x85", 1, 128, + 0 }, + { 96, 256, 96, 434, + "\x62\x7f\xfd\x55\x17\x6d\x65\x55\xda\x82\xb4\xeb\x87\xe6\x51\x90\x44" + "\xb8\x81\x33\x4c\x95\x78\x9d\x67\x07\x29\xaf\x05\x84\x12\x8b", + "\xc1\x5c\xb8\xab\xce\x00\x8f\x01\x5e\x27\x15\xae", "", + "\x00\x02\x24\xe6\x3d\x99\xe8\xb1\xa0\xa2\xab\xb4\xb4\x5b\xca\x15", + "\x59\xa1\xb9\x55\x22\xe9\x6a\x5f\xea\x0a\xe7\x7d\x17\x92\x23\xec", + "\xaa\xb2\xf3\x4d\xe6\xe5\xba\xc7\xcc\xf9\x36\x18", 1, 0, 128 }, + { 96, 256, 96, 435, + "\xbb\x65\xd8\x0b\x7a\x47\x82\xe0\x5f\xfc\xb7\x77\xe5\x95\x28\xba\xb8" + "\x7e\x20\xaa\x84\xdb\xe4\x58\x8e\x2a\x17\x03\xf8\x8c\x68\xca", + "\x56\x41\x0b\xb8\x2b\xb0\x54\x23\x4b\x5e\x62\xc1", + "\xff\xe0\x9f\xb3\x4f\x17\xb5\x17\x95\x6f\xbb\xb5\x8a\x62\x62\x3a", + "\x3a\xf8\xc0\x49\xa1\x93\xb1\xca\x39\x52\xee\xd0\xf5\x8f\x09\xdd", + "\x54\xee\x65\x4f\x5c\x44\xb8\x58\x76\x43\xd4\xc5\x8d\xe4\x02\x67", + "\x32\x37\xb2\xfa\x6a\xd7\x85\xa8\x82\xa3\x8e\x72", 1, 128, 128 }, + { 96, 256, 96, 436, + "\xde\x59\xc6\xda\xa2\x10\xca\x6b\xed\xd9\xdb\x7b\x30\xe8\x86\x03\x04" + "\x9b\x18\x0f\x6e\x31\x96\xb4\xc3\x3d\x8c\x51\x89\xb5\xc4\x50", + "\x1f\xc9\x68\x4e\x4d\x96\x8b\xfe\x27\x77\x50\x00", "", + "\xcd\x42\xfb\x94\xb1\x07\xa8\x89\x1b\x15\x9b\xf3\xbd\xb3\xed\xa8" + "\x44", + "\xfb\x48\xf5\x71\x63\x3d\x67\xd5\x34\xcd\x20\xb6\xc8\x81\x7e\x96" + "\x33", + "\x55\x1d\x3b\xb6\x86\xee\xdf\xdf\xf7\x76\xef\x19", 1, 0, 136 }, + { 96, 256, 96, 437, + "\x22\x23\x4e\x83\x14\x09\xb5\xfb\xec\x25\x2c\x78\x5d\x69\x4b\x00\x4a" + "\x59\xff\xda\x15\x6c\xff\x62\xf5\x70\x2b\x72\xfb\xf1\x00\xad", + "\x5b\x38\xb9\x53\x11\x5e\x80\x88\x43\x0e\xbb\xd8", + "\x0d\xfb\xea\x34\xbe\xbb\x2c\xcd\xeb\x12\x77\xe0\xb4\x4a\xcc\xfb", + "\x60\x04\x18\xcb\xef\x85\x64\x39\xe4\x0d\x83\x9f\x7b\x57\xc5\xe3" + "\x2e", + "\xda\xaa\xb3\xcc\xa5\xab\x11\xf9\xe1\xf4\x4c\xdb\xfe\x82\xb6\x0c" + "\x8f", + "\xae\xf2\xf1\xa9\x0f\xfa\x6e\x96\x89\x2f\x37\x28", 1, 128, 136 }, + { 96, 256, 96, 438, + "\xb8\x74\xb8\x69\xd0\x04\x50\x51\x4f\xa1\xf8\xfb\x94\x7c\xc0\x87\xe8" + "\x73\x2e\xd0\x76\x0b\x41\xb2\x21\xc6\x9c\xda\x04\x9c\xba\x02", + "\x41\x01\x2a\x5f\x5c\x6b\x70\xac\xee\x93\xbb\xa1", "", + "\x1b\xe4\x61\x3b\xb9\xa8\xa1\x24\x60\x66\x50\xde\x32\x62\xf2\x57\xfd" + "\x6b\xae\x4b\x7c\x27\xb4\xf0\xff\x36\xba\xee\x97\xbc\xb8", + "\x83\xcc\x85\xa0\x13\xc8\x2f\xe0\x7d\x24\xb3\x84\x80\xf3\x0d\x6e\x09" + "\x27\x4a\xf8\x80\xf1\x14\xe0\x8b\x56\x28\x54\x7a\x04\x2b", + "\xb7\x8d\x7f\x57\xe5\xa6\xad\x5d\x77\x08\x38\x76", 1, 0, 248 }, + { 96, 256, 96, 439, + "\x10\xf9\xd3\x90\xd9\xe8\x9f\xdd\x3b\xde\xd9\xcb\xcb\x6c\x98\x5f\x9c" + "\xfa\xe0\x07\x49\xfe\x7c\xd4\x0c\x83\xa6\xeb\x95\xb4\xdc\xeb", + "\x21\xa2\x28\x6f\xee\xe9\x73\x86\xec\x1d\x2a\x49", + "\x2d\xee\x72\xe8\x9b\x03\x97\x93\xf6\xa2\x8c\x92\x02\xd6\x26\x59", + "\x2c\xa3\x70\xd1\x4c\x09\xa5\xab\xa5\x32\x7b\x4d\xe3\x0a\x98\x3f\x6e" + "\x50\x21\xea\xa7\xb5\x74\x50\x89\x1e\xaf\x38\x6b\x7a\xe9", + "\xa7\x51\xee\x90\x93\x08\x18\x07\xb5\x24\x07\x59\x19\xfc\x64\xca\x80" + "\x6b\x3f\x5a\x29\xca\xb2\x6b\x06\x57\xe1\x63\x04\x2f\x96", + "\x74\x3d\xf3\xe0\x1f\x34\x49\x63\x45\x73\x57\x15", 1, 128, 248 }, + { 96, 256, 112, 440, + "\x08\xf5\xfe\x4c\x8f\x63\x93\xac\xcd\xcb\x56\x0a\x3c\x27\x10\x96\xff" + "\x0d\x9d\x67\x43\x8f\xff\xd3\x4d\xf7\x18\x65\x2c\x6b\x8e\xfe", + "\x25\xc5\xf8\x4f\xe6\xec\x3c\x2f\x7c\x1b\x7c\xc5", "", "", "", + "\x66\xb2\x30\x23\xe6\x08\xcd\x93\x91\x56\x7a\xa8\x5f\x5a", 1, 0, 0 }, + { 96, 256, 112, 441, + "\x84\x95\x29\x8b\x9c\x20\x8e\x6c\x5b\x23\x4e\x85\x6e\xec\xff\x6a\x11" + "\x4c\xd8\xb3\xae\xee\xb7\x45\xa1\x60\xff\xa3\x30\x5c\xf5\xef", + "\xcc\xe0\x26\x35\xc3\x77\x1f\xb5\xb6\x73\xf8\x8c", + "\xcc\x9a\xf3\xcc\xa9\xf3\xc2\xc1\x21\x1b\x23\x58\x1e\xc5\xfd\xd1", + "", "", "\x0d\x34\x89\x6c\x64\xb6\x78\x7d\xa0\xac\x7c\x03\xfa\x93", 1, + 128, 0 }, + { 96, 256, 112, 442, + "\x4f\x49\x65\x5c\x76\xa6\x29\xe5\x8c\xfb\x94\xc8\x51\xa9\x15\x10\xc2" + "\xf1\x28\xdc\x4b\xce\x1f\x1f\x11\xc3\xdc\x99\x43\x6d\x26\x8c", + "\x96\x7f\xde\x29\x67\x1d\x46\x54\xf9\xf6\x70\xc0", "", + "\xb4\xd1\x2c\x3e\xdf\x38\x02\xe2\x1f\x62\x4b\x71\x8b\x63\xfd\x6c", + "\x3b\xa4\xe2\xa4\x50\x7c\x0b\x6f\x5a\xe1\xbe\x29\xc3\x0b\x25\xe9", + "\x8c\xef\xa2\x49\x54\x73\xee\xe1\xb2\x2c\x3f\xa6\xef\x12", 1, 0, + 128 }, + { 96, 256, 112, 443, + "\x4d\xac\x9a\x0c\xbf\xc3\xdd\x29\x1d\x40\x6e\x68\x38\x89\xfc\x10\xe2" + "\xa0\xdd\x25\xd4\xd0\xb4\x3b\x11\x11\x1a\xa8\x28\x27\x39\xe9", + "\x86\x4a\xa8\xc8\x65\x58\x8c\x9a\x21\xae\xa7\xfd", + "\x5d\xb8\xe0\x96\x97\xd1\xff\x79\xa8\x86\x39\x5e\x40\xfb\x1a\x1d", + "\x5d\x2c\x63\x29\x60\xf8\x23\xcf\x72\x42\xbf\x61\xf9\x39\x13\x17", + "\xfd\x80\xa3\x82\x7d\xb1\x72\x42\xb2\xdf\x0c\xd8\xca\x96\xd9\x97", + "\xdf\xb5\x8a\x6b\xe4\xe7\xe0\x01\x0f\x7c\x74\x04\xb4\x67", 1, 128, + 128 }, + { 96, 256, 112, 444, + "\x37\x76\xa8\x4b\x86\x9e\xc4\xa7\x1e\xd8\x4a\x74\xe6\xa9\x8c\x42\xc0" + "\xff\xa2\x3f\x6e\xb2\xe2\x97\x0f\x13\x11\x21\xc5\xba\x69\xfe", + "\x72\xcd\xa6\xef\xb0\x82\x5c\x74\x0d\x19\xf4\x85", "", + "\x05\x05\xd2\x88\x90\x8d\x5c\x28\xe4\x72\x3d\x9d\x4b\x8b\x0f\xc0" + "\xba", + "\xdc\xd6\x2b\xdc\x23\xee\x8b\xdb\x7a\xfd\xec\xd4\x49\xcd\xb4\x99" + "\x4a", + "\x9a\xa0\xf8\xf0\x32\xbe\x13\x42\xad\x5d\x40\x99\xc3\xae", 1, 0, + 136 }, + { 96, 256, 112, 445, + "\x01\xfc\xbc\x4a\x3b\x2e\xe3\x21\x09\xcd\x0f\x27\xd8\x29\xe2\x0d\x1d" + "\x92\x03\xd6\xff\x81\x2e\xd9\x84\x1e\xf9\x08\x90\x4d\x74\xa8", + "\xae\x9e\xe7\xf9\xf5\x28\x78\x21\x58\x38\xf5\xcb", + "\x1b\xe1\x77\xd6\xe8\x86\x51\xc4\x0f\x6a\x1b\x53\x38\x17\xc2\x79", + "\x26\x8a\x75\xfb\x89\x0e\x8a\xf7\xc2\x4b\x63\xcf\xb8\x70\x80\xe0" + "\x28", + "\xe1\x44\x0c\x49\x55\x62\xf6\xc8\x56\x28\x81\x83\x23\x4e\x0a\xd2" + "\x2e", + "\xd9\x3e\x1c\x12\xbc\xc7\x66\x60\x12\x7d\xfd\x8c\x28\xa0", 1, 128, + 136 }, + { 96, 256, 112, 446, + "\xa0\x17\x45\xf5\x2f\x1f\x35\x64\xda\x0a\xdf\x84\x5f\xdb\xd4\x7a\x5b" + "\xd1\x86\x50\x92\x57\x95\x58\xf6\x7f\x67\xba\x07\xf2\x38\xa0", + "\x87\xd7\xce\xc6\x30\x1b\x81\xe3\xe0\x66\x6e\x27", "", + "\xd5\xc0\x12\x80\xac\xf0\xaf\xe7\x7d\xf7\x67\xff\x3c\x02\x8f\x52\xe3" + "\xd3\x78\x6a\x84\xcc\x7c\xc0\x07\x06\x61\xa8\x1c\x1f\xbd", + "\xcb\x07\xfc\x59\x62\xf7\xd3\x26\x86\x06\xf1\xd2\x24\xfd\x92\xb3\xc2" + "\x30\x26\x20\xf0\x33\x20\x78\x4a\x71\x18\x0d\x72\x65\x01", + "\x7f\x64\xeb\xeb\x84\xbc\xad\x46\x34\x7f\xf1\xf2\x74\x47", 1, 0, + 248 }, + { 96, 256, 112, 447, + "\x24\x0e\xf4\xec\x0a\x7b\x24\x01\x7c\x13\xe4\x61\x22\x7d\x11\xf6\x08" + "\xc4\x16\x98\x45\x7e\x94\x8f\x65\x7d\x82\xa1\x9d\x97\x05\x44", + "\x52\xec\x46\xf5\x27\x81\xbb\x7c\xd0\x0f\xbf\xd3", + "\x89\x9c\x64\xab\xbe\xc1\x46\x8e\xc5\xb8\x42\x7e\x61\xb9\x90\xab", + "\x2b\x25\x88\x2f\x82\x4b\x41\xea\xf4\xb2\x15\x0e\xb1\xfe\x8d\xc0\xf9" + "\xc7\x15\x6a\x41\x88\x1b\x39\xd1\x3d\xae\xc1\xf9\xb0\xb1", + "\xe9\x4e\x44\xb5\xe7\xbb\x26\xb2\x49\xb4\x8c\xaa\xf2\xa9\xab\x5a\x75" + "\x06\xff\x39\x66\x8f\xfe\xa6\xf6\x2b\xb0\x30\xfe\x5c\x87", + "\x1a\x85\x9a\xa8\x06\x26\x04\x72\xa5\x39\x79\xcc\x4e\xaa", 1, 128, + 248 }, + { 0, 128, 96, 448, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xbe\x17\x22\xa5\x81\x71\x95\xc5\x03\x81\x4b\xe1\xbd\x09\x31\x10", + "\x6f\x79\xa8\xcf\x92\xc8\x56\xb8\xf1\x6d\xee\x92", 0, 0, 128 }, + { 8, 128, 96, 449, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x21\x03\x89\x22\x69\x58\xde\xf4\xb4\x4f\x1e\x16\x86\x32\x11\x3c", + "\xd4\xe9\x7a\x10\x08\x00\xa5\xc1\x6b\xea\x4f\xdf", 0, 0, 128 }, + { 16, 128, 96, 450, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\x41", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x95\xbf\x20\x8e\x86\x73\xb9\xf9\xa3\x8f\x96\x09\xb5\xe7\x8f\x2a", + "\xdf\x81\x41\x91\x69\x6c\xf3\x12\x9f\xb4\x0d\xc0", 0, 0, 128 }, + { 32, 128, 96, 451, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\x41\x42\x43", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xa7\xda\xb4\xbf\xcf\x3e\x2a\xfa\x4b\x31\x9c\xfd\xc1\x7f\x15\xf1", + "\xc4\xe8\xb1\xfa\x0d\x79\x17\x77\x41\x7c\xe5\x2c", 0, 0, 128 }, + { 48, 128, 96, 452, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\x41\x42\x43\x44\x45", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xa8\x9a\x23\xf4\xf3\x2b\xa0\x09\xc3\xaa\x8f\xa0\x19\x1f\x84\xc5", + "\x65\x96\x21\xc2\xad\x5b\xc6\x1d\xe2\xce\x80\x46", 0, 0, 128 }, + { 112, 128, 96, 453, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xf9\xd0\x07\x8b\xcd\x56\x8b\xf9\x79\x24\xe6\xd7\x1f\x40\x60\x87", + "\x09\x5c\x0d\xa2\xea\x6d\xda\x5a\x87\x12\x1c\x2a", 0, 0, 128 }, + { 120, 128, 96, 454, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x67\x0c\xd0\x68\xe6\x5b\x08\xfc\xef\xb9\xc0\xaf\xa7\xbb\x5c\x33", + "\xda\xa0\x28\x10\xe9\x97\xe7\x97\xdd\xa9\x75\x5d", 0, 0, 128 }, + { 128, 128, 96, 455, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x5e\xdb\xeb\xec\x6c\x53\x3d\xac\x8c\xa8\x9f\xaf\x60\xd8\xc1\x37", + "\x94\x53\xd6\xa8\xff\x91\xb7\x14\xf3\x2c\x0b\x71", 0, 0, 128 }, + { 160, 128, 96, 456, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x5e\xdb\xeb\xec\x6c\x53\x3d\xac\x8c\xa8\x9f\xaf\x60\xd8\xc1\x37", + "\x94\x53\xd6\xa8\xff\x91\xb7\x14\xf3\x2c\x0b\x71", 0, 0, 128 }, + { 256, 128, 96, 457, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x5e\xdb\xeb\xec\x6c\x53\x3d\xac\x8c\xa8\x9f\xaf\x60\xd8\xc1\x37", + "\x94\x53\xd6\xa8\xff\x91\xb7\x14\xf3\x2c\x0b\x71", 0, 0, 128 }, + { 512, 128, 96, 458, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xc3\x2a\x76\x43\xab\x0f\x6e\xa3\x45\x8d\x7e\x63\xb0\xed\x64\x99", + "\x8f\xc9\xb6\xf4\x0b\x3a\x81\xfd\x5f\xd6\xc5\x3c", 0, 0, 128 }, + { 1024, 128, 96, 459, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xc3\x2a\x76\x43\xab\x0f\x6e\xa3\x45\x8d\x7e\x63\xb0\xed\x64\x99", + "\x8f\xc9\xb6\xf4\x0b\x3a\x81\xfd\x5f\xd6\xc5\x3c", 0, 0, 128 }, + { 2144, 128, 96, 460, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98" + "\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba" + "\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb" + "\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc" + "\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed" + "\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" + "\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xc3\x2a\x76\x43\xab\x0f\x6e\xa3\x45\x8d\x7e\x63\xb0\xed\x64\x99", + "\x8f\xc9\xb6\xf4\x0b\x3a\x81\xfd\x5f\xd6\xc5\x3c", 0, 0, 128 }, + { 0, 192, 96, 461, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x15\x69\x9c\x20\x19\x86\x88\xb9\xe4\x88\x2a\x65\x42\x81\x1a\xda", + "\xc6\x93\x17\xb9\x9b\x43\x08\x6b\x62\x1e\xb1\x4a", 0, 0, 128 }, + { 8, 192, 96, 462, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x7e\x84\xb5\xaa\x41\xd2\x21\x2b\x3a\x5d\x73\x0d\xf5\xb2\x0e\xaa", + "\x59\x24\xd8\xbd\x85\x31\x8b\x03\x3b\xf4\xf2\xfd", 0, 0, 128 }, + { 16, 192, 96, 463, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40\x41", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x81\x33\x67\xe3\xd1\xfa\x4e\xe4\xc4\x02\x45\x0f\x29\x46\xd1", + "\xcf\x6e\xee\x49\x5f\x94\xc0\x8f\xef\x7c\xe5\xb5", 0, 0, 128 }, + { 32, 192, 96, 464, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40\x41\x42\x43", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\xe9\xc6\x21\xdb\xe0\x57\xb8\x6a\xca\xef\x8a\xd0\xe3\x8e\xe0", + "\x6d\xd3\x3d\x42\x27\x8b\xb2\xf2\x7e\xab\x7a\x0b", 0, 0, 128 }, + { 48, 192, 96, 465, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40\x41\x42\x43\x44\x45", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x41\xbd\x6e\x62\x6e\xf1\xd4\xfa\x33\xe3\xe6\x2b\x6b\x71\xb2\x47", + "\xb0\x35\x06\xdf\x38\x08\x5e\x4f\x93\xee\x9e\xa4", 0, 0, 128 }, + { 112, 192, 96, 466, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x39\x0f\x24\xac\xc1\x13\xb4\x33\xe4\xb7\x85\xe9\x18\x3d\x48\x38", + "\x9b\xc0\x39\x7f\xee\x59\xe6\x99\x0c\x3b\xbc\x81", 0, 0, 128 }, + { 120, 192, 96, 467, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\xd1\x08\x59\x06\x25\xee\x4a\xf6\x6b\x7c\xb6\x63\xdf\x50\xc1", + "\xcf\xea\xbe\x25\x26\x50\xd1\x25\x84\x47\x8b\xd0", 0, 0, 128 }, + { 128, 192, 96, 468, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x17\xef\x53\xd9\x25\xbb\xa4\xe1\xf3\x35\xd0\x01\x86\xb7\xc3\x8e", + "\x8f\x64\x77\xda\x94\xef\x6f\x63\xc4\xd1\xa0\x71", 0, 0, 128 }, + { 160, 192, 96, 469, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x17\xef\x53\xd9\x25\xbb\xa4\xe1\xf3\x35\xd0\x01\x86\xb7\xc3\x8e", + "\x8f\x64\x77\xda\x94\xef\x6f\x63\xc4\xd1\xa0\x71", 0, 0, 128 }, + { 256, 192, 96, 470, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x17\xef\x53\xd9\x25\xbb\xa4\xe1\xf3\x35\xd0\x01\x86\xb7\xc3\x8e", + "\x8f\x64\x77\xda\x94\xef\x6f\x63\xc4\xd1\xa0\x71", 0, 0, 128 }, + { 512, 192, 96, 471, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x81\xa8\xa0\x87\xcf\x96\xd5\x8f\x64\x86\x8b\x18\x9e\xdd\x0b\xb4", + "\xc5\xcc\x4f\x2b\x5a\x46\x0c\x1d\x22\x9a\x6b\xa8", 0, 0, 128 }, + { 1024, 192, 96, 472, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x81\xa8\xa0\x87\xcf\x96\xd5\x8f\x64\x86\x8b\x18\x9e\xdd\x0b\xb4", + "\xc5\xcc\x4f\x2b\x5a\x46\x0c\x1d\x22\x9a\x6b\xa8", 0, 0, 128 }, + { 2144, 192, 96, 473, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98" + "\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba" + "\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb" + "\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc" + "\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed" + "\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" + "\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x81\xa8\xa0\x87\xcf\x96\xd5\x8f\x64\x86\x8b\x18\x9e\xdd\x0b\xb4", + "\xc5\xcc\x4f\x2b\x5a\x46\x0c\x1d\x22\x9a\x6b\xa8", 0, 0, 128 }, + { 0, 256, 96, 474, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3c\xd9\xba\xce\x5f\x5d\xc7\x7c\x89\xc2\xbc\x13\x90\x65\xe7\x97", + "\x99\xf6\x05\x8e\xeb\x8e\x3a\x80\x36\xaa\xda\xb8", 0, 0, 128 }, + { 8, 256, 96, 475, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb5\xf0\x44\x21\x1e\x18\xb1\x35\x72\xe2\xea\x70\xed\x17\x83\x53", + "\x24\xcf\x3f\x3f\x36\x9b\x69\x2f\xe7\x30\x97\x0c", 0, 0, 128 }, + { 16, 256, 96, 476, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40\x41", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xc0\x00\x74\x6e\xd8\x24\x6d\x20\xed\xda\x90\xc0\x4f\x38\x0b\xa8", + "\xaf\x82\x09\x34\x87\xd3\xa5\xd4\x87\x2f\xf9\xe2", 0, 0, 128 }, + { 32, 256, 96, 477, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40\x41\x42\x43", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb0\x0b\xd7\xb8\xcd\x03\x1c\x16\x8e\x37\x40\x7e\xb0\x9f\x06\x2e", + "\xbf\xca\xe1\x10\xc7\x37\xbb\xe7\x57\x96\x7f\x4e", 0, 0, 128 }, + { 48, 256, 96, 478, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40\x41\x42\x43\x44\x45", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xae\x28\x65\xf8\x6b\xa6\x32\x8c\xe1\x31\xa4\x9c\xd4\x99\xf9\x36", + "\x43\x90\xb5\x4f\x3b\x7c\xbf\x9e\x54\xa2\x20\x48", 0, 0, 128 }, + { 112, 256, 96, 479, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x56\x80\xeb\xd1\x6a\xe4\x46\xe9\xe2\xd0\x7f\xba\xac\x7a\xbd\x0b", + "\xd7\x2d\x01\x57\x82\xfd\x94\xd7\x6d\x2f\x68\x2f", 0, 0, 128 }, + { 120, 256, 96, 480, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x6a\x61\x1e\x6f\xb6\x7d\x83\x1b\x4b\x09\x61\x69\xf2\xe8\x66\x47", + "\x0c\x39\xbb\x24\x62\xbb\xaf\xf7\x19\x39\xee\x1f", 0, 0, 128 }, + { 128, 256, 96, 481, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x68\x18\x41\xa0\xa0\x13\x92\x39\x07\xc6\x69\xef\xd3\xff\xd0\x69", + "\x05\x4c\xd6\xf1\xcd\xa1\xb1\xbf\x91\xe0\x10\x0f", 0, 0, 128 }, + { 160, 256, 96, 482, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x68\x18\x41\xa0\xa0\x13\x92\x39\x07\xc6\x69\xef\xd3\xff\xd0\x69", + "\x05\x4c\xd6\xf1\xcd\xa1\xb1\xbf\x91\xe0\x10\x0f", 0, 0, 128 }, + { 256, 256, 96, 483, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x68\x18\x41\xa0\xa0\x13\x92\x39\x07\xc6\x69\xef\xd3\xff\xd0\x69", + "\x05\x4c\xd6\xf1\xcd\xa1\xb1\xbf\x91\xe0\x10\x0f", 0, 0, 128 }, + { 512, 256, 96, 484, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xe7\x3c\xc2\xee\x05\x0a\x0e\x7b\x34\x5a\xeb\x10\x00\xc4\x81\xc3", + "\x18\x2d\xab\xaf\xf0\x40\x0d\xe7\x08\x74\x6b\x6a", 0, 0, 128 }, + { 1024, 256, 96, 485, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xe7\x3c\xc2\xee\x05\x0a\x0e\x7b\x34\x5a\xeb\x10\x00\xc4\x81\xc3", + "\x18\x2d\xab\xaf\xf0\x40\x0d\xe7\x08\x74\x6b\x6a", 0, 0, 128 }, + { 2144, 256, 96, 486, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43" + "\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51\x52\x53\x54" + "\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f\x60\x61\x62\x63\x64\x65" + "\x66\x67\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f\x70\x71\x72\x73\x74\x75\x76" + "\x77\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f\x80\x81\x82\x83\x84\x85\x86\x87" + "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98" + "\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9" + "\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba" + "\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb" + "\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc" + "\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed" + "\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe" + "\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xe7\x3c\xc2\xee\x05\x0a\x0e\x7b\x34\x5a\xeb\x10\x00\xc4\x81\xc3", + "\x18\x2d\xab\xaf\xf0\x40\x0d\xe7\x08\x74\x6b\x6a", 0, 0, 128 }, + { 96, 128, 16, 487, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\x19\x8c", 0, 0, 128 }, + { 96, 128, 24, 488, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\x19\x8c\x08", 0, 0, 128 }, + { 96, 128, 40, 489, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\x23\x1a\x2d\x8f\x6a", 0, 0, 128 }, + { 96, 128, 56, 490, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\xb1\xbb\xf3\x88\x35\x07\xcd", 0, 0, 128 }, + { 96, 128, 72, 491, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\xcd\x0a\xe6\x3f\x3a\x30\xf7\xfb\x5b", 0, 0, 128 }, + { 96, 128, 88, 492, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\x53\x5e\x32\xac\x41\x68\x16\x61\x5e\x5a\x20", 0, 0, 128 }, + { 96, 128, 104, 493, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\xa6\xc5\x84\x58\xd3\x96\x9d\xa9\xcb\x08\x49\xf9\x5e", 0, 0, 128 }, + { 96, 128, 120, 494, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xd3\xda\xb1\xee\x49\x4c\xc2\x29\x09\x9d\x6c\xac\x7d\xf1\x4a\xdd", + "\xf7\x87\x9f\xb7\xfe\x88\xdd\x74\xcb\x8e\x96\xfd\xa1\xd2\xeb", 0, 0, + 128 }, + { 96, 192, 16, 495, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x95\xeb", 0, 0, 128 }, + { 96, 192, 24, 496, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x95\xeb\x98", 0, 0, 128 }, + { 96, 192, 40, 497, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x3f\xd4\x0d\xd8\xe0", 0, 0, 128 }, + { 96, 192, 56, 498, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x79\x3c\xa5\xd3\x51\xe6\x8c", 0, 0, 128 }, + { 96, 192, 72, 499, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x63\xa0\x98\x7f\xff\xf1\x31\x3c\xaa", 0, 0, 128 }, + { 96, 192, 88, 500, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x0e\xe4\x0f\x14\x47\x5b\x7e\x28\x75\x29\x83", 0, 0, 128 }, + { 96, 192, 104, 501, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\x40\xa4\xfc\x82\xd4\x29\xa0\x09\x1c\x96\x2d\x71\x52", 0, 0, 128 }, + { 96, 192, 120, 502, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x1c\x49\x32\x45\x15\xa3\x46\xd4\x24\xee\xd6\xfe\xd9\xbd\xdc\x17", + "\xf8\xf9\xbd\xc6\xb8\x50\x6a\xfd\x3a\xe5\x4a\x0a\x67\xe1\x85", 0, 0, + 128 }, + { 96, 256, 16, 503, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\x8f\x8b", 0, 0, 128 }, + { 96, 256, 24, 504, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\x8f\x8b\x32", 0, 0, 128 }, + { 96, 256, 40, 505, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\xa9\x4e\x19\xf3\x4c", 0, 0, 128 }, + { 96, 256, 56, 506, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\x5e\x90\x21\x8c\xac\xa4\x70", 0, 0, 128 }, + { 96, 256, 72, 507, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\x38\x5c\xef\x2c\x25\x99\xfa\xa9\x60", 0, 0, 128 }, + { 96, 256, 88, 508, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\xe8\x1c\x27\x00\x20\xed\xd9\x3b\xa7\xe5\x64", 0, 0, 128 }, + { 96, 256, 104, 509, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\xc5\xd5\xf2\x9a\xf5\xc0\xdb\x44\x4a\xc2\x61\x8b\x9d", 0, 0, 128 }, + { 96, 256, 120, 510, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50\x51", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x3b\x68\x29\xd5\xde\xb4\x7c\xa9\xf1\x0a\xbf\x48\x15\x64\xae\xe1", + "\xf2\xf6\xd8\xb6\xab\x69\xc8\xe1\x00\x39\xb5\x75\x4f\x55\x37", 0, 0, + 128 }, + { 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0 } +}; diff --git a/test/wycheproof/aes_cmac_test.json.c b/test/wycheproof/aes_cmac_test.json.c new file mode 100644 index 00000000..911556f0 --- /dev/null +++ b/test/wycheproof/aes_cmac_test.json.c @@ -0,0 +1,1774 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* AES-CMAC, 0.8r12 */ +#include "mac_test.h" +const struct mac_test aes_cmac_test_json[] = { + { 128, 128, 1, + "\xe3\x4f\x15\xc7\xbd\x81\x99\x30\xfe\x9d\x66\xe0\xc1\x66\xe6\x1c", + "", + "\xd4\x7a\xfc\xa1\xd8\x57\xa5\x93\x34\x05\xb1\xeb\x7a\x5c\xb7\xaf", 1, + 0, NULL, 0 }, + { 128, 128, 2, + "\xe1\xe7\x26\x67\x7f\x48\x93\x89\x0f\x8c\x02\x7f\x9d\x8e\xf8\x0d", + "\x3f", + "\x15\xf8\x56\xbb\xed\x3b\x32\x19\x52\xa5\x84\xb3\xc4\x43\x7a\x63", 1, + 8, NULL, 0 }, + { 128, 128, 3, + "\xb1\x51\xf4\x91\xc4\xc0\x06\xd1\xf2\x82\x14\xaa\x3d\xa9\xa9\x85", + "\x27\xd9", + "\xbd\xbb\xeb\xac\x98\x2d\xd6\x2b\x9f\x68\x26\x18\xa6\xa6\x04\xe9", 1, + 16, NULL, 0 }, + { 128, 128, 4, + "\xc3\x6f\xf1\x5f\x72\x77\x7e\xe2\x1d\xee\xc0\x7b\x63\xc1\xa0\xcd", + "\x50\xb4\x28", + "\xbe\x0c\x3e\xde\x15\x75\x68\xaf\x39\x40\x23\xeb\x9a\x7c\xc9\x83", 1, + 24, NULL, 0 }, + { 128, 128, 5, + "\x32\xb9\xc5\xc7\x8c\x3a\x06\x89\xa8\x60\x52\x42\x0f\xa1\xe8\xfc", + "\x0b\x92\x62\xec", + "\x57\xe1\x50\x68\x56\xc5\x5d\xd3\x2c\xd9\xca\x82\x1a\xdb\x6c\x81", 1, + 32, NULL, 0 }, + { 128, 128, 6, + "\x43\x15\x1b\xba\xef\x36\x72\x77\xeb\xfc\x97\x50\x9d\x0a\xa4\x9c", + "\xea\xa9\x12\x73\xe7", + "\xe0\x1a\xdc\x3b\xe6\xa7\x62\x18\x24\x23\x2c\x42\x85\xdd\x35\xb9", 1, + 40, NULL, 0 }, + { 128, 128, 7, + "\x48\x14\x40\x29\x85\x25\xcc\x26\x1f\x81\x59\x15\x9a\xed\xf6\x2d", + "\x61\x23\xc5\x56\xc5\xcc", + "\xa2\x81\xe0\xd2\xd5\x37\x8d\xfd\xcc\x13\x10\xfd\x97\x82\xca\x56", 1, + 48, NULL, 0 }, + { 128, 128, 8, + "\x9c\xa2\x6e\xb8\x87\x31\xef\xbf\x7f\x81\x0d\x5d\x95\xe1\x96\xac", + "\x7e\x48\xf0\x61\x83\xaa\x40", + "\xfc\x81\x76\x1f\x2f\x7b\x4c\xe1\x3b\x53\xd3\x6e\x32\x67\x73\x32", 1, + 56, NULL, 0 }, + { 128, 128, 9, + "\x48\xf0\xd0\x3e\x41\xcc\x55\xc4\xb5\x8f\x73\x7b\x5a\xcd\xea\x32", + "\xf4\xa1\x33\xaa\x6d\x59\x85\xa0", + "\x1f\x1c\xd0\x32\x7c\x02\xe6\xd0\x00\x86\x91\x59\x37\xdd\x61\xd9", 1, + 64, NULL, 0 }, + { 128, 128, 10, + "\x1c\x95\x88\x49\xf3\x19\x96\xb2\x89\x39\xce\x51\x30\x87\xd1\xbe", + "\xb0\xd2\xfe\xe1\x1b\x8e\x2f\x86\xb7", + "\x55\x5f\x46\x21\x51\xf7\xdd\x16\xde\x69\x8d\x63\x9f\xb2\x67\x60", 1, + 72, NULL, 0 }, + { 128, 128, 11, + "\x39\xde\x0e\xbe\xa9\x7c\x09\xb2\x30\x1a\x90\x00\x9a\x42\x32\x53", + "\x81\xe5\xc3\x3b\x4c\x62\x08\x52\xf0\x44", + "\x9b\x00\x4f\x15\xb7\xf6\xf3\x66\x37\x49\x54\xe6\x4b\xc5\x8f\x5f", 1, + 80, NULL, 0 }, + { 128, 128, 12, + "\x91\x65\x6d\x8f\xc0\xac\xed\x60\xdd\xb1\xc4\x00\x6d\x0d\xde\x53", + "\x7b\x3e\x44\x0f\xe5\x66\x79\x00\x64\xb2\xec", + "\x76\x67\x2e\xd1\x6c\x29\xbe\x44\x9e\x0c\x80\x78\x5c\xc3\x8e\x89", 1, + 88, NULL, 0 }, + { 128, 128, 13, + "\xaf\x7d\x51\x34\x72\x0b\x53\x86\x15\x8d\x51\xea\x12\x6e\x7c\xf9", + "\x7c\xc6\xfc\xc9\x25\xc2\x0f\x3c\x83\xb5\x56\x7c", + "\x2d\xc5\xc8\x8c\xf3\xb8\x0a\xb6\xc0\x19\x9f\x40\xbe\x90\x4a\xbc", 1, + 96, NULL, 0 }, + { 128, 128, 14, + "\x4e\xd5\x67\x53\xde\x6f\x75\xa0\x32\xeb\xab\xca\x3c\xe2\x79\x71", + "\x0c\x8c\x0f\x56\x19\xd9\xf8\xda\x53\x39\x28\x12\x85", + "\xea\xb4\x36\x6d\x97\xe9\x9a\x08\x50\xf0\x77\x32\x9a\xd0\x58\xc0", 1, + 104, NULL, 0 }, + { 128, 128, 15, + "\xbe\xba\x50\xc9\x36\xb6\x96\xc1\x5e\x25\x04\x6d\xff\xb2\x3a\x64", + "\x82\x1e\xa8\x53\x2f\xba\xbf\xfb\x6e\x3d\x21\x2e\x9b\x46", + "\x22\xf3\x3c\xab\x09\xc1\x73\xf7\x5d\x34\x01\xfe\x44\xef\xee\xad", 1, + 112, NULL, 0 }, + { 128, 128, 16, + "\x50\x1d\x81\xeb\xf9\x12\xdd\xb8\x7f\xbe\x3b\x7a\xac\x14\x37\xbc", + "\x23\x68\xe3\xc3\x63\x6b\x5e\x8e\x94\xd2\x08\x1a\xdb\xf7\x98", + "\xae\xb7\x84\xa3\x82\x51\x68\xdd\xd6\x1f\x72\xd0\x20\x21\x25\xe6", 1, + 120, NULL, 0 }, + { 128, 128, 17, + "\xe0\x9e\xaa\x5a\x3f\x5e\x56\xd2\x79\xd5\xe7\xa0\x33\x73\xf6\xea", + "\xef\x4e\xab\x37\x18\x1f\x98\x42\x3e\x53\xe9\x47\xe7\x05\x0f\xd0", + "\x40\xfa\xcf\x0e\x2f\xb5\x1b\x73\xa7\x47\x26\x81\xb0\x33\xd6\xdc", 1, + 128, NULL, 0 }, + { 128, 128, 18, + "\x83\x1e\x66\x4c\x9e\x3f\x0c\x30\x94\xc0\xb2\x7b\x9d\x90\x8e\xb2", + "\x26\x60\x3b\xb7\x6d\xd0\xa0\x18\x07\x91\xc4\xed\x4d\x3b\x05\x88" + "\x07", + "\xa8\x14\x4c\x8b\x24\xf2\xaa\x47\xd9\xc1\x60\xcf\xf4\xab\x17\x16", 1, + 136, NULL, 0 }, + { 128, 128, 19, + "\x54\x9b\xd2\x82\xee\x21\xb4\xd7\xc3\xb1\xd0\x2e\x3e\xe2\x0e\xf7", + "\xd8\x4b\xf7\x3c\x5e\xec\xbd\x38\x44\x4f\x1a\x73\x55\x6e\x2f\xa3\x25" + "\x3f\x4c\x54\xd6\x91\x65\x45", + "\x7e\xd4\x58\xaf\xe0\x2f\x4a\x51\x3f\x59\x71\x5b\x66\x4b\x1b\xbe", 1, + 192, NULL, 0 }, + { 128, 128, 20, + "\x9b\xd3\x90\x2e\xd0\x99\x6c\x86\x9b\x57\x22\x72\xe7\x6f\x38\x89", + "\xa7\xba\x19\xd4\x9e\xe1\xea\x02\xf0\x98\xaa\x8e\x30\xc7\x40\xd8\x93" + "\xa4\x45\x6c\xcc\x29\x40\x40\x48\x4e\xd8\xa0\x0a\x55\xf9\x3e", + "\x45\x08\x22\x18\xc2\xd0\x5e\xef\x32\x24\x7f\xeb\x11\x33\xd0\xa3", 1, + 256, NULL, 0 }, + { 128, 128, 21, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x96\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 22, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x43\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 23, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7a\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 24, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x95\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 25, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x40\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 26, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x79\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 27, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x17\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc2\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xfb\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdc\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x81\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xce\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\xda\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\x31\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\x4a\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x89\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x92\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7b\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x8a\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x91\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x78\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\xd6\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\xb2\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x0b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4d\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xae\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x97\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\xcc\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\x2f\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x16\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x19\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xc9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x4f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xaf\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x85\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc4\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1d\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x36\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9e\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1e\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x35\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9d\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x9c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\xb7\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x1f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xab", 0, + 0, NULL, 0 }, + { 128, 128, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x30", 0, + 64, NULL, 0 }, + { 128, 128, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9f", 0, + 128, NULL, 0 }, + { 128, 128, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xa8", 0, + 0, NULL, 0 }, + { 128, 128, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x33", 0, + 64, NULL, 0 }, + { 128, 128, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x9c", 0, + 128, NULL, 0 }, + { 128, 128, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\xea", 0, + 0, NULL, 0 }, + { 128, 128, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\x71", 0, + 64, NULL, 0 }, + { 128, 128, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\xde", 0, + 128, NULL, 0 }, + { 128, 128, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\x56\x4c\x39\xae\x7d\x1c\x5a\x31\x2a", 0, + 0, NULL, 0 }, + { 128, 128, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\x32\xaf\xe9\x84\x44\x37\x38\xcd\xb1", 0, + 64, NULL, 0 }, + { 128, 128, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x8b\x96\x6f\xc5\x39\x9f\x74\x80\x1e", 0, + 128, NULL, 0 }, + { 128, 128, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x96\xdd\x6e\x5a\x88\x2c\xbd\x56\x4d\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x43\x80\x2e\xb1\x93\x1f\x00\x32\xae\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7a\xcf\xbb\xca\x7a\x2e\xa6\x8b\x97\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\xda\x88\x2c\xbd\xd6\x4c\x39\xae\x7d\x1c\x5a\x31\xaa", 0, + 0, NULL, 0 }, + { 128, 128, 82, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\x31\x93\x1f\x00\xb2\xaf\xe9\x84\x44\x37\x38\xcd\x31", 0, + 64, NULL, 0 }, + { 128, 128, 83, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\x4a\x7a\x2e\xa6\x0b\x96\x6f\xc5\x39\x9f\x74\x80\x9e", 0, + 128, NULL, 0 }, + { 128, 128, 84, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x97\xdd\x6e\x5a\x88\x2c\xbd\xd6\x4c\x39\xae\x7d\x1c\x5a\x31\x2a", 0, + 0, NULL, 0 }, + { 128, 128, 85, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x42\x80\x2e\xb1\x93\x1f\x00\xb2\xaf\xe9\x84\x44\x37\x38\xcd\xb1", 0, + 64, NULL, 0 }, + { 128, 128, 86, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7b\xcf\xbb\xca\x7a\x2e\xa6\x0b\x96\x6f\xc5\x39\x9f\x74\x80\x1e", 0, + 128, NULL, 0 }, + { 128, 128, 87, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x68\x22\x91\xa5\x77\xd3\x42\xa9\xb3\xc6\x51\x82\xe3\xa5\xce\x55", 0, + 0, NULL, 0 }, + { 128, 128, 88, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xbd\x7f\xd1\x4e\x6c\xe0\xff\xcd\x50\x16\x7b\xbb\xc8\xc7\x32\xce", 0, + 64, NULL, 0 }, + { 128, 128, 89, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x84\x30\x44\x35\x85\xd1\x59\x74\x69\x90\x3a\xc6\x60\x8b\x7f\x61", 0, + 128, NULL, 0 }, + { 128, 128, 90, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, NULL, 0 }, + { 128, 128, 91, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, NULL, 0 }, + { 128, 128, 92, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, NULL, 0 }, + { 128, 128, 93, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, NULL, 0 }, + { 128, 128, 94, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, NULL, 0 }, + { 128, 128, 95, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, NULL, 0 }, + { 128, 128, 96, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x17\x5d\xee\xda\x08\xac\x3d\xd6\xcc\xb9\x2e\xfd\x9c\xda\xb1\x2a", 0, + 0, NULL, 0 }, + { 128, 128, 97, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc2\x00\xae\x31\x13\x9f\x80\xb2\x2f\x69\x04\xc4\xb7\xb8\x4d\xb1", 0, + 64, NULL, 0 }, + { 128, 128, 98, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xfb\x4f\x3b\x4a\xfa\xae\x26\x0b\x16\xef\x45\xb9\x1f\xf4\x00\x1e", 0, + 128, NULL, 0 }, + { 128, 128, 99, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", + "\x96\xdc\x6f\x5b\x89\x2d\xbc\x57\x4d\x38\xaf\x7c\x1d\x5b\x30\xab", 0, + 0, NULL, 0 }, + { 128, 128, 100, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x43\x81\x2f\xb0\x92\x1e\x01\x33\xae\xe8\x85\x45\x36\x39\xcc\x30", 0, + 64, NULL, 0 }, + { 128, 128, 101, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7a\xce\xba\xcb\x7b\x2f\xa7\x8a\x97\x6e\xc4\x38\x9e\x75\x81\x9f", 0, + 128, NULL, 0 }, + { 192, 128, 102, + "\x3d\x6b\xf9\xed\xae\x6d\x88\x1e\xad\xe0\xff\x8c\x70\x76\xa4\x83\x5b" + "\x71\x32\x0c\x1f\x36\xb6\x31", + "", + "\xa8\xdd\x15\xfe\x2c\xe3\x49\x5e\xc5\xb6\x66\x74\x4e\xc2\x92\x20", 1, + 0, NULL, 0 }, + { 192, 128, 103, + "\x91\x54\x29\x74\x34\x35\xc2\x89\x97\xa3\x3b\x33\xb6\x57\x4a\x95\x3d" + "\x81\xda\xe0\xe7\x03\x2e\x6a", + "\x58", + "\xe1\x3b\x3f\x7f\x7f\x51\x0c\x3a\x05\x9d\xf7\xa6\x8c\x7e\x2a\xd5", 1, + 8, NULL, 0 }, + { 192, 128, 104, + "\xf0\xc2\x88\xba\x26\xb2\x84\xf9\xfb\x32\x1b\x44\x4a\x65\x17\xb3\xcd" + "\xda\x1a\x79\x9d\x55\xfd\xff", + "\x0f\x7e", + "\x06\xef\x84\x7f\x5f\x9d\xbf\x03\xa4\xf2\x83\xda\x8c\x40\x02\x20", 1, + 16, NULL, 0 }, + { 192, 128, 105, + "\x6b\x55\xe4\xd4\xfd\x68\x47\xa8\x0a\x6b\xfb\x0d\xcc\x0a\xa9\x3f\x9f" + "\xd7\x97\xfc\x5c\x50\x29\x2e", + "\x33\xf5\x30", + "\xdd\x13\x50\x53\xa4\x7c\xa8\xf2\x82\xc2\x99\xe8\x3b\x8c\x57\xc4", 1, + 24, NULL, 0 }, + { 192, 128, 106, + "\x1e\xb2\x1a\x9e\x99\x5a\x8e\x45\xc9\xe7\x1e\xcb\xd6\xfe\x61\x5b\x3e" + "\x03\x18\x00\x7c\x64\xb6\x44", + "\x3a\xa7\x3c\x48", + "\x1e\x93\xff\xf8\x46\x93\x4a\x6e\xea\x05\x75\xee\xcb\x0f\x0e\x1f", 1, + 32, NULL, 0 }, + { 192, 128, 107, + "\x71\x0e\x2d\x5d\x4a\x9f\x0b\xc7\xe5\x07\x96\x65\x5e\x04\x6a\x18\xcc" + "\x57\x69\xd7\x76\x43\x55\xda", + "\x7e\x4c\x69\x0a\x88", + "\x01\x6d\x4d\xf0\x6c\x68\xa6\xa7\x88\xa9\xea\x05\x2e\x1b\x55\x0d", 1, + 40, NULL, 0 }, + { 192, 128, 108, + "\xd8\xc0\x9e\xa4\x00\x77\x9b\x63\xe7\x74\xbd\xac\xd0\xcb\x7b\x5d\xd6" + "\xf7\x36\xca\x23\xd5\x2a\xcf", + "\xe9\x52\x02\x80\x97\x3b", + "\x80\x30\xae\x9f\x98\xf5\xd2\x0c\x60\x89\xf6\xb1\xbd\x87\xc2\x9e", 1, + 48, NULL, 0 }, + { 192, 128, 109, + "\x8e\x67\xe9\xa0\x86\x3b\x55\xbe\xd4\x08\x86\x6f\x1c\xbc\x05\x35\x7a" + "\xbe\x3f\x9d\x79\xf4\x06\xf2", + "\x48\x80\xb4\x12\x28\x7a\x0b", + "\xbc\xaf\x50\x78\x5f\x06\x2a\x8f\xb8\xdd\x3c\x2c\x4c\xea\xd2\xe1", 1, + 56, NULL, 0 }, + { 192, 128, 110, + "\x28\xd8\xda\x67\x80\x64\x10\xe5\x56\x5b\xcc\x5a\x9d\x7a\xb9\xfb\x35" + "\x74\x13\xfa\x01\x58\x37\x8c", + "\x00\x4e\x3f\x4a\x4e\x6d\xb9\x55", + "\xc4\xc2\xc0\x87\x6b\xe9\xea\xbe\xb5\xa9\x56\xda\x53\x84\x6b\x08", 1, + 64, NULL, 0 }, + { 192, 128, 111, + "\xdc\x96\x8d\xd8\x9f\xd6\x02\xbb\x7e\xca\x6f\x3a\x8a\x13\xe4\xf5\x9c" + "\x08\xd0\x2a\x51\x4b\x19\x34", + "\x41\xa2\x53\x54\xef\xeb\x1b\xc3\xb8", + "\xf3\x3a\x62\xca\xf3\x97\xf9\xaf\xf7\x1f\xe4\x29\x41\xba\x41\xd8", 1, + 72, NULL, 0 }, + { 192, 128, 112, + "\x76\x58\x95\x1c\x0f\x62\x0d\x82\xaf\xd9\x27\x56\xcc\x2d\x79\x83\xb7" + "\x9d\xa3\xe5\x6f\xdd\x1b\x78", + "\xf0\xe8\x2f\xb5\xc5\x66\x6f\x4a\xf4\x9f", + "\x4d\x72\x4d\x05\xf3\x40\x29\x67\xeb\x65\xae\x1e\x32\xd5\x46\x9e", 1, + 80, NULL, 0 }, + { 192, 128, 113, + "\xd9\x57\x4c\x3a\x22\x1b\x98\x66\x90\x93\x1f\xaa\xc5\x25\x8d\x9d\x3c" + "\x52\x36\x2b\x2c\xb9\xb0\x54", + "\x17\x8e\xa8\x40\x4b\xa5\x4e\xe4\xe4\x52\x2c", + "\x64\xa0\xe0\xb6\x75\x73\x09\xab\x58\xd7\x4f\x72\xc3\x10\xe4\x73", 1, + 88, NULL, 0 }, + { 192, 128, 114, + "\x70\x44\x09\xba\xb2\x80\x85\xc4\x49\x81\xf2\x8f\x75\xdd\x14\x3a\x4f" + "\x74\x71\x06\xf6\x3f\x26\x2e", + "\xcd\xa5\x70\x9e\x7f\x11\x56\x24\xe7\x4a\xb0\x31", + "\x6a\xb2\x07\x43\x34\xbe\x14\xa9\x5b\x6a\x24\x1f\x89\x7a\x43\xde", 1, + 96, NULL, 0 }, + { 192, 128, 115, + "\xd8\xd0\x6e\xf6\xa5\x3b\xbf\xf5\xc8\xf1\x2d\x79\x1b\x8f\x4c\x67\xe5" + "\x74\xbf\x44\x07\x36\xd1\xcc", + "\xa1\x17\x1e\xae\x19\x79\xf4\x83\x45\xdd\x94\x85\xa0", + "\x7a\xa5\x7c\xf9\x8b\x24\x89\x7c\xc9\x23\x0e\x33\x16\x75\x8e\x61", 1, + 104, NULL, 0 }, + { 192, 128, 116, + "\x71\x12\x9e\x78\x16\x13\xf3\x9d\x9a\xc3\x9f\xbd\xe2\x62\x8b\x44\xc2" + "\x50\xc1\x4d\xeb\x5e\xf9\xe2", + "\x96\x75\x93\xcc\x64\xbc\xbf\x7f\x3c\x58\xd0\x4c\xb8\x2b", + "\x6c\xc4\x88\xb0\xa4\x0e\xad\xbe\x4b\xce\xe2\x62\x32\x39\xd1\x26", 1, + 112, NULL, 0 }, + { 192, 128, 117, + "\x85\x0f\xc8\x59\xe9\xf7\xb8\x9a\x36\x76\x11\xde\xe6\x69\x8f\x33\x96" + "\x2d\x82\x45\xca\x8d\xc3\x31", + "\x58\x6f\x4f\x17\x1a\xf1\x16\x51\x90\x61\xa8\xe0\xe7\x79\x40", + "\xfb\x11\xa3\x60\xc9\x77\x69\x91\xd7\x3d\x6e\x41\xd0\x77\x10\xa2", 1, + 120, NULL, 0 }, + { 192, 128, 118, + "\xf4\xbf\xa5\xaa\x4f\x0f\x4d\x62\xcf\x73\x6c\xd2\x96\x9c\x43\xd5\x80" + "\xfd\xb9\x2f\x27\x53\xbe\xdb", + "\x0e\x23\x9f\x23\x97\x05\xb2\x82\xce\x22\x00\xfe\x20\xde\x11\x65", + "\xab\x20\xa6\xcf\x60\x87\x36\x65\xb1\xd6\x99\x9b\x05\xc7\xf9\xc6", 1, + 128, NULL, 0 }, + { 192, 128, 119, + "\xcf\xd3\xf6\x88\x73\xd8\x1a\x27\xd2\xbf\xce\x87\x6c\x79\xf6\xe6\x09" + "\x07\x4d\xec\x39\xe3\x46\x14", + "\xb1\x97\x3c\xb2\x5a\xa8\x7e\xf9\xd1\xa8\x88\x8b\x0a\x0f\x5c\x04" + "\xc6", + "\xb9\x5a\x01\x6b\x83\xa0\xae\x41\x94\x02\x33\x33\xc8\xa7\x34\x5a", 1, + 136, NULL, 0 }, + { 192, 128, 120, + "\x64\x8a\x44\x46\x8d\x67\xbb\x67\x44\xb2\x35\xee\x7a\x3f\xcd\x6e\xd4" + "\xbd\xc2\x9e\xc5\xb5\xfa\x1a", + "\xc5\x9d\x0d\x69\x81\xcc\xa1\xbe\x1d\x55\x19\xfc\x78\x81\xe6\xd2\x30" + "\xf3\x9f\x6c\x12\xa9\xe8\x27", + "\xa1\xb9\x62\x72\xae\x7f\x9a\xef\x56\x72\x71\x79\x5f\x21\xd1\xd3", 1, + 192, NULL, 0 }, + { 192, 128, 121, + "\x9d\x11\xab\xc1\xfc\xb2\x48\xa4\x36\x59\x8e\x69\x5b\xe1\x2c\x3c\x2e" + "\xd9\x0a\x18\xba\x09\xd6\x2c", + "\xaa\x51\x82\xca\xe2\xa8\xfb\x06\x8c\x0b\x3f\xb2\xbe\x3e\x57\xae\x52" + "\x3d\x13\xdf\xfd\x1a\x94\x45\x87\x70\x7c\x2b\x67\x44\x7f\x3f", + "\x85\x97\xd9\xa0\x4d\x1c\x27\x1d\x61\xd4\x2f\x00\x7b\x43\x51\x75", 1, + 256, NULL, 0 }, + { 192, 128, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xed\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc8\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf9\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xee\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xcb\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xfa\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\x6c\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x49\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x78\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x13\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x12\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1a\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x8e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\x5f\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x80\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa1\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x61\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x68\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa2\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x62\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x6b\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x95\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\x4b\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x09\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd8\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x90\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x56\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\x59\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x11\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\xd7\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xf3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x3c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x1b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7b\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7a\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf1\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xcb\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1a\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7d\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xc8\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x19\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 163, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7e\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 164, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\x4a\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 165, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x9b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 166, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\xfc\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 167, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x91", 0, + 0, NULL, 0 }, + { 192, 128, 168, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x91", 0, + 64, NULL, 0 }, + { 192, 128, 169, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8d", 0, + 128, NULL, 0 }, + { 192, 128, 170, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x92", 0, + 0, NULL, 0 }, + { 192, 128, 171, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x92", 0, + 64, NULL, 0 }, + { 192, 128, 172, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x8e", 0, + 128, NULL, 0 }, + { 192, 128, 173, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\xd0", 0, + 0, NULL, 0 }, + { 192, 128, 174, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\xd0", 0, + 64, NULL, 0 }, + { 192, 128, 175, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\xcc", 0, + 128, NULL, 0 }, + { 192, 128, 176, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x15\xd9\xd3\x7a\x6e\xca\x1f\xc9\x10", 0, + 0, NULL, 0 }, + { 192, 128, 177, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\xcb\x91\x1c\x7b\xe6\x1b\xe7\xca\x10", 0, + 64, NULL, 0 }, + { 192, 128, 178, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x89\x57\x3b\xf0\x4e\x7c\xde\x68\x0c", 0, + 128, NULL, 0 }, + { 192, 128, 179, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xed\x12\x39\x0e\xa0\xa7\xed\x15\xd8\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 180, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc8\x13\x07\xdf\x60\x85\x9a\xcb\x90\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 181, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf9\x1b\xde\x00\x69\xa6\xe3\x89\x56\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 182, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x8e\xa0\xa7\xed\x95\xd9\xd3\x7a\x6e\xca\x1f\xc9\x90", 0, + 0, NULL, 0 }, + { 192, 128, 183, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\x5f\x60\x85\x9a\x4b\x91\x1c\x7b\xe6\x1b\xe7\xca\x90", 0, + 64, NULL, 0 }, + { 192, 128, 184, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x80\x69\xa6\xe3\x09\x57\x3b\xf0\x4e\x7c\xde\x68\x8c", 0, + 128, NULL, 0 }, + { 192, 128, 185, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xec\x12\x39\x0e\xa0\xa7\xed\x95\xd9\xd3\x7a\x6e\xca\x1f\xc9\x10", 0, + 0, NULL, 0 }, + { 192, 128, 186, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc9\x13\x07\xdf\x60\x85\x9a\x4b\x91\x1c\x7b\xe6\x1b\xe7\xca\x10", 0, + 64, NULL, 0 }, + { 192, 128, 187, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf8\x1b\xde\x00\x69\xa6\xe3\x09\x57\x3b\xf0\x4e\x7c\xde\x68\x0c", 0, + 128, NULL, 0 }, + { 192, 128, 188, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\x13\xed\xc6\xf1\x5f\x58\x12\xea\x26\x2c\x85\x91\x35\xe0\x36\x6f", 0, + 0, NULL, 0 }, + { 192, 128, 189, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x36\xec\xf8\x20\x9f\x7a\x65\x34\x6e\xe3\x84\x19\xe4\x18\x35\x6f", 0, + 64, NULL, 0 }, + { 192, 128, 190, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x07\xe4\x21\xff\x96\x59\x1c\x76\xa8\xc4\x0f\xb1\x83\x21\x97\x73", 0, + 128, NULL, 0 }, + { 192, 128, 191, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, NULL, 0 }, + { 192, 128, 192, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, NULL, 0 }, + { 192, 128, 193, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, NULL, 0 }, + { 192, 128, 194, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, NULL, 0 }, + { 192, 128, 195, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, NULL, 0 }, + { 192, 128, 196, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, NULL, 0 }, + { 192, 128, 197, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\x6c\x92\xb9\x8e\x20\x27\x6d\x95\x59\x53\xfa\xee\x4a\x9f\x49\x10", 0, + 0, NULL, 0 }, + { 192, 128, 198, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x49\x93\x87\x5f\xe0\x05\x1a\x4b\x11\x9c\xfb\x66\x9b\x67\x4a\x10", 0, + 64, NULL, 0 }, + { 192, 128, 199, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x78\x9b\x5e\x80\xe9\x26\x63\x09\xd7\xbb\x70\xce\xfc\x5e\xe8\x0c", 0, + 128, NULL, 0 }, + { 192, 128, 200, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "", + "\xed\x13\x38\x0f\xa1\xa6\xec\x14\xd8\xd2\x7b\x6f\xcb\x1e\xc8\x91", 0, + 0, NULL, 0 }, + { 192, 128, 201, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xc8\x12\x06\xde\x61\x84\x9b\xca\x90\x1d\x7a\xe7\x1a\xe6\xcb\x91", 0, + 64, NULL, 0 }, + { 192, 128, 202, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf9\x1a\xdf\x01\x68\xa7\xe2\x88\x56\x3a\xf1\x4f\x7d\xdf\x69\x8d", 0, + 128, NULL, 0 }, + { 256, 128, 203, + "\x7b\xf9\xe5\x36\xb6\x6a\x21\x5c\x22\x23\x3f\xe2\xda\xaa\x74\x3a\x89" + "\x8b\x9a\xcb\x9f\x78\x02\xde\x70\xb4\x0e\x3d\x6e\x43\xef\x97", + "", + "\x73\x6c\x7b\x56\x95\x7d\xb7\x74\xc5\xdd\xf7\xc7\xa7\x0b\xa8\xa8", 1, + 0, NULL, 0 }, + { 256, 128, 204, + "\xe7\x54\x07\x6c\xea\xb3\xfd\xaf\x4f\x9b\xca\xb7\xd4\xf0\xdf\x0c\xbb" + "\xaf\xbc\x87\x73\x1b\x8f\x9b\x7c\xd2\x16\x64\x72\xe8\xee\xbc", + "\x40", + "\x9d\x47\x48\x2c\x2d\x92\x52\xba\xce\x43\xa7\x5a\x83\x35\xb8\xb8", 1, + 8, NULL, 0 }, + { 256, 128, 205, + "\xea\x3b\x01\x6b\xdd\x38\x7d\xd6\x4d\x83\x7c\x71\x68\x38\x08\xf3\x35" + "\xdb\xdc\x53\x59\x8a\x4e\xa8\xc5\xf9\x52\x47\x3f\xaf\xaf\x5f", + "\x66\x01", + "\xc7\xc4\x4e\x31\xc4\x66\x33\x49\x92\xd6\xf9\xde\x3c\x77\x16\x34", 1, + 16, NULL, 0 }, + { 256, 128, 206, + "\x73\xd4\x70\x96\x37\x85\x7d\xaf\xab\x6a\xd8\xb2\xb0\xa5\x1b\x06\x52" + "\x47\x17\xfe\xdf\x10\x02\x96\x64\x4f\x7c\xfd\xaa\xe1\x80\x5b", + "\xf1\xd3\x00", + "\xb7\x08\x66\x03\xa8\x5e\x11\xfc\xeb\x8c\xad\xea\x9b\xd3\x09\x39", 1, + 24, NULL, 0 }, + { 256, 128, 207, + "\xd5\xc8\x1b\x39\x9d\x4c\x0d\x15\x83\xa1\x3d\xa5\x6d\xe6\xd2\xdc\x45" + "\xa6\x6e\x7b\x47\xc2\x4a\xb1\x19\x2e\x24\x6d\xc9\x61\xdd\x77", + "\x2a\xe6\x3c\xbf", + "\xba\x38\x3a\x3a\x15\xc9\xdf\x64\xbb\xa5\x0d\x61\x11\x13\xa0\x24", 1, + 32, NULL, 0 }, + { 256, 128, 208, + "\x25\x21\x20\x3f\xa0\xdd\xdf\x59\xd8\x37\xb2\x83\x0f\x87\xb1\xaa\x61" + "\xf9\x58\x15\x5d\xf3\xca\x4d\x1d\xf2\x45\x7c\xb4\x28\x4d\xc8", + "\xaf\x3a\x01\x5e\xa1", + "\xb4\x57\x13\x7c\x54\x89\x08\xc6\x29\xf7\x14\xfe\x83\xb1\xed\x90", 1, + 40, NULL, 0 }, + { 256, 128, 209, + "\x66\x5a\x02\xbc\x26\x5a\x66\xd0\x17\x75\x09\x1d\xa5\x67\x26\xb6\x66" + "\x8b\xfd\x90\x3c\xb7\xaf\x66\xfb\x1b\x78\xa8\xa0\x62\xe4\x3c", + "\x3f\x56\x93\x5d\xef\x3f", + "\xb6\xd6\xfd\xe9\x3f\xc8\x5d\xe2\x89\xb3\x6b\x44\x6d\x77\xb4\x23", 1, + 48, NULL, 0 }, + { 256, 128, 210, + "\xfa\xcd\x75\xb2\x22\x21\x38\x00\x47\x30\x5b\xc9\x81\xf5\x70\xe2\xa1" + "\xaf\x38\x92\x8e\xa7\xe2\x05\x9e\x3a\xf5\xfc\x6b\x82\xb4\x93", + "\x57\xbb\x86\xbe\xed\x15\x6f", + "\x8b\x1e\xf7\x2d\x0a\x61\x27\x35\xb0\x8e\xfe\xf9\x81\xf2\x13\xc2", 1, + 56, NULL, 0 }, + { 256, 128, 211, + "\x50\x5a\xa9\x88\x19\x80\x9e\xf6\x3b\x9a\x36\x8a\x1e\x8b\xc2\xe9\x22" + "\xda\x45\xb0\x3c\xe0\x2d\x9a\x79\x66\xb1\x50\x06\xdb\xa2\xd5", + "\x2e\x4e\x7e\xf7\x28\xfe\x11\xaf", + "\xf7\x96\x06\xb8\x3a\x77\x06\xa2\xa1\x9e\x06\x8b\xce\x81\x88\x98", 1, + 64, NULL, 0 }, + { 256, 128, 212, + "\xf9\x42\x09\x38\x42\x80\x8b\xa4\x7f\x64\xe4\x27\xf7\x35\x1d\xde\x6b" + "\x95\x46\xe6\x6d\xe4\xe7\xd6\x0a\xa6\xf3\x28\x18\x27\x12\xcf", + "\x85\x2a\x21\xd9\x28\x48\xe6\x27\xc7", + "\xa5\xa8\x77\xf2\x2a\xc7\x43\xb7\xfb\x9e\x05\x0d\x2e\x3d\xdb\x02", 1, + 72, NULL, 0 }, + { 256, 128, 213, + "\x64\xbe\x16\x2b\x39\xc6\xe5\xf1\xfe\xd9\xc3\x2d\x9f\x67\x4d\x9a\x8c" + "\xde\x6e\xaa\x24\x43\x21\x4d\x86\xbd\x4a\x1f\xb5\x3b\x81\xb4", + "\x19\x5a\x3b\x29\x2f\x93\xba\xff\x0a\x2c", + "\x6e\xa1\x72\xe5\xc4\xd2\xfa\xc0\x75\xca\x60\x2d\xe5\x75\x7a\x62", 1, + 80, NULL, 0 }, + { 256, 128, 214, + "\xb2\x59\xa5\x55\xd4\x4b\x8a\x20\xc5\x48\x9e\x2f\x38\x39\x2d\xda\xa6" + "\xbe\x9e\x35\xb9\x83\x3b\x67\xe1\xb5\xfd\xf6\xcb\x3e\x4c\x6c", + "\xaf\xd7\x31\x17\x33\x0c\x6e\x85\x28\xa6\xe4", + "\x68\x02\x0b\xfc\x9b\xd7\x3f\xd8\x0d\x3c\xe5\x81\xba\x3b\x12\x08", 1, + 88, NULL, 0 }, + { 256, 128, 215, + "\x2c\x6f\xc6\x2d\xaa\x77\xba\x8c\x68\x81\xb3\xdd\x69\x89\x89\x8f\xef" + "\x64\x66\x63\xcc\x7b\x0a\x3d\xb8\x22\x8a\x70\x7b\x85\xf2\xdc", + "\x0f\xf5\x4d\x6b\x67\x59\x12\x0c\x2e\x8a\x51\xe3", + "\x11\x0e\xdd\x72\x7a\x9b\xf7\xfa\x11\xa6\x35\x8a\xfe\x61\x7d\x9d", 1, + 96, NULL, 0 }, + { 256, 128, 216, + "\xab\xab\x81\x5d\x51\xdf\x29\xf7\x40\xe4\xe2\x07\x9f\xb7\x98\xe0\x15" + "\x28\x36\xe6\xab\x57\xd1\x53\x6a\xe8\x92\x9e\x52\xc0\x6e\xb8", + "\xf0\x05\x8d\x41\x2a\x10\x4e\x53\xd8\x20\xb9\x5a\x7f", + "\x1f\xa2\x4c\x66\x25\xa0\xf8\xe1\xfc\x37\x82\x7a\xc8\x4d\x3c\xc4", 1, + 104, NULL, 0 }, + { 256, 128, 217, + "\x3d\x5d\xa1\xaf\x83\xf7\x28\x74\x58\xbf\xf7\xa7\x65\x1e\xa5\xd8\xdb" + "\x72\x25\x94\x01\x33\x3f\x6b\x82\x09\x69\x96\xdd\x7e\xaf\x19", + "\xaa\xcc\x36\x97\x2f\x18\x30\x57\x91\x9f\xf5\x7b\x49\xe1", + "\x86\x87\x65\xa8\xfa\x6a\xa8\x98\xdd\xec\x0f\x41\x23\xe9\x96\xbe", 1, + 112, NULL, 0 }, + { 256, 128, 218, + "\xc1\x9b\xdf\x31\x4c\x6c\xf6\x43\x81\x42\x54\x67\xf4\x2a\xef\xa1\x7c" + "\x1c\xc9\x35\x8b\xe1\x6c\xe3\x1b\x1d\x21\x48\x59\xce\x86\xaa", + "\x5d\x06\x6a\x92\xc3\x00\xe9\xb6\xdd\xd6\x3a\x7c\x13\xae\x33", + "\xb9\x68\x18\xb7\xac\xaf\x87\x9c\x7a\x7f\x82\x71\x37\x5a\x69\x14", 1, + 120, NULL, 0 }, + { 256, 128, 219, + "\x61\x2e\x83\x78\x43\xce\xae\x7f\x61\xd4\x96\x25\xfa\xa7\xe7\x49\x4f" + "\x92\x53\xe2\x0c\xb3\xad\xce\xa6\x86\x51\x2b\x04\x39\x36\xcd", + "\xcc\x37\xfa\xe1\x5f\x74\x5a\x2f\x40\xe2\xc8\xb1\x92\xf2\xb3\x8d", + "\x4b\x88\xe1\x93\x00\x0c\x5a\x4b\x23\xe9\x5c\x7f\x2b\x26\x53\x0b", 1, + 128, NULL, 0 }, + { 256, 128, 220, + "\x73\x21\x6f\xaf\xd0\x02\x2d\x0d\x6e\xe2\x71\x98\xb2\x27\x25\x78\xfa" + "\x8f\x04\xdd\x9f\x44\x46\x7f\xbb\x64\x37\xaa\x45\x64\x1b\xf7", + "\xd5\x24\x7b\x8f\x6c\x3e\xdc\xbf\xb1\xd5\x91\xd1\x3e\xce\x23\xd2" + "\xf5", + "\x86\x91\x1c\x7d\xa5\x1d\xc0\x82\x3d\x6e\x93\xd4\x29\x0d\x1a\xd4", 1, + 136, NULL, 0 }, + { 256, 128, 221, + "\x04\x27\xa7\x0e\x25\x75\x28\xf3\xab\x70\x64\x0b\xba\x1a\x5d\xe1\x2c" + "\xf3\x88\x5d\xd4\xc8\xe2\x84\xfb\xbb\x55\xfe\xb3\x52\x94\xa5", + "\x13\x93\x7f\x85\x44\xf4\x42\x70\xd0\x11\x75\xa0\x11\xf7\x67\x0e\x93" + "\xfa\x6b\xa7\xef\x02\x33\x6e", + "\xcc\xb2\xc5\x1b\xfb\xe2\x59\x8f\x91\x09\xfc\x70\xed\x07\xf0\xeb", 1, + 192, NULL, 0 }, + { 256, 128, 222, + "\x96\xe1\xe4\x89\x6f\xb2\xcd\x05\xf1\x33\xa6\xa1\x00\xbc\x56\x09\xa7" + "\xac\x3c\xa6\xd8\x17\x21\xe9\x22\xda\xdd\x69\xad\x07\xa8\x92", + "\x91\xa1\x7e\x4d\xfc\xc3\x16\x6a\x1a\xdd\x26\xff\x0e\x7c\x12\x05\x6e" + "\x8a\x65\x4f\x28\xa6\xde\x24\xf4\xba\x73\x9c\xeb\x5b\x5b\x18", + "\x92\x5f\x17\x7d\x85\xea\x29\x7e\xf1\x4b\x20\x3f\xe4\x09\xf9\xab", 1, + 256, NULL, 0 }, + { 256, 128, 223, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6a\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 224, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd7\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 225, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 226, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x69\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 227, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd4\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 228, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x5b\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 229, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xeb\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 230, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x56\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 231, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 232, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf1\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 233, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x08\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 234, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xef\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 235, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x13\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 236, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\xfc\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 237, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\xbb\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 238, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd9\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 239, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3b\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 240, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5e\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 241, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xda\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 242, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x38\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 243, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5d\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 244, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x90\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 245, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\x22\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 246, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x10\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 247, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1e\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 248, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xeb\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 249, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xcb\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 250, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x9f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 251, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\x6a\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 252, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\x4a\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 253, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x20\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 254, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x00\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 255, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xc2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 256, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x88\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 257, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0a\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 258, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6c\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 259, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x77\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 260, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7c\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 261, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x28\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 262, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x74\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 263, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7f\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 264, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x2b\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 265, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\xf6\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 266, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\xfd\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 267, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\xa9\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 268, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xfa", 0, + 0, NULL, 0 }, + { 256, 128, 269, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xed", 0, + 64, NULL, 0 }, + { 256, 128, 270, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2c", 0, + 128, NULL, 0 }, + { 256, 128, 271, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xf9", 0, + 0, NULL, 0 }, + { 256, 128, 272, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xee", 0, + 64, NULL, 0 }, + { 256, 128, 273, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x2f", 0, + 128, NULL, 0 }, + { 256, 128, 274, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\xbb", 0, + 0, NULL, 0 }, + { 256, 128, 275, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\xac", 0, + 64, NULL, 0 }, + { 256, 128, 276, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\x6d", 0, + 128, NULL, 0 }, + { 256, 128, 277, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x10\x1f\x00\x89\x72\x76\x91\xb7\x7b", 0, + 0, NULL, 0 }, + { 256, 128, 278, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\xa2\xea\x20\x0b\x29\x7d\x2a\xcc\x6c", 0, + 64, NULL, 0 }, + { 256, 128, 279, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x90\xca\xe2\x6d\xad\x29\xbb\xa3\xad", 0, + 128, NULL, 0 }, + { 256, 128, 280, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6a\xf0\xa2\x93\xd8\xcb\xa0\x10\x1e\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 281, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd7\x09\x71\x7c\x3a\x4e\xf8\xa2\xeb\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 282, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\xee\x3f\x3b\x5f\x83\xe2\x90\xcb\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 283, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x13\xd8\xcb\xa0\x90\x1f\x00\x89\x72\x76\x91\xb7\xfb", 0, + 0, NULL, 0 }, + { 256, 128, 284, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\xfc\x3a\x4e\xf8\x22\xea\x20\x0b\x29\x7d\x2a\xcc\xec", 0, + 64, NULL, 0 }, + { 256, 128, 285, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\xbb\x5f\x83\xe2\x10\xca\xe2\x6d\xad\x29\xbb\xa3\x2d", 0, + 128, NULL, 0 }, + { 256, 128, 286, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6b\xf0\xa2\x93\xd8\xcb\xa0\x90\x1f\x00\x89\x72\x76\x91\xb7\x7b", 0, + 0, NULL, 0 }, + { 256, 128, 287, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd6\x09\x71\x7c\x3a\x4e\xf8\x22\xea\x20\x0b\x29\x7d\x2a\xcc\x6c", 0, + 64, NULL, 0 }, + { 256, 128, 288, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x59\xee\x3f\x3b\x5f\x83\xe2\x10\xca\xe2\x6d\xad\x29\xbb\xa3\xad", 0, + 128, NULL, 0 }, + { 256, 128, 289, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x94\x0f\x5d\x6c\x27\x34\x5f\xef\xe0\xff\x76\x8d\x89\x6e\x48\x04", 0, + 0, NULL, 0 }, + { 256, 128, 290, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x29\xf6\x8e\x83\xc5\xb1\x07\x5d\x15\xdf\xf4\xd6\x82\xd5\x33\x13", 0, + 64, NULL, 0 }, + { 256, 128, 291, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa6\x11\xc0\xc4\xa0\x7c\x1d\x6f\x35\x1d\x92\x52\xd6\x44\x5c\xd2", 0, + 128, NULL, 0 }, + { 256, 128, 292, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, NULL, 0 }, + { 256, 128, 293, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, NULL, 0 }, + { 256, 128, 294, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, NULL, 0 }, + { 256, 128, 295, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, NULL, 0 }, + { 256, 128, 296, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, NULL, 0 }, + { 256, 128, 297, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, NULL, 0 }, + { 256, 128, 298, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xeb\x70\x22\x13\x58\x4b\x20\x90\x9f\x80\x09\xf2\xf6\x11\x37\x7b", 0, + 0, NULL, 0 }, + { 256, 128, 299, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x56\x89\xf1\xfc\xba\xce\x78\x22\x6a\xa0\x8b\xa9\xfd\xaa\x4c\x6c", 0, + 64, NULL, 0 }, + { 256, 128, 300, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\x6e\xbf\xbb\xdf\x03\x62\x10\x4a\x62\xed\x2d\xa9\x3b\x23\xad", 0, + 128, NULL, 0 }, + { 256, 128, 301, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x6a\xf1\xa3\x92\xd9\xca\xa1\x11\x1e\x01\x88\x73\x77\x90\xb6\xfa", 0, + 0, NULL, 0 }, + { 256, 128, 302, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xd7\x08\x70\x7d\x3b\x4f\xf9\xa3\xeb\x21\x0a\x28\x7c\x2b\xcd\xed", 0, + 64, NULL, 0 }, + { 256, 128, 303, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\xef\x3e\x3a\x5e\x82\xe3\x91\xcb\xe3\x6c\xac\x28\xba\xa2\x2c", 0, + 128, NULL, 0 }, + { 0, 128, 304, "", "\x00\xb9\x44\x93\x26\xd3\x94\x16", "", 0, 64, NULL, + 0 }, + { 8, 128, 305, "\x0f", "\x45\x38\xb7\x9a\x13\x97\xe2\xaa", "", 0, 64, + NULL, 0 }, + { 64, 128, 306, "\xa8\x8e\x38\x5a\xf7\x18\x51\x48", + "\xdc\x63\xb7\xef\x08\x09\x6e\x4f", "", 0, 64, NULL, 0 }, + { 160, 128, 307, + "\x00\x3a\x22\x80\x08\xd3\x90\xb6\x45\x92\x9d\xf7\x3a\x2b\x2b\xdd\x82" + "\x98\x91\x8d", + "\xad\x1d\x3c\x31\x22\xab\x7a\xc6", "", 0, 64, NULL, 0 }, + { 320, 128, 308, + "\x94\xba\xaa\xc1\x50\xe2\x64\x5a\xe1\xec\x19\x39\xc7\xbc\xef\xb7\x3f" + "\x6e\xdb\x14\x6f\xae\x02\x28\x9b\x6c\x63\x26\xff\x39\xbc\x26\x5d\x61" + "\x2b\xef\x27\x27\xfa\x72", + "\xe3\xf7\x5a\x88\x6c\x4a\x55\x91", "", 0, 64, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/aes_gcm_test.json.c b/test/wycheproof/aes_gcm_test.json.c new file mode 100644 index 00000000..bfa3a3d1 --- /dev/null +++ b/test/wycheproof/aes_gcm_test.json.c @@ -0,0 +1,2528 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* AES-GCM, 0.8r12 */ +#include "aead_test.h" +const struct aead_test aes_gcm_test_json[] = { + { 96, 128, 128, 1, + "\x5b\x96\x04\xfe\x14\xea\xdb\xa9\x31\xb0\xcc\xf3\x48\x43\xda\xb9", + "\x02\x83\x18\xab\xc1\x82\x40\x29\x13\x81\x41\xa2", "", + "\x00\x1d\x0c\x23\x12\x87\xc1\x18\x27\x84\x55\x4c\xa3\xa2\x19\x08", + "\x26\x07\x3c\xc1\xd8\x51\xbe\xff\x17\x63\x84\xdc\x98\x96\xd5\xff", + "\x0a\x3e\xa7\xa5\x48\x7c\xb5\xf7\xd7\x0f\xb6\xc5\x8d\x03\x85\x54", 1, + 0, 128 }, + { 96, 128, 128, 2, + "\x5b\x96\x04\xfe\x14\xea\xdb\xa9\x31\xb0\xcc\xf3\x48\x43\xda\xb9", + "\x92\x1d\x25\x07\xfa\x80\x07\xb7\xbd\x06\x7d\x34", + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x00\x1d\x0c\x23\x12\x87\xc1\x18\x27\x84\x55\x4c\xa3\xa2\x19\x08", + "\x49\xd8\xb9\x78\x3e\x91\x19\x13\xd8\x70\x94\xd1\xf6\x3c\xc7\x65", + "\x1e\x34\x8b\xa0\x7c\xca\x2c\xf0\x4c\x61\x8c\xb4\xd4\x3a\x5b\x92", 1, + 128, 128 }, + { 96, 128, 128, 3, + "\xaa\x02\x3d\x04\x78\xdc\xb2\xb2\x31\x24\x98\x29\x3d\x9a\x91\x29", + "\x04\x32\xbc\x49\xac\x34\x41\x20\x81\x28\x81\x27", + "\xaa\xc3\x92\x31\x12\x98\x72\xa2", + "\x20\x35\xaf\x31\x3d\x13\x46\xab\x00\x15\x4f\xea\x78\x32\x21\x05", + "\xee\xa9\x45\xf3\xd0\xf9\x8c\xc0\xfb\xab\x47\x2a\x0c\xf2\x4e\x87", + "\x4b\xb9\xb4\x81\x25\x19\xda\xdf\x9e\x12\x32\x01\x6d\x06\x81\x33", 1, + 64, 128 }, + { 96, 128, 128, 4, + "\xbe\xdc\xfb\x5a\x01\x1e\xbc\x84\x60\x0f\xcb\x29\x6c\x15\xaf\x0d", + "\x43\x8a\x54\x7a\x94\xea\x88\xdc\xe4\x6c\x6c\x85", "", "", "", + "\x96\x02\x47\xba\x5c\xde\x02\xe4\x1a\x31\x3c\x4c\x01\x36\xed\xc3", 1, + 0, 0 }, + { 96, 128, 128, 5, + "\x38\x4e\xa4\x16\xac\x3c\x2f\x51\xa7\x6e\x7d\x82\x26\x34\x6d\x4e", + "\xb3\x0c\x08\x47\x27\xad\x1c\x59\x2a\xc2\x1d\x12", "", "\x35", + "\x54", + "\x7c\x1e\x4a\xe8\x8b\xb2\x7e\x56\x38\x34\x3c\xb9\xfd\x3f\x63\x37", 1, + 0, 8 }, + { 96, 128, 128, 6, + "\xca\xe3\x1c\xd9\xf5\x55\x26\xeb\x03\x82\x41\xfc\x44\xca\xc1\xe5", + "\xb5\xe0\x06\xde\xd5\x53\x11\x0e\x6d\xc5\x65\x29", "", + "\xd1\x09\x89\xf2\xc5\x2e\x94\xad", + "\xa0\x36\xea\xd0\x31\x93\x90\x3f", + "\x3b\x62\x69\x40\xe0\xe9\xf0\xcb\xea\x8e\x18\xc4\x37\xfd\x60\x11", 1, + 0, 64 }, + { 96, 128, 128, 7, + "\xdd\x61\x97\xcd\x63\xc9\x63\x91\x9c\xf0\xc2\x73\xef\x6b\x28\xbf", + "\xec\xb0\xc4\x2f\x70\x00\xef\x0e\x6f\x95\xf2\x4d", "", + "\x4d\xcc\x14\x85\x36\x58\x66\xe2\x5a\xc3\xf2\xca\x6a\xba\x97", + "\x8a\x99\x92\x38\x8e\x73\x5f\x80\xee\x18\xf4\xa6\x3c\x10\xad", + "\x14\x86\xa9\x1c\xcc\xf9\x2c\x9a\x5b\x00\xf7\xb0\xe0\x34\x89\x1c", 1, + 0, 120 }, + { 96, 128, 128, 8, + "\xff\xdf\x42\x28\x36\x1e\xa1\xf8\x16\x58\x52\x13\x6b\x34\x80\xf7", + "\x0e\x16\x66\xf2\xdc\x65\x2f\x77\x08\xfb\x8f\x0d", "", + "\x25\xb1\x2e\x28\xac\x0e\xf6\xea\xd0\x22\x6a\x3b\x22\x88\xc8\x00", + "\xf7\xbd\x37\x9d\x13\x04\x77\x17\x6b\x8b\xb3\xcb\x23\xdb\xbb\xaa", + "\x1e\xe6\x51\x3c\xe3\x0c\x78\x73\xf5\x9d\xd4\x35\x0a\x58\x8f\x42", 1, + 0, 128 }, + { 96, 128, 128, 9, + "\xc1\x5e\xd2\x27\xdd\x2e\x23\x7e\xcd\x08\x7e\xaa\xaa\xd1\x9e\xa4", + "\x96\x5f\xf6\x64\x31\x16\xac\x14\x43\xa2\xde\xc7", "", + "\xfe\xe6\x2f\xde\x97\x3f\xe0\x25\xad\x6b\x32\x2d\xcd\xf3\xc6\x3f" + "\xc7", + "\x0d\xe5\x1f\xe4\xf7\xf2\xd1\xf0\xf9\x17\x56\x9f\x5c\x6d\x1b\x00" + "\x9c", + "\x6c\xd8\x52\x14\x22\xc0\x17\x7e\x83\xef\x1b\x7a\x84\x5d\x97\xdb", 1, + 0, 136 }, + { 96, 128, 128, 10, + "\xa8\xee\x11\xb2\x6d\x7c\xeb\x7f\x17\xea\xa1\xe4\xb8\x3a\x2c\xf6", + "\xfb\xbc\x04\xfd\x6e\x02\x5b\x71\x93\xeb\x57\xf6", "", + "\xc0\x8f\x08\x5e\x6a\x9e\x0e\xf3\x63\x62\x80\xc1\x1e\xcf\xad\xf0\xc1" + "\xe7\x29\x19\xff\xc1\x7e\xaf", + "\x7c\xd9\xf4\xe4\xf3\x65\x70\x4f\xff\x3b\x99\x00\xaa\x93\xba\x54\xb6" + "\x72\xba\xc5\x54\x27\x56\x50", + "\xf4\xeb\x19\x32\x41\x22\x6d\xb0\x17\xb3\x2e\xc3\x8c\xa4\x72\x17", 1, + 0, 192 }, + { 96, 128, 128, 11, + "\x28\xff\x3d\xef\x08\x17\x93\x11\xe2\x73\x4c\x6d\x1c\x4e\x28\x71", + "\x32\xbc\xb9\xb5\x69\xe3\xb8\x52\xd3\x7c\x76\x6a", "\xc3", + "\xdf\xc6\x1a\x20\xdf\x85\x05\xb5\x3e\x3c\xd5\x9f\x25\x77\x0d\x50\x18" + "\xad\xd3\xd6", + "\xf5\x8d\x45\x32\x12\xc2\xc8\xa4\x36\xe9\x28\x36\x72\xf5\x79\xf1\x19" + "\x12\x29\x78", + "\x59\x01\x13\x1d\x07\x60\xc8\x71\x59\x01\xd8\x81\xfd\xfd\x3b\xc0", 1, + 8, 160 }, + { 96, 128, 128, 12, + "\xe6\x3a\x43\x21\x6c\x08\x86\x72\x10\xe2\x48\x85\x9e\xb5\xe9\x9c", + "\x9c\x3a\x42\x63\xd9\x83\x45\x66\x58\xaa\xd4\xb1", + "\x83\x4a\xfd\xc5\xc7\x37\x18\x6b", + "\xb1\x4d\xa5\x6b\x04\x62\xdc\x05\xb8\x71\xfc\x81\x52\x73\xff\x48\x10" + "\xf9\x2f\x4b", + "\xbf\x86\x46\x16\xc2\x34\x75\x09\xca\x9b\x10\x44\x63\x79\xb9\xbd\xbb" + "\x3b\x8f\x64", + "\xa9\x7d\x25\xb4\x90\x39\x0b\x53\xc5\xdb\x91\xf6\xee\x2a\x15\xb8", 1, + 64, 160 }, + { 96, 128, 128, 13, + "\x38\x44\x98\x90\x23\x4e\xb8\xaf\xab\x0b\xbf\x82\xe2\x38\x54\x54", + "\x33\xe9\x06\x58\x41\x6e\x7c\x1a\x7c\x00\x5f\x11", + "\x40\x20\x85\x5c\x66\xac\x45\x95\x05\x83\x95\xf3\x67\x20\x1c\x4c", + "\xf7\x62\x77\x6b\xf8\x31\x63\xb3\x23\xca\x63\xa6\xb3\xad\xea\xc1\xe1" + "\x35\x72\x62", + "\xa6\xf2\xef\x3c\x7e\xf7\x4a\x12\x6d\xd2\xd5\xf6\x67\x39\x64\xe2\x7d" + "\x5b\x34\xb6", + "\xb8\xbb\xdc\x4f\x50\x14\xbc\x75\x2c\x8b\x4e\x9b\x87\xf6\x50\xa3", 1, + 128, 160 }, + { 96, 128, 128, 14, + "\x6a\x68\x67\x1d\xfe\x32\x3d\x41\x98\x94\x38\x1f\x85\xeb\x63\xfd", + "\x9f\x0d\x85\xb6\x05\x71\x1f\x34\xcd\x2a\x35\xba", + "\x76\xeb\x5f\x14\x72\x50\xfa\x3c\x12\xbf\xf0\xa6\xe3\x93\x4a\x0b\x16" + "\x86\x0c\xf1\x16\x46\x77\x3b", + "\x0f\xc6\x78\x99\xc3\xf1\xbb\xe1\x96\xd9\x0f\x1e\xca\x37\x97\x38\x92" + "\x30\xaa\x37", + "\xbd\x64\x80\x2c\xfe\xba\xeb\x48\x7d\x3a\x8f\x76\xce\x94\x3a\x37\xb3" + "\x47\x2d\xd5", + "\xfc\xe9\xa5\xb5\x30\xc7\xd7\xaf\x71\x8b\xe1\xec\x0a\xe9\xed\x4d", 1, + 192, 160 }, + { 96, 128, 128, 15, + "\xe1\x22\x60\xfc\xd3\x55\xa5\x1a\x0d\x01\xbb\x1f\x6f\xa5\x38\xc2", + "\x5d\xfc\x37\x36\x6f\x56\x88\x27\x51\x47\xd3\xf9", "", + "\xd9\x02\xde\xea\xb1\x75\xc0\x08\x32\x9a\x33\xbf\xac\xcd\x5c\x0e\xb3" + "\xa6\xa1\x52\xa1\x51\x0e\x7d\xb0\x4f\xa0\xaf\xf7\xce\x42\x88\x53\x0d" + "\xb6\xa8\x0f\xa7\xfe\xa5\x82\xaa\x7d\x46\xd7\xd5\x6e\x70\x8d\x2b\xb0" + "\xc5\xed\xd3\xd2\x66\x48\xd3\x36\xc3\x62\x0e\xa5\x5e", + "\xd3\x3b\xf6\x72\x2f\xc2\x93\x84\xfa\xd7\x5f\x99\x02\x48\xb9\x52\x8e" + "\x09\x59\xaa\x67\xec\x66\x86\x9d\xc3\x99\x6c\x67\xa2\xd5\x59\xe7\xd7" + "\x7c\xe5\x95\x5f\x8c\xad\x2a\x4d\xf5\xfd\xc3\xac\xcc\xaf\xa7\xbc\x0d" + "\xef\x53\xd8\x48\x11\x12\x56\x90\x3e\x5a\xdd\x04\x20", + "\x8b\xc8\x33\xde\x51\x08\x63\xb4\xb4\x32\xc3\xcb\xf4\x5a\xa7\xcc", 1, + 0, 512 }, + { 96, 128, 128, 16, + "\x3c\x55\xf8\x8e\x9f\xaa\x0d\x68\xab\x50\xd0\x2b\x47\x16\x12\x76", + "\xd7\x67\xc4\x8d\x20\x37\xb4\xbd\x2c\x23\x1b\xbd", "", + "\x5d\x6a\xdd\x48\xe7\xa5\x70\x4e\x54\xf9\xc2\x82\x9a\x9b\x42\x83\xdc" + "\xe0\xd3\xa6\x5b\x13\x3e\xba\x37\x93\xc4\xfb\xfa\x1d\x8e\x3a\x25\x39" + "\xd0\xd4\xf3\xde\x38\x15\x98\xce\x5b\x23\x60\x17\x3f\xbd\x14\x94\x76" + "\xc3\x16\x92\xc5\xd6\xe8\x72\xfc\xe4\x02\x19\x37\x89\x49\xc2\xe7\x0b" + "\x5f\x1b\x9f\x0a\x1d\x5f\x38\x35\x2a\xd8\x14\xb2\xa0\x35\xbb\x3f\x3f" + "\x26\x42\x5d\x83\x1a\x2f\x7a\x5e\x65\xc5\xdf\xcd\x91\xa3\x15\xc2\xb2" + "\x4f\x53\xa6\x62\x60\x5e\xa4\x08\x57\xdd\x98\x0e\x9b\xe5\xcd\xad\x00" + "\x0c\x56\x9f\x2d\x20\x4d\x4b\xd3\xb0", + "\x17\xd7\x2d\x90\xbd\x23\xe0\x76\xd8\x36\x4a\x87\xec\xb9\xac\x58\xac" + "\xc5\xde\x46\x29\xbf\xd5\x90\x40\x9b\x8b\xf1\xfc\xd3\xa2\xf6\x02\x73" + "\x1b\x46\x14\xce\xc1\x5e\x77\x3e\xa6\x5a\x65\xe7\x21\x09\x94\x25\x6b" + "\xf5\x45\x0a\x25\xac\xb5\x27\x26\x9c\x06\x5f\x2e\x2f\x22\x79\xd1\xfe" + "\x8b\x3e\xda\x98\xdc\xf8\x7b\x34\x8f\x15\x28\x37\x7b\xbd\xd2\x58\x35" + "\x5d\x46\xe0\x35\x33\x04\x83\xd8\x09\x7e\x80\xc7\xde\x9b\xbb\x60\x6d" + "\xdf\x72\x3f\x29\x09\x21\x7f\xfd\xd1\x8e\x8b\xdb\xd7\xb0\x80\x62\xf1" + "\xdc\xba\x96\x0e\x5c\x0d\x29\x0f\x5f", + "\x09\x0b\x8c\x2e\xc9\x8e\x41\x16\x18\x6d\x0e\x5f\xbe\xfe\xb9\xc2", 1, + 0, 1024 }, + { 96, 128, 128, 17, + "\xa2\x94\xe7\x0f\xa2\xac\x10\xa1\xfb\x00\xc5\x88\xb8\x88\xb6\x73", + "\xdf\xe2\x0d\x1c\x43\x50\xe6\x23\x5d\x98\x7a\xf1", "", + "\x6e\xd1\xd7\xd6\x18\xd1\x58\x74\x1f\x52\x07\x80\x06\xf2\x84\x94\xba" + "\x72\xa2\x45\x4f\x27\x16\x0a\xe8\x72\x27\x93\xfc\xeb\xc5\x38\xeb\xc2" + "\xf6\x7c\x3a\xce\x3e\x0f\xe7\xc4\x7b\x9e\x74\xe0\x81\x18\x2b\x47\xc9" + "\x30\x14\x4e\x3f\xc8\x0d\x0a\xd5\x06\x11\xc3\xaf\xcf\xe2\xdb\xc5\x27" + "\x9e\xdb\xbb\xa0\x87\xc0\xe3\x90\x35\x5f\x3d\xaf\xfc\xd2\x5a\xd4\xde" + "\xa0\x07\xc2\x84\xad\x92\xe7\xfc\xbe\xcb\x43\x8f\xb6\x06\x23\xff\x89" + "\xa5\x99\xdc\xa2\xaa\xc1\x41\xb2\x66\x51\x38\x6c\xa5\x5b\x73\x9b\x94" + "\x90\x1e\xf6\xdb\x60\x9c\x34\x4d\x8a\xcf\x45\x44\x56\x8e\x31\xbb\x09" + "\x36\x11\x12\x75\x4b\x1c\x0c\x6a\x3c\x87\x5b\xd9\x45\x3b\x0e\xe0\x08" + "\x14\x12\x15\x13\x98\xa2\x94\xec\xad\x75\xad\xd5\x21\x61\x1d\xb5\x28" + "\x8b\x60\xac\x3c\x01\x28\xf6\xe9\x43\x66\xb6\x9e\x65\x9e\x6a\xa6\x6f" + "\x05\x8a\x3a\x35\x71\x06\x4e\xdb\xb0\xf0\x5c\x11\xe5\xdd\xe9\x38\xfb" + "\x46\xc3\x93\x5d\xd5\x19\x3a\x4e\x56\x64\x68\x8f\x0a\xe6\x7c\x29\xb7" + "\xcc\x49\xa7\x96\x31\x40\xf8\x2e\x31\x1a\x20\xc9\x8c\xd3\x4f\xbc\xab" + "\x7b\x4b\x51\x5a\xe8\x65\x57\xe6\x20\x99\xe3\xfc\x37\xb9\x59\x5c\x85" + "\xa7\x5c", + "\x5b\xc6\xdb\xaf\xc4\x01\x10\x1c\x7a\x08\xc8\x1d\x6c\x27\x91\xaa\x14" + "\x7c\xe0\x93\xaa\xd1\x72\xbe\x18\x37\x9c\x74\x73\x84\xa5\x4a\x41\xa7" + "\x47\xba\x95\x5c\xad\xe8\xfd\xfb\x89\x67\xaa\x80\x8b\x43\xfe\xe3\xd7" + "\x57\xcc\x80\xf1\x11\x63\xb8\x00\xe5\xe5\x9d\xf9\x32\x75\x7f\x76\xc4" + "\x0b\x3d\x9c\xba\x44\x9a\xaf\x11\xe4\xf8\x0e\x00\x3b\x1f\x38\x4e\xaf" + "\xa4\xf7\x6e\x81\xb1\x3c\x09\xec\x1a\xd8\x8e\x76\x50\xc7\x50\xd4\x42" + "\xfe\x46\xd2\x25\xa3\x73\xe8\xa1\xb5\x64\xb4\x91\x5a\x5c\x6c\x51\x3c" + "\xfd\xfa\x22\xd9\x29\xd5\x74\x1c\xa5\xeb\xef\xae\xdc\xba\x63\x6c\x7c" + "\x3b\xbe\xf1\x88\x63\xfd\xc1\x26\xb4\xb4\x51\x61\x10\x49\xc3\x5d\x81" + "\x4f\xc2\xeb\x7e\x4b\x8f\x1a\x89\x95\xec\xb4\xa3\xc8\x66\x52\xa0\x68" + "\xc0\xb2\xa3\xe1\xc5\x94\x1d\x59\xc2\x10\xb4\x58\xd5\xd5\xd3\xb0\x64" + "\x20\xec\x20\x53\x46\x5c\xcc\xec\xa7\xc2\x0f\x67\x40\x49\x85\x46\x03" + "\x79\xe2\xee\x80\x6a\x46\xe8\x40\x9d\xfa\xb2\xe0\xdd\x67\xea\x3c\xf4" + "\x6d\x5a\xd4\xeb\x78\x75\x68\x27\x35\x8c\x3e\xf1\xfd\xbd\x07\xc3\x38" + "\x34\xf3\xd9\xec\xa3\xff\x13\xb7\x44\xa0\x10\x59\xa6\xc1\x7a\x31\x5a" + "\x8f\xd4", + "\xc7\x58\x7e\x7d\xa4\x1b\xed\x68\x2c\x37\x37\x7e\xa4\x32\x40\x29", 1, + 0, 2056 }, + { 96, 128, 128, 18, + "\xc4\xb0\x34\x35\xb9\x1f\xc5\x2e\x09\xef\xf2\x7e\x4d\xc3\xfb\x42", + "\x50\x46\xe7\xe0\x8f\x07\x47\xe1\xef\xcc\xb0\x9e", + "\x75\xfc\x90\x78\xb4\x88\xe9\x50\x3d\xcb\x56\x8c\x88\x2c\x9e\xec\x24" + "\xd8\x0b\x04\xf0\x95\x8c\x82\xaa\xc8\x48\x4f\x02\x5c\x90\x43\x41\x48" + "\xdb\x8e\x9b\xfe\x29\xc7\xe0\x71\xb7\x97\x45\x7c\xb1\x69\x5a\x5e\x5a" + "\x63\x17\xb8\x36\x90\xba\x05\x38\xfb\x11\xe3\x25\xca", + "\x8e\x88\x7b\x22\x4e\x8b\x89\xc8\x2e\x9a\x64\x1c\xf5\x79\xe6\x87\x9e" + "\x11\x11\xc7", + "\xb6\x78\x68\x12\x57\x4a\x25\x4e\xb4\x3b\x1c\xb1\xd1\x75\x35\x64\xc6" + "\xb5\x20\xe9", + "\xad\x8c\x09\x61\x0d\x50\x8f\x3d\x0f\x03\xcc\x52\x3c\x0d\x5f\xcc", 1, + 512, 160 }, + { 96, 128, 128, 19, + "\x7e\x37\xd5\x6e\x6b\x1d\x01\x72\xd4\x0d\x64\xd6\x11\x1d\xd4\x24", + "\x51\x7c\x55\xc2\xec\x9b\xfe\xa9\x0a\xdd\xc2\xbd", + "\x8e\xd8\xa9\xbe\x4c\x3d\x32\xa5\x09\x84\x34\xee\x5c\x0c\x4f\xc2\x0f" + "\x78\xef\x5e\x25\xed\x8b\x72\xa8\x40\xa4\x63\xe3\x6b\x67\xb8\x81\xe0" + "\x48\xb5\xe4\x9f\x51\x5b\x25\x41\xad\x5c\xe4\xeb\xb3\xa9\x17\xc1\x6b" + "\xcd\xc0\xdc\x3c\xb5\x2b\xb4\xed\x5a\x1d\xff\xcf\x1e\x18\x66\x54\x4e" + "\x8d\xb1\x03\xb2\xad\x99\xc6\xfa\x6e\x7d\xe1\xd8\xb4\x5b\xff\x57\xec" + "\x87\x2f\x1c\xfc\x78\xb0\xe4\x87\x0f\x6f\x20\x0f\xf1\x29\x1c\xae\x03" + "\x3d\xef\xc3\x32\x7b\xa8\x27\x92\xba\x43\x8e\x35\xc4\xbf\xbb\x68\x4f" + "\xec\x5c\xe5\xe3\xae\x16\x7d\x01\xd7", + "\x6a\x7d\xea\x03\xc1\xbb\xa7\x0b\xe8\xc7\x3d\xa4\x7d\x5e\xe0\x6d\x72" + "\xa2\x74\x30", + "\xcf\xb6\x31\x79\x07\x67\xd0\x64\x5d\x8e\xc6\xf2\x3b\xf7\xfa\x8b\x19" + "\xce\x79\xee", + "\xc5\x76\x7d\xda\xa7\x47\x15\x84\x46\x23\x17\x66\xbd\x20\x49\x0c", 1, + 1024, 160 }, + { 96, 128, 128, 20, + "\x30\x76\x74\x14\x08\xf7\x34\xce\x25\xd4\x8f\x98\x2e\x8b\x84\x4b", + "\xa2\x71\x2e\xac\x5e\x06\xd3\xcc\x28\x64\xaa\x8b", + "\x18\x52\x6e\x4e\xfd\x99\x5a\x0b\xf6\x40\x5d\x9f\x90\x67\x25\xc2\x90" + "\x27\x89\x58\xd4\x95\x54\x97\x4d\x8f\xe0\x25\xe7\x86\x0d\xaa\x22\x5c" + "\x12\x85\xb0\x57\x39\x16\xa4\xb6\x74\x1f\x7c\xc2\xe2\x9c\xe4\xe5\x25" + "\xe1\x2f\x43\x6c\xb7\xce\x0a\xd4\x7d\xf3\xd0\xf5\xbd\x80\xfb\x27\xe4" + "\x76\x35\xa4\x98\x5f\xda\xed\xf0\xe8\x21\xf1\xc8\x95\x99\x85\xca\xc4" + "\x9c\x97\xa4\xa0\x24\x38\xd9\x2b\x4a\xfd\x4c\x85\x5d\xcc\x7e\xf4\x1e" + "\xcf\xc3\x68\x66\x33\x4f\xcc\x05\xb2\xbb\x93\xef\x13\xf0\x0c\x5e\xa9" + "\xb9\x21\xe8\xa5\x19\xd7\x7f\x64\x8e\x0e\xfe\x9b\x5a\x62\x30\x5a\x2e" + "\xcf\x7d\x49\x99\x66\x3a\x6d\xdf\xca\x51\x7f\x1f\x36\xf0\x89\x9b\x0b" + "\xde\xf9\xf4\x33\xc4\xbb\x26\x63\xc0\xcc\x1b\xb6\x16\xe7\xd1\x94\x9e" + "\x52\x2b\xec\x85\x48\x5d\x37\x1d\x11\x34\xc9\x0e\xed\xe7\x5e\x86\x5d" + "\xc7\xbe\x40\x5b\x54\xc3\x3f\x0a\xcb\xac\xe6\xcf\x78\x0c\x78\x03\x5b" + "\x80\x35\xb6\xea\x3f\x56\x2a\x8d\x30\xa1\x56\xc1\x99\xfd\xaf\xd2\x5b" + "\xe0\x6e\xe8\x95\x58\x11\x95\xef\x12\x5c\xb4\xe6\x29\xe4\xf1\x8e\x0b" + "\xee\x97\x9d\x31\x51\x38\x96\xdb\x84\x66\xe4\x48\xe6\xb4\x60\x0a\x31" + "\x67\x57", + "\x41\x4e\xc6\xb1\x49\xe5\x47\x35\x30\x2d\xad\xa8\x88\xb9\x8b\x7f\xdb" + "\x4c\x12\x7c", + "\xe4\xd3\xf4\x89\x8c\xb3\xd9\x73\x26\x41\xd1\xf8\xd9\xd8\x89\xb2\xc9" + "\x8a\xf9\x30", + "\x76\xd4\xfb\xb6\x9d\x52\x9b\x64\x17\x5b\x32\x8b\xe0\x0b\x10\x68", 1, + 2056, 160 }, + { 96, 128, 128, 21, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "", + "\xeb\xd4\xa3\xe1\x0c\xf6\xd4\x1c\x50\xae\xae\x00\x75\x63\xb0\x72", + "\xf6\x2d\x84\xd6\x49\xe5\x6b\xc8\xcf\xed\xc5\xd7\x4a\x51\xe2\xf7", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 96, 128, 128, 22, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "", + "\xd5\x93\xc4\xd8\x22\x4f\x1b\x10\x0c\x35\xe4\xf6\xc4\x00\x65\x43", + "\x43\x1f\x31\xe6\x84\x09\x31\xfd\x95\xf9\x4b\xf8\x82\x96\xff\x69", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 96, 128, 128, 23, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd9\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 24, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xda\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 25, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\x58\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 26, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x85\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 27, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\x3c\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x33\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x30\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\x69\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x89\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x08\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xe7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7b\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x87\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x84\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x06\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x82", 0, + 0, 128 }, + { 96, 128, 128, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x81", 0, + 0, 128 }, + { 96, 128, 128, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\xc3", 0, + 0, 128 }, + { 96, 128, 128, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\xe9\x88\xc7\x7a\xd3\x86\x3e\x60\x03", 0, + 0, 128 }, + { 96, 128, 128, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd9\x84\x7d\xbc\x32\x6a\x06\xe9\x89\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\x3c\x32\x6a\x06\x69\x88\xc7\x7a\xd3\x86\x3e\x60\x83", 0, + 0, 128 }, + { 96, 128, 128, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd8\x84\x7d\xbc\x32\x6a\x06\x69\x88\xc7\x7a\xd3\x86\x3e\x60\x03", 0, + 0, 128 }, + { 96, 128, 128, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\x27\x7b\x82\x43\xcd\x95\xf9\x16\x77\x38\x85\x2c\x79\xc1\x9f\x7c", 0, + 0, 128 }, + { 96, 128, 128, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 128, 128, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 128, 128, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\x58\x04\xfd\x3c\xb2\xea\x86\x69\x08\x47\xfa\x53\x06\xbe\xe0\x03", 0, + 0, 128 }, + { 96, 128, 128, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xeb\x15\x6d\x08\x1e\xd6\xb6\xb5\x5f\x46\x12\xf0\x21\xd8\x7b\x39", + "\xd9\x85\x7c\xbd\x33\x6b\x07\xe8\x89\xc6\x7b\xd2\x87\x3f\x61\x82", 0, + 0, 128 }, + { 64, 128, 128, 50, + "\xaa\x02\x3d\x04\x78\xdc\xb2\xb2\x31\x24\x98\x29\x3d\x9a\x91\x29", + "\x04\x32\xbc\x49\xac\x34\x41\x20", + "\xaa\xc3\x92\x31\x12\x98\x72\xa2", + "\x20\x35\xaf\x31\x3d\x13\x46\xab\x00\x15\x4f\xea\x78\x32\x21\x05", + "\x64\xc3\x6b\xb3\xb7\x32\x03\x4e\x3a\x7d\x04\xef\xc5\x19\x77\x85", + "\xb7\xd0\xdd\x70\xb0\x0d\x65\xb9\x7c\xfd\x08\x0f\xf4\xb8\x19\xd1", 1, + 64, 128 }, + { 64, 128, 128, 51, + "\xf3\x43\x47\x25\xc8\x2a\x7f\x8b\xb0\x7d\xf1\xf8\x12\x2f\xb6\xc9", + "\x28\xe9\xb7\x85\x17\x24\xba\xe3", "", "", "", + "\x44\xac\xa0\x0f\x42\xe4\x19\x9b\x82\x9a\x55\xe6\x9b\x07\x3d\x9e", 1, + 0, 0 }, + { 64, 128, 128, 52, + "\xde\xb6\x22\x33\x55\x9b\x57\x47\x66\x02\xb5\xad\xac\x57\xc7\x7f", + "\xd0\x84\x54\x7d\xe5\x5b\xbc\x15", "", + "\xd8\x98\x6d\xf0\x24\x1e\xd3\x29\x75\x82\xc0\xc2\x39\xc7\x24\xcb", + "\x03\xe1\xa1\x68\xa7\xe3\x77\xa9\x13\x87\x9b\x29\x6a\x1b\x5f\x9c", + "\x32\x90\xaa\x95\xaf\x50\x5a\x74\x2f\x51\x7f\xab\xcc\x9b\x20\x94", 1, + 0, 128 }, + { 128, 128, 128, 53, + "\x20\x34\xa8\x25\x47\x27\x6c\x83\xdd\x32\x12\xa8\x13\x57\x2b\xce", + "\x32\x54\x20\x2d\x85\x47\x34\x81\x23\x98\x12\x7a\x3d\x13\x44\x21", + "\x1a\x02\x93\xd8\xf9\x02\x19\x05\x89\x02\x13\x90\x13\x90\x81\x90\xbc" + "\x49\x08\x90\xd3\xff\x12\xa3", + "\x02\xef\xd2\xe5\x78\x23\x12\x82\x7e\xd5\xd2\x30\x18\x9a\x2a\x34\x2b" + "\x27\x7c\xe0\x48\x46\x21\x93", + "\x64\x06\x9c\x2d\x58\x69\x05\x61\xf2\x7e\xe1\x99\xe6\xb4\x79\xb6\x36" + "\x9e\xec\x68\x86\x72\xbd\xe9", + "\x9b\x7a\xba\xdd\x6e\x69\xc1\xd9\xec\x92\x57\x86\x53\x4f\x50\x75", 1, + 192, 192 }, + { 128, 128, 128, 54, + "\xb6\x7b\x1a\x6e\xfd\xd4\x0d\x37\x08\x0f\xbe\x8f\x80\x47\xae\xb9", + "\xfa\x29\x4b\x12\x99\x72\xf7\xfc\x5b\xbd\x5b\x96\xbb\xa8\x37\xc9", + "", "", "", + "\xa2\xcf\x26\x48\x15\x17\xec\x25\x08\x5c\x5b\x17\xd0\x78\x61\x83", 1, + 0, 0 }, + { 128, 128, 128, 55, + "\x20\x9e\x6d\xbf\x2a\xd2\x6a\x10\x54\x45\xfc\x02\x07\xcd\x9e\x9a", + "\x94\x77\x84\x9d\x6c\xcd\xfc\xa1\x12\xd9\x2e\x53\xfa\xe4\xa7\xca", + "", "\x01", "\xfd", + "\x03\x2d\xf7\xbb\xa5\xd8\xea\x1a\x14\xf1\x6f\x70\xbd\x0e\x14\xec", 1, + 0, 8 }, + { 128, 128, 128, 56, + "\xa5\x49\x44\x2e\x35\x15\x40\x32\xd0\x7c\x86\x66\x00\x6a\xa6\xa2", + "\x51\x71\x52\x45\x68\xe8\x1d\x97\xe8\xc4\xde\x4b\xa5\x6c\x10\xa0", + "", "\x11\x82\xe9\x35\x96\xca\xc5\x60\x89\x46\x40\x0b\xc7\x3f\x3a", + "\x2f\x33\x30\x87\xbd\xca\x58\x21\x9f\x9b\xfc\x27\x3e\x45\xcc", + "\xe0\x6d\x1e\xf4\x73\x13\x29\x57\xad\x37\xea\xef\x29\x73\x3c\xa0", 1, + 0, 120 }, + { 128, 128, 128, 57, + "\xcf\xb4\xc2\x6f\x12\x6f\x6a\x0a\xcb\x8e\x4e\x22\x0f\x6c\x56\xcd", + "\x12\x75\x11\x54\x99\xae\x72\x22\x68\x51\x5b\xf0\xc1\x64\xb4\x9c", + "", + "\x09\xdf\xd7\xf0\x80\x27\x52\x57\xcf\x97\xe7\x6f\x96\x6b\x1a\xd9", + "\xa7\x80\xbd\x01\xc8\x08\x85\x15\x6c\x88\xa9\x73\x26\x4c\x8e\xe5", + "\x2a\xde\xff\xa6\x82\xc8\xd8\xa8\x1f\xad\xa7\xd9\xfc\xdd\x2e\xe2", 1, + 0, 128 }, + { 128, 128, 128, 58, + "\x0b\x11\xef\x3a\x08\xc0\x29\x70\xf7\x42\x81\xc8\x60\x69\x1c\x75", + "\x95\xc1\xdd\x8c\x0f\x17\x05\xec\xe6\x89\x37\x90\x1f\x7a\xdd\x7b", + "", + "\xf6\x93\xd4\xed\xd8\x25\xdb\xb0\x61\x8d\x91\x11\x31\x28\x88\x0d\xbe" + "\xbb\x23\xe2\x5d\x00\xed\x1f\x07\x7d\x87\x0b\xe9\xcc\x75\x36", + "\x7e\x47\xe1\x0f\xe3\xc6\xfb\xfa\x38\x17\x70\xea\xf5\xd4\x8d\x14\x82" + "\xe7\x1e\x0c\x44\xdf\xf1\xe3\x0c\xa6\xf9\x5d\x92\x05\x20\x84", + "\xd0\x14\x44\xfa\x5d\x9c\x49\x96\x29\xd1\x74\xff\x39\x27\xa1\xac", 1, + 0, 256 }, + { 128, 128, 128, 59, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xf9\x5f\xde\x4a\x75\x19\x13\x20\x2a\xee\xee\x32\xa0\xb5\x57\x53", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x00\x07\x8d\x10\x9d\x92\x14\x3f\xcd\x5d\xf5\x67\x21\xb8\x84\xfa\xc6" + "\x4a\xc7\x76\x2c\xc0\x9e\xea\x2a\x3c\x68\xe9\x2a\x17\xbd\xb5\x75\xf8" + "\x7b\xda\x18\xbe\x56\x4e", + "\x15\x2a\x65\x04\x5f\xe6\x74\xf9\x76\x27\x42\x7a\xf5\xbe\x22\xda", 1, + 0, 320 }, + { 128, 128, 128, 60, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x7b\x95\xb8\xc3\x56\x81\x0a\x84\x71\x1d\x68\x15\x0a\x1b\x77\x50", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x84\xd4\xc9\xc0\x8b\x4f\x48\x28\x61\xe3\xa9\xc6\xc3\x5b\xc4\xd9\x1d" + "\xf9\x27\x37\x45\x13\xbf\xd4\x9f\x43\x6b\xd7\x3f\x32\x52\x85\xda\xef" + "\x4f\xf7\xe1\x3d\x46\xa6", + "\x21\x3a\x3c\xb9\x38\x55\xd1\x8e\x69\x33\x7e\xee\x66\xae\xec\x07", 1, + 0, 320 }, + { 128, 128, 128, 61, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x1a\x55\x2e\x67\xcd\xc4\xdc\x1a\x33\xb8\x24\x87\x4e\xbf\x0b\xed", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x94\x8c\xa3\x7a\x8e\x66\x49\xe8\x8a\xef\xfb\x1c\x59\x8f\x36\x07\x00" + "\x77\x02\x41\x7e\xa0\xe0\xbc\x3c\x60\xad\x5a\x94\x98\x86\xde\x96\x8c" + "\xf5\x3e\xa6\x46\x2a\xed", + "\x99\xb3\x81\xbf\xa2\xaf\x97\x51\xc3\x9d\x1b\x6e\x86\xd1\xbe\x6a", 1, + 0, 320 }, + { 128, 128, 128, 62, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xdd\x9d\x0b\x4a\x0c\x3d\x68\x15\x24\xbf\xfc\xa3\x1d\x90\x76\x61", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x64\xb1\x93\x14\xc3\x1a\xf4\x5a\xcc\xdf\x7e\x3c\x4d\xb7\x9f\x0d\x94" + "\x8c\xa3\x7a\x8e\x66\x49\xe8\x8a\xef\xfb\x1c\x59\x8f\x36\x07\x00\x77" + "\x02\x41\x7e\xa0\xe0\xbc", + "\x52\x81\xef\xc7\xf1\x3a\xc8\xe1\x4c\xcf\x5d\xca\x7b\xfb\xfd\xd1", 1, + 0, 320 }, + { 128, 128, 128, 63, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x57\xc5\x64\x3c\x4e\x37\xb4\x04\x1d\xb7\x94\xcf\xe8\xe1\xf0\xf4", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x2b\xb6\x9c\x3e\x5d\x1f\x91\x81\x5c\x6b\x87\xa0\xd5\xbb\xea\x71\x64" + "\xb1\x93\x14\xc3\x1a\xf4\x5a\xcc\xdf\x7e\x3c\x4d\xb7\x9f\x0d\x94\x8c" + "\xa3\x7a\x8e\x66\x49\xe8", + "\xa3\xea\x2c\x09\xee\x4f\x8c\x8a\x12\xf4\x5c\xdd\xf9\xae\xff\x81", 1, + 0, 320 }, + { 128, 128, 128, 64, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x99\x82\x1c\x2d\xd5\xda\xec\xde\xd0\x73\x00\xf5\x77\xf7\xaf\xf1", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x12\x7a\xf9\xb3\x9e\xcd\xfc\x57\xbb\x11\xa2\x84\x7c\x7c\x2d\x3d\x8f" + "\x93\x8f\x40\xf8\x77\xe0\xc4\xaf\x37\xd0\xfe\x9a\xf0\x33\x05\x2b\xd5" + "\x37\xc4\xae\x97\x8f\x60", + "\x07\xeb\x2f\xe4\xa9\x58\xf8\x43\x4d\x40\x68\x48\x99\x50\x7c\x7c", 1, + 0, 320 }, + { 128, 128, 128, 65, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x5e\x4a\x39\x00\x14\x23\x58\xd1\xc7\x74\xd8\xd1\x24\xd8\xd2\x7d", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x0c\xf6\xae\x47\x15\x6b\x14\xdc\xe0\x3c\x8a\x07\xa2\xe1\x72\xb1\x12" + "\x7a\xf9\xb3\x9e\xcd\xfc\x57\xbb\x11\xa2\x84\x7c\x7c\x2d\x3d\x8f\x93" + "\x8f\x40\xf8\x77\xe0\xc4", + "\xf1\x45\xc2\xdc\xaf\x33\x9e\xed\xe4\x27\xbe\x93\x43\x57\xea\xc0", 1, + 0, 320 }, + { 128, 128, 128, 66, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xd4\x12\x56\x76\x56\x29\x84\xc0\xfe\x7c\xb0\xbd\xd1\xa9\x54\xe8", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xf0\xc6\xff\xc1\x8b\xd4\x6d\xf5\x56\x91\x85\xa9\xaf\xd1\x69\xeb\x0c" + "\xf6\xae\x47\x15\x6b\x14\xdc\xe0\x3c\x8a\x07\xa2\xe1\x72\xb1\x12\x7a" + "\xf9\xb3\x9e\xcd\xfc\x57", + "\xfa\xcd\x0b\xfe\x87\x01\xb7\xb4\xa2\xba\x96\xd9\x8a\xf5\x2b\xd9", 1, + 0, 320 }, + { 128, 128, 128, 67, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xb9\x7e\xc6\x2a\x5e\x59\x00\xcc\xf9\xe4\xbe\x33\x2e\x33\x60\x91", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xd6\x92\x8e\x09\x4c\x06\xe0\xa7\xc4\xdb\x42\x18\x4c\xf7\x52\x9e\x95" + "\xde\x88\xb7\x67\xed\xeb\xe9\xb3\x43\x00\x0b\xe3\xda\xb4\x7e\xa0\x8b" + "\x74\x42\x93\xee\xd6\x98", + "\xa0\x3e\x72\x9d\xcf\xd7\xa0\x31\x55\x65\x5f\xec\xe8\xaf\xfd\x7e", 1, + 0, 320 }, + { 128, 128, 128, 68, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x7e\xb6\xe3\x07\x9f\xa0\xb4\xc3\xee\xe3\x66\x17\x7d\x1c\x1d\x1d", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xd8\x2c\xe5\x87\x71\xbf\x64\x87\x11\x6b\xf8\xe9\x64\x21\x87\x7e\xd6" + "\x92\x8e\x09\x4c\x06\xe0\xa7\xc4\xdb\x42\x18\x4c\xf7\x52\x9e\x95\xde" + "\x88\xb7\x67\xed\xeb\xe9", + "\x1e\x43\x92\x68\x28\xbc\x9a\x16\x14\xc7\xb1\x63\x90\x96\xc1\x95", 1, + 0, 320 }, + { 128, 128, 128, 69, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x03\x14\xfc\xd1\x0f\xdd\x67\x5d\x3c\x61\x29\x62\xc9\x31\xf6\x35", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xa1\x97\xa3\x7a\x5d\x79\x69\x70\x78\x53\x6b\xc2\x7f\xe4\x6c\xd8\xd4" + "\x75\x52\x6d\x90\x44\xaa\x94\xf0\x88\xa0\x54\xf8\xe3\x80\xc6\x4f\x79" + "\x41\x47\x95\xc6\x14\x80", + "\xf0\x8b\xad\xdf\x0b\x52\x85\xc9\x1f\xc0\x6a\x67\xfe\x47\x08\xca", 1, + 0, 320 }, + { 128, 128, 128, 70, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xc4\xdc\xd9\xfc\xce\x24\xd3\x52\x2b\x66\xf1\x46\x9a\x1e\x8b\xb9", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x14\x9f\xde\x9a\xbb\xd3\xa4\x3c\x25\x48\x57\x5e\x0d\xb9\xfb\x84\xa1" + "\x97\xa3\x7a\x5d\x79\x69\x70\x78\x53\x6b\xc2\x7f\xe4\x6c\xd8\xd4\x75" + "\x52\x6d\x90\x44\xaa\x94", + "\x62\xa4\xb6\x87\x5c\x28\x83\x45\xd6\xa4\x54\x39\x9e\xac\x1a\xfa", 1, + 0, 320 }, + { 128, 128, 128, 71, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "", + "\xbe\xc6\xfa\x05\xc1\x71\x8b\x9b\x84\xc4\x73\x45\xbb\xed\x7d\xcb", + "\x45\xa3\xf8\x9d\x02\x91\x8b\xfd\x0c\x81\x61\x65\x8c\xcc\x97\x95", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 128, 128, 128, 72, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "", + "\x4d\x82\x63\x9c\x39\xd3\xf3\x49\x0e\xe9\x03\xdd\x0b\xe7\xaf\xcf", + "\x1c\xd5\xa0\x62\x14\x23\x5c\xeb\x04\x4d\x4b\xad\x7b\x04\x73\x12", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 96, 256, 128, 73, + "\x92\xac\xe3\xe3\x48\xcd\x82\x10\x92\xcd\x92\x1a\xa3\x54\x63\x74\x29" + "\x9a\xb4\x62\x09\x69\x1b\xc2\x8b\x87\x52\xd1\x7f\x12\x3c\x20", + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb", + "\x00\x00\x00\x00\xff\xff\xff\xff", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09", + "\xe2\x7a\xbd\xd2\xd2\xa5\x3d\x2f\x13\x6b", + "\x9a\x4a\x25\x79\x52\x93\x01\xbc\xfb\x71\xc7\x8d\x40\x60\xf5\x2c", 1, + 64, 80 }, + { 96, 256, 128, 74, + "\x29\xd3\xa4\x4f\x87\x23\xdc\x64\x02\x39\x10\x0c\x36\x54\x23\xa3\x12" + "\x93\x4a\xc8\x02\x39\x21\x2a\xc3\xdf\x34\x21\xa2\x09\x81\x23", + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb", + "\xaa\xbb\xcc\xdd\xee\xff", "", "", + "\x2a\x7d\x77\xfa\x52\x6b\x82\x50\xcb\x29\x60\x78\x92\x6b\x50\x20", 1, + 48, 0 }, + { 96, 256, 128, 75, + "\x80\xba\x31\x92\xc8\x03\xce\x96\x5e\xa3\x71\xd5\xff\x07\x3c\xf0\xf4" + "\x3b\x6a\x2a\xb5\x76\xb2\x08\x42\x6e\x11\x40\x9c\x09\xb9\xb0", + "\x4d\xa5\xbf\x8d\xfd\x58\x52\xc1\xea\x12\x37\x9d", "", "", "", + "\x47\x71\xa7\xc4\x04\xa4\x72\x96\x6c\xea\x8f\x73\xc8\xbf\xe1\x7a", 1, + 0, 0 }, + { 96, 256, 128, 76, + "\xcc\x56\xb6\x80\x55\x2e\xb7\x50\x08\xf5\x48\x4b\x4c\xb8\x03\xfa\x50" + "\x63\xeb\xd6\xea\xb9\x1f\x6a\xb6\xae\xf4\x91\x6a\x76\x62\x73", + "\x99\xe2\x3e\xc4\x89\x85\xbc\xcd\xee\xab\x60\xf1", "", "\x2a", + "\x06", + "\x63\x3c\x1e\x97\x03\xef\x74\x4f\xff\xfb\x40\xed\xf9\xd1\x43\x55", 1, + 0, 8 }, + { 96, 256, 128, 77, + "\x51\xe4\xbf\x2b\xad\x92\xb7\xaf\xf1\xa4\xbc\x05\x55\x0b\xa8\x1d\xf4" + "\xb9\x6f\xab\xf4\x1c\x12\xc7\xb0\x0e\x60\xe4\x8d\xb7\xe1\x52", + "\x4f\x07\xaf\xed\xfd\xc3\xb6\xc2\x36\x18\x23\xd3", "", + "\xbe\x33\x08\xf7\x2a\x2c\x6a\xed", + "\xcf\x33\x2a\x12\xfd\xee\x80\x0b", + "\x60\x2e\x8d\x7c\x47\x99\xd6\x2c\x14\x0c\x9b\xb8\x34\x87\x6b\x09", 1, + 0, 64 }, + { 96, 256, 128, 78, + "\x67\x11\x96\x27\xbd\x98\x8e\xda\x90\x62\x19\xe0\x8c\x0d\x0d\x77\x9a" + "\x07\xd2\x08\xce\x8a\x4f\xe0\x70\x9a\xf7\x55\xee\xec\x6d\xcb", + "\x68\xab\x7f\xdb\xf6\x19\x01\xda\xd4\x61\xd2\x3c", "", + "\x51\xf8\xc1\xf7\x31\xea\x14\xac\xdb\x21\x0a\x6d\x97\x3e\x07", + "\x43\xfc\x10\x1b\xff\x4b\x32\xbf\xad\xd3\xda\xf5\x7a\x59\x0e", + "\xec\x04\xaa\xcb\x71\x48\xa8\xb8\xbe\x44\xcb\x7e\xaf\x4e\xfa\x69", 1, + 0, 120 }, + { 96, 256, 128, 79, + "\x59\xd4\xea\xfb\x4d\xe0\xcf\xc7\xd3\xdb\x99\xa8\xf5\x4b\x15\xd7\xb3" + "\x9f\x0a\xcc\x8d\xa6\x97\x63\xb0\x19\xc1\x69\x9f\x87\x67\x4a", + "\x2f\xcb\x1b\x38\xa9\x9e\x71\xb8\x47\x40\xad\x9b", "", + "\x54\x9b\x36\x5a\xf9\x13\xf3\xb0\x81\x13\x1c\xcb\x6b\x82\x55\x88", + "\xf5\x8c\x16\x69\x01\x22\xd7\x53\x56\x90\x7f\xd9\x6b\x57\x0f\xca", + "\x28\x75\x2c\x20\x15\x30\x92\x81\x8f\xab\xa2\xa3\x34\x64\x0d\x6e", 1, + 0, 128 }, + { 96, 256, 128, 80, + "\x3b\x24\x58\xd8\x17\x6e\x16\x21\xc0\xcc\x24\xc0\xc0\xe2\x4c\x1e\x80" + "\xd7\x2f\x7e\xe9\x14\x9a\x4b\x16\x61\x76\x62\x96\x16\xd0\x11", + "\x45\xaa\xa3\xe5\xd1\x6d\x2d\x42\xdc\x03\x44\x5d", "", + "\x3f\xf1\x51\x4b\x1c\x50\x39\x15\x91\x8f\x0c\x0c\x31\x09\x4a\x6e" + "\x1f", + "\x73\xa6\xb6\xf4\x5f\x6c\xcc\x51\x31\xe0\x7f\x2c\xaa\x1f\x2e\x2f" + "\x56", + "\x2d\x73\x79\xec\x1d\xb5\x95\x2d\x4e\x95\xd3\x0c\x34\x0b\x1b\x1d", 1, + 0, 136 }, + { 96, 256, 128, 81, + "\x02\x12\xa8\xde\x50\x07\xed\x87\xb3\x3f\x1a\x70\x90\xb6\x11\x4f\x9e" + "\x08\xce\xfd\x96\x07\xf2\xc2\x76\xbd\xcf\xdb\xc5\xce\x9c\xd7", + "\xe6\xb1\xad\xf2\xfd\x58\xa8\x76\x2c\x65\xf3\x1b", "", + "\x10\xf1\xec\xf9\xc6\x05\x84\x66\x5d\x9a\xe5\xef\xe2\x79\xe7\xf7\x37" + "\x7e\xea\x69\x16\xd2\xb1\x11", + "\x08\x43\xff\xf5\x2d\x93\x4f\xc7\xa0\x71\xea\x62\xc0\xbd\x35\x1c\xe8" + "\x56\x78\xcd\xe3\xea\x2c\x9e", + "\x73\x55\xfd\xe5\x99\x00\x67\x15\x05\x38\x13\xce\x69\x62\x37\xa8", 1, + 0, 192 }, + { 96, 256, 128, 82, + "\xb2\x79\xf5\x7e\x19\xc8\xf5\x3f\x2f\x96\x3f\x5f\x25\x19\xfd\xb7\xc1" + "\x77\x9b\xe2\xca\x2b\x3a\xe8\xe1\x12\x8b\x7d\x6c\x62\x7f\xc4", + "\x98\xbc\x2c\x74\x38\xd5\xcd\x76\x65\xd7\x6f\x6e", "\xc0", + "\xfc\xc5\x15\xb2\x94\x40\x8c\x86\x45\xc9\x18\x3e\x3f\x4e\xce\xe5\x12" + "\x78\x46\xd1", + "\xeb\x55\x00\xe3\x82\x59\x52\x86\x6d\x91\x12\x53\xf8\xde\x86\x0c\x00" + "\x83\x1c\x81", + "\xec\xb6\x60\xe1\xfb\x05\x41\xec\x41\xe8\xd6\x8a\x64\x14\x1b\x3a", 1, + 8, 160 }, + { 96, 256, 128, 83, + "\xcd\xcc\xfe\x3f\x46\xd7\x82\xef\x47\xdf\x4e\x72\xf0\xc0\x2d\x9c\x7f" + "\x77\x4d\xef\x97\x0d\x23\x48\x6f\x11\xa5\x7f\x54\x24\x7f\x17", + "\x37\x61\x87\x89\x46\x05\xa8\xd4\x5e\x30\xde\x51", + "\x95\x68\x46\xa2\x09\xe0\x87\xed", + "\xe2\x8e\x0e\x9f\x9d\x22\x46\x3a\xc0\xe4\x26\x39\xb5\x30\xf4\x21\x02" + "\xfd\xed\x75", + "\xfe\xca\x44\x95\x24\x47\x01\x5b\x5d\xf1\xf4\x56\xdf\x8c\xa4\xbb\x4e" + "\xee\x2c\xe2", + "\x08\x2e\x91\x92\x4d\xee\xb7\x78\x80\xe1\xb1\xc8\x4f\x9b\x8d\x30", 1, + 64, 160 }, + { 96, 256, 128, 84, + "\xf3\x23\x64\xb1\xd3\x39\xd8\x2e\x4f\x13\x2d\x8f\x4a\x0e\xc1\xff\x7e" + "\x74\x65\x17\xfa\x07\xef\x1a\x7f\x42\x2f\x4e\x25\xa4\x81\x94", + "\x5a\x86\xa5\x0a\x0e\x8a\x17\x9c\x73\x4b\x99\x6d", + "\xab\x2a\xc7\xc4\x4c\x60\xbd\xf8\x22\x8c\x78\x84\xad\xb2\x01\x84", + "\x43\x89\x1b\xcc\xb5\x22\xb1\xe7\x2a\x6b\x53\xcf\x31\xc0\x74\xe9\xd6" + "\xc2\xdf\x8e", + "\x43\xdd\xa8\x32\xe9\x42\xe2\x86\xda\x31\x4d\xaa\x99\xbe\xf5\x07\x1d" + "\x9d\x2c\x78", + "\xc3\x92\x25\x83\x47\x6c\xed\x57\x54\x04\xdd\xb8\x5d\xd8\xcd\x44", 1, + 128, 160 }, + { 96, 256, 128, 85, + "\xff\x00\x89\xee\x87\x0a\x4a\x39\xf6\x45\xb0\xa5\xda\x77\x4f\x7a\x59" + "\x11\xe9\x69\x6f\xc9\xca\xd6\x46\x45\x2c\x2a\xa8\x59\x5a\x12", + "\xbc\x2a\x77\x57\xd0\xce\x2d\x8b\x1f\x14\xcc\xd9", + "\x97\x2a\xb4\xe0\x63\x90\xca\xae\x8f\x99\xdd\x6e\x21\x87\xbe\x6c\x7f" + "\xf2\xc0\x8a\x24\xbe\x16\xef", + "\x74\x8b\x28\x03\x16\x21\xd9\x5e\xe6\x18\x12\xb4\xb4\xf4\x7d\x04\xc6" + "\xfc\x2f\xf3", + "\xa9\x29\xee\x7e\x67\xc7\xa2\xf9\x1b\xbc\xec\x63\x89\xa3\xca\xf4\x3a" + "\xb4\x93\x05", + "\xeb\xec\x67\x74\xb9\x55\xe7\x89\x59\x1c\x82\x2d\xab\x73\x9e\x12", 1, + 192, 160 }, + { 96, 256, 128, 86, + "\x5b\x1d\x10\x35\xc0\xb1\x7e\xe0\xb0\x44\x47\x67\xf8\x0a\x25\xb8\xc1" + "\xb7\x41\xf4\xb5\x0a\x4d\x30\x52\x22\x6b\xaa\x1c\x6f\xb7\x01", + "\xd6\x10\x40\xa3\x13\xed\x49\x28\x23\xcc\x06\x5b", "", + "\xd0\x96\x80\x31\x81\xbe\xef\x9e\x00\x8f\xf8\x5d\x5d\xdc\x38\xdd\xac" + "\xf0\xf0\x9e\xe5\xf7\xe0\x7f\x1e\x40\x79\xcb\x64\xd0\xdc\x8f\x5e\x67" + "\x11\xcd\x49\x21\xa7\x88\x7d\xe7\x6e\x26\x78\xfd\xc6\x76\x18\xf1\x18" + "\x55\x86\xbf\xea\x9d\x4c\x68\x5d\x50\xe4\xbb\x9a\x82", + "\xc7\xd1\x91\xb6\x01\xf8\x6c\x28\xb6\xa1\xbd\xef\x6a\x57\xb4\xf6\xee" + "\x3a\xe4\x17\xbc\x12\x5c\x38\x1c\xdf\x1c\x4d\xac\x18\x4e\xd1\xd8\x4f" + "\x11\x96\x20\x6d\x62\xca\xd1\x12\xb0\x38\x84\x57\x20\xe0\x2c\x06\x11" + "\x79\xa8\x83\x6f\x02\xb9\x3f\xa7\x00\x83\x79\xa6\xbf", + "\xf1\x56\x12\xf6\xc4\x0f\x2e\x0d\xb6\xdc\x76\xfc\x48\x22\xfc\xfe", 1, + 0, 512 }, + { 96, 256, 128, 87, + "\xd7\xad\xdd\x38\x89\xfa\xdf\x8c\x89\x3e\xee\x14\xba\x2b\x7e\xa5\xbf" + "\x56\xb4\x49\x90\x48\x69\x61\x5b\xd0\x5d\x5f\x11\x4c\xf3\x77", + "\x8a\x3a\xd2\x6b\x28\xcd\x13\xba\x65\x04\xe2\x60", "", + "\xc8\x77\xa7\x6b\xf5\x95\x56\x07\x72\x16\x7c\x6e\x3b\xcc\x70\x53\x05" + "\xdb\x9c\x6f\xcb\xeb\x90\xf4\xfe\xa8\x51\x16\x03\x8b\xc5\x3c\x3f\xa5" + "\xb4\xb4\xea\x0d\xe5\xcc\x53\x4f\xbe\x1c\xf9\xae\x44\x82\x4c\x6c\x2c" + "\x0a\x5c\x88\x5b\xd8\xc3\xcd\xc9\x06\xf1\x26\x75\x73\x7e\x43\x4b\x98" + "\x3e\x1e\x23\x1a\x52\xa2\x75\xdb\x5f\xb1\xa0\xca\xc6\xa0\x7b\x3b\x7d" + "\xcb\x19\x48\x2a\x5d\x3b\x06\xa9\x31\x7a\x54\x82\x6c\xea\x6b\x36\xfc" + "\xe4\x52\xfa\x9b\x54\x75\xe2\xaa\xf2\x54\x99\x49\x9d\x8a\x89\x32\xa1" + "\x9e\xb9\x87\xc9\x03\xbd\x85\x02\xfe", + "\x53\xcc\x8c\x92\x0a\x85\xd1\xac\xcb\x88\x63\x6d\x08\xbb\xe4\x86\x9b" + "\xfd\xd9\x6f\x43\x7b\x2e\xc9\x44\x51\x21\x73\xa9\xc0\xfe\x7a\x47\xf8" + "\x43\x41\x33\x98\x9b\xa7\x7d\xda\x56\x1b\x7e\x37\x01\xb9\xa8\x3c\x3b" + "\xa7\x66\x0c\x66\x6b\xa5\x9f\xef\x96\x59\x8e\xb6\x21\x54\x4c\x63\x80" + "\x6d\x50\x9a\xc4\x76\x97\x41\x2f\x95\x64\xeb\x0a\x2e\x1f\x72\xf6\x59" + "\x9f\x56\x66\xaf\x34\xcf\xfc\xa0\x65\x73\xff\xb4\xf4\x7b\x02\xf5\x9f" + "\x21\xc6\x43\x63\xda\xec\xb9\x77\xb4\x41\x5f\x19\xfd\xda\x3c\x9a\xae" + "\x50\x66\xa5\x7b\x66\x9f\xfa\xa2\x57", + "\x5e\x63\x37\x4b\x51\x9e\x6c\x36\x08\x32\x19\x43\xd7\x90\xcf\x9a", 1, + 0, 1024 }, + { 96, 256, 128, 88, + "\x31\x7b\xa3\x31\x30\x7f\x3a\x3d\x3d\x82\xee\x1f\xda\xb7\x0f\x62\xa1" + "\x55\xaf\x14\xda\xf6\x31\x30\x7a\x61\xb1\x87\xd4\x13\xe5\x33", + "\xa6\x68\x7c\xf5\x08\x35\x6b\x17\x46\x25\xde\xaa", "", + "\x32\xc1\xd0\x91\x07\xc5\x99\xd3\xcc\xe4\xe7\x82\x17\x9c\x96\x6c\x6e" + "\xf9\x63\x68\x9d\x45\x35\x1d\xbe\x0f\x6f\x88\x1d\xb2\x73\xe5\x4d\xb7" + "\x6f\xc4\x8f\xdc\x5d\x30\xf0\x89\xda\x83\x83\x01\xa5\xf9\x24\xbb\xa3" + "\xc0\x44\xe1\x9b\x3e\xd5\xaa\x6b\xe8\x71\x18\x55\x40\x04\xca\x30\xe0" + "\x32\x43\x37\xd9\x87\x83\x94\x12\xbf\x8f\x8b\xbd\xd5\x37\x20\x5d\x4b" + "\x0e\x21\x20\xe9\x65\x37\x32\x35\xd6\xcb\xd2\xfb\x37\x76\xba\x0a\x38" + "\x4e\xc1\xd9\xb7\xc6\x31\xa0\x37\x9f\xf9\x97\xc3\xf9\x74\xa6\xf7\xbb" + "\xf4\xfd\x23\x01\x62\x11\xf5\xfc\x10\xac\xad\xb5\xe4\x00\xd2\xff\x0f" + "\xdf\xd1\x93\xf5\xc6\xfc\x6d\x4f\x72\x71\xdf\xd1\x34\x9e\xd8\x0f\xbe" + "\xda\xeb\xb1\x55\xb9\xb0\x2f\xb3\x07\x44\x95\xd5\x5f\x9a\x24\x55\xf5" + "\x9b\xf6\xf1\x13\x19\x1a\x02\x9c\x6b\x0b\xa7\x5d\x97\xcd\xc0\xc8\x4f" + "\x13\x18\x36\x33\x7f\x29\xf9\xd9\x6c\xa4\x48\xee\xc0\xcc\x46\xd1\xca" + "\x8b\x37\x35\x66\x19\x79\xd8\x33\x02\xfe\xc0\x8f\xff\xcf\x5e\x58\xf1" + "\x2b\x1e\x70\x50\x65\x7b\x1b\x97\xc6\x4a\x4e\x07\xe3\x17\xf5\x54\xf8" + "\x31\x0b\x6c\xcb\x49\xf3\x6d\x48\xc5\x78\x16\xd2\x49\x52\xaa\xda\x71" + "\x1d\x4f", + "\xd7\xee\xbc\x95\x87\xaa\x21\x13\x6f\xa3\x8b\x41\xcf\x0e\x2d\xb0\x3a" + "\x7e\xa2\xba\x9e\xad\xdf\x83\xd3\x3f\x78\x10\x93\x61\x7b\xf5\x0f\x49" + "\xb2\xbf\xe2\xf7\x17\x3b\x11\x39\x12\xe2\xe1\x77\x5f\x40\xed\xfe\xd8" + "\xb3\xb0\x09\x9b\x9e\x1c\x22\x0d\xd1\x03\xbe\x61\x66\x21\x0b\x01\x02" + "\x9f\xeb\x24\xed\x9e\x20\x61\x4e\xdd\xc3\xce\xbe\x41\xb0\x07\x9a\x9a" + "\x8c\x11\x7b\x59\x6c\x90\x28\x8e\xff\xd3\x79\x6f\xbd\x0c\x7e\x8e\xab" + "\x00\x60\x9a\x64\xbe\x3a\xd9\x59\x7c\xdb\xf3\xa8\x18\xc2\x60\xcd\x93" + "\x8b\xdf\x23\x2e\x40\x59\xae\x35\xa2\x57\x1a\x83\x88\x87\xfc\x19\x69" + "\x12\x17\x94\x86\xe0\x46\xa6\x22\x27\xa4\xca\xdd\xce\x38\xcb\xbc\x37" + "\x58\x7b\xb9\x43\x9e\xc6\x37\x60\x2b\x68\x18\xc5\xcb\xe3\xc7\x1a\x7c" + "\x41\x43\x96\x05\x33\xdc\x74\x17\x4b\xd3\x15\xc8\xdb\x22\x7b\x69\xb5" + "\x5b\xb7\xfc\x30\xba\x1d\x52\x13\xa7\x52\xec\x33\x92\x50\x43\xce\xfb" + "\xc1\xa6\x29\x43\xee\x5f\x34\xd5\xda\x01\x79\x9e\x69\x09\x4d\x73\x2a" + "\xef\x52\xf8\xe0\x36\x98\x0d\x00\x70\xe2\x2e\x17\x3c\x67\xc4\xbb\xcc" + "\xa6\x1c\xc1\xee\xdb\xd6\x01\x65\x16\xc5\x92\x14\x48\x19\xdf\x13\x20" + "\x4d\xee", + "\xbf\x05\x40\xd3\x4b\x20\xf7\x61\x10\x1b\xc6\x08\xb0\x24\x58\xf2", 1, + 0, 2056 }, + { 96, 256, 128, 89, + "\x2c\xe6\xb4\xc1\x5f\x85\xfb\x2d\xa5\xcc\x6c\x26\x94\x91\xee\xf2\x81" + "\x98\x03\x09\x18\x12\x49\xeb\xf2\x83\x2b\xd6\xd0\x73\x2d\x0b", + "\xc0\x64\xfa\xe9\x17\x3b\x17\x3f\xd6\xf1\x1f\x34", + "\x49\x8d\x30\x75\xb0\x9f\xed\x99\x82\x80\x58\x3d\x61\xbb\x36\xb6\xce" + "\x41\xf1\x30\x06\x3b\x80\x82\x4d\x15\x86\xe1\x43\xd3\x49\xb1\x26\xb1" + "\x6a\xa1\x0f\xe5\x73\x43\xed\x22\x3d\x63\x64\xee\x60\x22\x57\xfe\x31" + "\x3a\x7f\xc9\xbf\x90\x88\xf0\x27\x79\x5b\x8d\xc1\xd3", + "\xf8\xa2\x7a\x4b\xaf\x00\xdc\x05\x55\xd2\x22\xf2\xfa\x4f\xb4\x2d\xc6" + "\x66\xea\x3c", + "\xae\xd5\x8d\x8a\x25\x2f\x74\x0d\xba\x4b\xf6\xd3\x67\x73\xbd\x5b\x41" + "\x23\x4b\xba", + "\x01\xf9\x3d\x74\x56\xaa\x18\x4e\xbb\x49\xbe\xa4\x72\xb6\xd6\x5d", 1, + 512, 160 }, + { 96, 256, 128, 90, + "\x44\xc8\xd0\xcd\xb8\xf7\xe7\x36\xcf\xd9\x97\xc8\x72\xa5\xd9\xc5\xef" + "\x30\xaf\xbe\x44\xb6\x56\x66\x06\xb9\x0a\xa5\xe3\xe8\xb7\x97", + "\x6f\x39\xaf\xba\x02\x1e\x4c\x36\xeb\x92\x96\x2e", + "\x98\xd1\xca\x17\x88\xcb\xeb\x30\x0e\xa5\xc6\xb1\xee\xc9\x5e\xb2\x34" + "\x71\x77\x20\x14\x00\x91\x3d\x45\x22\x56\x22\xb6\x27\x3e\xec\x8a\x74" + "\xc3\xf1\x2c\x8d\x52\x48\xda\xbe\xe5\x86\x22\x97\x86\xff\x19\x2c\x4d" + "\xf0\xc7\x95\x47\xf7\xad\x6a\x92\xd7\x8d\x9f\x89\x52\x75\x86\x35\x78" + "\x3a\xdd\x2a\x59\x77\xd3\x86\xe0\xae\xf7\x64\x82\x21\x1d\x2c\x3a\xe9" + "\x8d\xe4\xba\xad\xb3\xf8\xb3\x5b\x51\x04\x64\x75\x5d\xc7\x5c\xeb\x2b" + "\xf2\x5b\x23\x33\x17\x52\x3f\x39\x9a\x6c\x50\x7d\xb2\x14\xf0\x85\xfa" + "\x28\x18\xf0\xd3\x70\x2b\x10\x95\x2b", + "\x2e\x6f\x40\xf9\xd3\x72\x58\x36\xac\x0c\x85\x81\x77\x93\x8f\xd6\x7b" + "\xe1\x94\x32", + "\xb4\x24\x28\xf8\x09\x4e\xf7\xe6\x5c\x9e\x8c\x45\xef\x3e\x95\xc2\x8c" + "\xe0\x7d\x72", + "\x32\xb2\x5d\xfb\xb8\x96\xd0\xf9\xd7\x9c\x82\x3b\xdd\x8e\x5d\x06", 1, + 1024, 160 }, + { 96, 256, 128, 91, + "\xe4\x00\x03\xd6\xe0\x8a\xb8\x0b\x4b\xfc\x84\x00\xef\x11\x29\x45\xa9" + "\x01\xec\x64\xa1\xb6\x53\x6c\xa9\x26\x65\x09\x0d\x60\x8b\xc4", + "\x9f\x09\x5d\xaf\xe6\xf6\xe0\xfb\xaf\xbb\xe0\x2e", + "\x42\x2d\x5e\xfc\xff\xe3\x64\x90\x59\x84\x53\x3f\x0a\x57\x9d\x80\xb1" + "\x8b\xda\x7b\x29\xe6\xe4\x64\x98\xef\xfb\xa5\x3c\x35\x01\x12\xc0\xbb" + "\xb8\xdc\x4c\xe0\x3b\xb0\xc6\x9e\x1d\x0b\xaa\x19\xf0\x63\x71\x08\xaa" + "\x4a\x16\xb0\x9a\x28\x1f\x23\x28\x39\xd8\x7b\x6d\x0e\x42\xbe\x1b\xaa" + "\x7c\x67\xf1\xbe\x97\x0e\xa1\x69\xd3\x96\x0b\x9f\xe0\xa6\x1f\x11\xcd" + "\x2e\xb7\x39\x8c\x19\xe6\x41\xfe\xb4\x3f\x77\x8e\x25\x7a\x39\x70\x63" + "\xdb\x5b\x3a\x67\x07\xe9\xdb\x62\x38\x70\x54\xf9\xf9\xd4\x4f\x14\x35" + "\x83\xe6\x3e\xda\xd4\x5a\x00\x25\x1e\x51\x73\xd7\x50\x5f\x22\xa8\xbc" + "\xe2\x32\xe5\x6c\x2c\x27\x6a\x58\x03\x3a\xe3\x0d\x5d\xbf\x4e\x35\xa8" + "\x62\xe4\x2a\xf5\x73\xbe\x38\xc6\x40\x6d\x9b\x4c\x7a\xcb\xf2\x75\xfe" + "\x36\xc0\xec\xf2\xc4\x64\x28\x98\xa3\x0e\x61\x46\xfa\xc9\x92\xa1\x64" + "\x05\xf9\x83\x12\x12\x6b\x7a\x37\x22\xf5\xdf\xb7\xdd\x4e\x49\x11\xc1" + "\x42\x6b\x2e\x01\xd0\x4e\x9b\xe6\xdb\x37\x71\x10\x0f\x7d\x7d\x42\x82" + "\xe4\xea\x58\x5f\x36\x46\x24\x1e\x80\x7c\xa6\x4f\x06\xa7\xfa\x9b\x70" + "\x03\xd7\x10\xb8\x01\xd6\x6f\x51\x7d\x2d\x5e\xbd\x74\x08\x72\xde\xba" + "\x13\xd0", + "\x38\xc3\xf4\x4b\xc5\x76\x5d\xe1\xf3\xd1\xc3\x68\x4c\xd0\x9c\xdd\xef" + "\xaf\x29\x8d", + "\xd4\xa7\x9f\x72\x94\x87\x93\x59\x50\xec\x03\x2e\x69\x0a\xb8\xfe\x25" + "\xc4\x15\x8e", + "\x87\x6d\x2f\x33\x4f\x47\x96\x8b\x10\xc1\x03\x85\x9d\x43\x6d\xb8", 1, + 2056, 160 }, + { 96, 256, 128, 92, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "", + "\x56\x10\x08\xfa\x07\xa6\x8f\x5c\x61\x28\x5c\xd0\x13\x46\x4e\xaf", + "\x23\x29\x3e\x9b\x07\xca\x7d\x1b\x0c\xae\x7c\xc4\x89\xa9\x73\xb3", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 96, 256, 128, 93, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "", + "\xc6\x15\x22\x44\xce\xa1\x97\x8d\x3e\x0b\xc2\x74\xcf\x8c\x0b\x3b", + "\x7c\xb6\xfc\x7c\x6a\xbc\x00\x9e\xfe\x95\x51\xa9\x9f\x36\xa4\x21", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 96, 256, 128, 94, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9d\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 95, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9e\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 96, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x1c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 97, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe9\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 98, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\x76\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 99, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd9\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 100, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xda\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 101, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\x71\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 102, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbe\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 103, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\x3f\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 104, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\xa8\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 105, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x73\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 106, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xeb\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 107, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xe8\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 108, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\x6a\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdc", 0, + 0, 128 }, + { 96, 256, 128, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\xdf", 0, + 0, 128 }, + { 96, 256, 128, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\x9d", 0, + 0, 128 }, + { 96, 256, 128, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbf\x88\x72\x32\xea\xb5\x90\x5d", 0, + 0, 128 }, + { 96, 256, 128, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9d\xe8\xfe\xf6\xd8\xab\x1b\xf1\xbe\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\x76\xd8\xab\x1b\x71\xbf\x88\x72\x32\xea\xb5\x90\xdd", 0, + 0, 128 }, + { 96, 256, 128, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9c\xe8\xfe\xf6\xd8\xab\x1b\x71\xbf\x88\x72\x32\xea\xb5\x90\x5d", 0, + 0, 128 }, + { 96, 256, 128, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x63\x17\x01\x09\x27\x54\xe4\x0e\x40\x77\x8d\xcd\x15\x4a\x6f\x22", 0, + 0, 128 }, + { 96, 256, 128, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 256, 128, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 256, 128, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x1c\x68\x7e\x76\x58\x2b\x9b\x71\x3f\x08\xf2\xb2\x6a\x35\x10\x5d", 0, + 0, 128 }, + { 96, 256, 128, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\xb2\x06\x14\x57\xc0\x75\x9f\xc1\x74\x9f\x17\x4e\xe1\xcc\xad\xfa", + "\x9d\xe9\xff\xf7\xd9\xaa\x1a\xf0\xbe\x89\x73\x33\xeb\xb4\x91\xdc", 0, + 0, 128 }, + { 128, 192, 128, 121, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x02\x9e\x0e\x77\x7d\xb0\x92\xb1\x25\x35\xd0\x43\x01\x2f\x09\xba", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xf8\x3c\xee\x46\x73\x36\xe1\xa0\x9b\x75\xf2\x4e\x9b\x43\x85\xc9\x9c" + "\x13\xe6\xaf\x72\x22\x56\xa6\x61\x29\xec\xe9\x61\xfe\x80\x3b\x16\x7b" + "\xad\x20\x6f\x50\x17\xfb", + "\x09\x33\x8a\x42\xf0\xac\xc1\x4f\x97\xc0\x64\xf5\x2f\x5f\x16\x88", 1, + 0, 320 }, + { 128, 192, 128, 122, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xf1\xbe\x3b\x06\xb7\xfe\xac\x07\xe7\xea\xb6\x29\xf5\x56\x04\x7b", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x0b\x32\xb6\x48\xa2\xc2\x8e\x9e\xdd\x7c\xee\x08\xee\xeb\x90\x00\x34" + "\xca\xe7\x21\x5e\x5a\xb1\xe2\x01\xbd\x2e\xed\x10\x32\xc5\xa9\x78\x66" + "\xba\x58\x2a\x34\x58\xa4", + "\x90\xbe\x36\x06\xde\x58\xbd\x77\x8f\xa5\xbe\xff\x4a\x41\x02\xbd", 1, + 0, 320 }, + { 128, 192, 128, 123, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xde\x9e\xb6\x3b\x1d\xae\xd3\x21\xa1\x1b\x75\x47\xcc\x9e\x22\x3c", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x57\x5e\x2e\xce\xc2\xb3\xc7\x2d\x4e\x80\x83\x0d\x0d\x85\x9a\xd9\xe4" + "\x2c\x29\xc4\xa6\x8d\x8d\x9d\x8d\x23\x43\x4d\xe2\xcd\x07\x73\x3b\xe4" + "\x9d\x62\xac\x1a\xe0\x85", + "\x6e\x4d\x63\x96\x12\x5a\x10\xdf\x54\x43\xbd\x0c\xbc\x85\x66\xd1", 1, + 0, 320 }, + { 128, 192, 128, 124, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x40\xbb\x0a\xbe\xbc\x48\x3f\xf6\xd5\x67\x12\x41\xff\x5d\x66\xc6", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x2a\x81\x88\x88\xd1\xf0\x9f\x32\xaa\x7b\xee\xdd\x28\x69\xb4\x46\x57" + "\x5e\x2e\xce\xc2\xb3\xc7\x2d\x4e\x80\x83\x0d\x0d\x85\x9a\xd9\xe4\x2c" + "\x29\xc4\xa6\x8d\x8d\x9d", + "\xdc\x48\x1f\x17\x25\x45\x26\x8e\xff\x63\xab\x04\x90\x40\x3d\xc3", 1, + 0, 320 }, + { 128, 192, 128, 125, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x20\xd5\xcf\x30\x5e\x63\x0a\x8f\x49\xe3\xbb\x4b\xab\x18\xab\xc9", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x96\xd3\x6b\x79\x5f\x8e\x7e\xdf\x6a\x8e\x0d\xbc\xd2\x0d\x6c\x07\x2a" + "\x81\x88\x88\xd1\xf0\x9f\x32\xaa\x7b\xee\xdd\x28\x69\xb4\x46\x57\x5e" + "\x2e\xce\xc2\xb3\xc7\x2d", + "\x8a\x3a\x22\xbf\x25\x92\x95\x8b\x93\x02\x92\xaa\x47\xf5\x90\xe8", 1, + 0, 320 }, + { 128, 192, 128, 126, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x25\x53\x58\xa7\x1a\x0e\x57\x31\xf6\xdd\x6c\xe2\x8e\x15\x8a\xe6", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xcf\xce\x3d\x92\x0f\x0e\x01\xf0\xbb\x49\xa7\x51\x95\x5b\x23\x6d\x1b" + "\x88\x7b\xae\xfd\x25\xc4\x7f\x41\x30\x3c\x46\xd5\xc7\xbf\x9c\xa4\xc2" + "\xc4\x5a\x8f\x1e\x66\x56", + "\x2d\xb9\xdc\x1b\x7f\xd3\x15\xdf\x1c\x95\x43\x24\x32\xfc\xf4\x74", 1, + 0, 320 }, + { 128, 192, 128, 127, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xbb\x76\xe4\x22\xbb\xe8\xbb\xe6\x82\xa1\x0b\xe4\xbd\xd6\xce\x1c", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x69\xa2\x41\x69\x79\x2e\x9a\x07\xf6\xe6\xf4\x73\x6f\xa9\x72\xdc\xcf" + "\xce\x3d\x92\x0f\x0e\x01\xf0\xbb\x49\xa7\x51\x95\x5b\x23\x6d\x1b\x88" + "\x7b\xae\xfd\x25\xc4\x7f", + "\x82\xad\x96\x7f\x7a\xc1\x90\x84\x35\x4f\x69\xa7\x51\x44\x3f\xb2", 1, + 0, 320 }, + { 128, 192, 128, 128, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xdb\x18\x21\xac\x59\xc3\x8e\x9f\x1e\x25\xa2\xee\xe9\x93\x03\x13", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x4e\x44\x17\xa8\x3b\xea\xc1\xeb\x7e\x24\x45\x6a\x05\xf6\xba\x55\x69" + "\xa2\x41\x69\x79\x2e\x9a\x07\xf6\xe6\xf4\x73\x6f\xa9\x72\xdc\xcf\xce" + "\x3d\x92\x0f\x0e\x01\xf0", + "\x47\x2d\x5d\xd5\x82\xdc\x05\xef\x5f\xc4\x96\xb6\x12\x02\x3c\xb2", 1, + 0, 320 }, + { 128, 192, 128, 129, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xf7\xa0\x2e\xcc\xa0\x30\x64\xb2\xef\x3c\xce\x9f\xea\xb7\x9f\x07", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x6f\x8e\x17\x4e\xfc\xa3\x09\x72\x99\xf7\x84\xef\xd4\xca\xff\x0b\xf1" + "\x68\xc3\xe5\x16\x5b\x9a\xd3\xd2\x00\x62\x00\x98\x48\x04\x4e\xef\x8f" + "\x31\xf7\xd2\xfe\xad\x05", + "\xca\xff\x72\x38\x26\xdf\x15\x09\x34\xae\xe3\x20\x1b\xa1\x75\xe7", 1, + 0, 320 }, + { 128, 192, 128, 130, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x69\x85\x92\x49\x01\xd6\x88\x65\x9b\x40\xa9\x99\xd9\x74\xdb\xfd", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xaf\x19\x30\x90\xce\x3d\x43\xa3\x88\xa1\xd2\x94\xa0\x96\x16\x90\x6f" + "\x8e\x17\x4e\xfc\xa3\x09\x72\x99\xf7\x84\xef\xd4\xca\xff\x0b\xf1\x68" + "\xc3\xe5\x16\x5b\x9a\xd3", + "\x3b\x08\x95\x8b\xe1\x28\x6c\x2b\x4a\xcb\xa0\x2b\x36\x74\xad\xb2", 1, + 0, 320 }, + { 128, 192, 128, 131, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x3f\x11\x88\x54\x6c\x65\xed\x0f\xc5\x5e\x75\x03\x2c\x68\xee\x44", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x5d\xec\xcf\x83\x8b\x2c\xf5\xf8\x69\xc9\x0d\x2a\x61\x11\x60\xb1\xe5" + "\x78\xab\x81\x21\xb9\x37\x35\xcb\xa4\xa1\x93\x06\x47\xb8\xc4\xc8\x4b" + "\xf7\x76\x33\x3e\xe4\x5a", + "\xc1\x4d\x52\x20\x8f\x0f\x51\xb8\x16\xa4\x89\x71\xea\xf8\xff\x7e", 1, + 0, 320 }, + { 128, 192, 128, 132, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xa1\x34\x34\xd1\xcd\x83\x01\xd8\xb1\x22\x12\x05\x1f\xab\xaa\xbe", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xd2\xca\xe1\x68\x4a\xa4\x07\xa1\x3a\x2e\x2d\xa5\x35\x7e\x29\xf5\x5d" + "\xec\xcf\x83\x8b\x2c\xf5\xf8\x69\xc9\x0d\x2a\x61\x11\x60\xb1\xe5\x78" + "\xab\x81\x21\xb9\x37\x35", + "\xea\x2d\x01\x80\x99\xcd\x79\x25\xc5\x07\xce\xf0\xce\xdd\xb0\xae", 1, + 0, 320 }, + { 128, 192, 128, 133, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "", + "\x5c\x7d\x3f\x81\xd4\xb5\x05\x5e\xd6\xf8\xdb\x53\x61\x45\x87\xa4", + "\x54\x1b\x83\x5d\xc8\x28\xd5\x41\x07\x3f\x7d\x7d\x75\x04\xeb\xf5", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 128, 192, 128, 134, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "", + "\x6a\x34\x7a\xd1\x19\x0e\x72\xed\xe6\x11\x04\x4e\x74\x75\xf0\xeb", + "\xa3\xf3\x61\x54\x33\x1c\x19\x66\x24\x56\x4b\xc3\x95\xe4\x9c\x3b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 128, 192, 128, 135, + "\xfa\xe2\xa1\x41\x97\xc7\xd1\x14\x00\x61\xfe\x7c\x3d\x11\xd9\xf7\x7c" + "\x79\x56\x2e\x35\x93\xa9\x9b", + "\xbc\x28\x43\x39\x53\x77\x2d\x57\xbb\xd9\x33\x10\x0c\xd4\x7a\x56", + "", "", "", + "\x1b\xb9\x43\x31\xf2\x6c\xad\x24\x03\x6c\xfe\xff\x34\xb8\x9a\xaf", 1, + 0, 0 }, + { 128, 192, 128, 136, + "\xce\xe9\xab\xbc\x26\xb6\x3e\x16\x9f\x0c\xed\x62\x1f\xe2\x1d\x95\x90" + "\x4e\x75\xb8\x81\xd9\x3e\x6b", + "\x1e\x82\x59\xe0\xa4\x3e\x57\x10\x68\xf7\x01\xcd\x20\x64\xfc\x0c", + "", "\x46", "\xdc", + "\xaf\x1f\x55\x35\xb1\x25\xb3\x4f\xc4\x66\x90\x2e\xa4\x0c\xb3\xa2", 1, + 0, 8 }, + { 128, 192, 128, 137, + "\x18\x9f\x0b\xd3\x90\xba\x40\x63\x25\x86\xa4\x5c\x39\x73\x5c\x2b\x87" + "\x11\x33\x29\xc8\x00\xf3\x94", + "\xc8\x44\x42\xd6\x97\x5f\x03\x59\x73\x7d\xe0\xfa\x82\x8f\x95\x8e", + "", "\xb4\xbc\xd7\xb8\xee\xca\x30\x50\xdd\x17\x68\x2c\x6a\x91\x4e", + "\x2a\xab\x5c\x87\xdc\xb4\xa4\xda\xe4\xe9\x75\xdd\xb6\x5a\xab", + "\x6b\x03\xb7\x55\x7c\x71\x31\xe2\x35\x2e\x49\x5d\x54\xe6\x1a\xef", 1, + 0, 120 }, + { 128, 192, 128, 138, + "\xb0\x72\x4f\x15\xdf\x5b\x79\x2c\x2f\x49\xbc\x51\xdf\x0a\xc5\xaa\xd6" + "\x9b\xe0\x03\x09\x81\x61\x3c", + "\x13\xcd\x52\x6e\xc7\x7b\x58\xf6\x2d\x48\xd0\x3f\x8b\x88\xf2\xb8", + "", + "\x8d\xa3\xab\x9c\x3d\x19\x5b\x04\xdf\x45\x2a\xd2\x39\x53\xda\x4d", + "\xd1\x27\xfd\x2e\x67\xc0\x88\x7d\x90\xeb\x92\xb9\x1f\x35\x7d\x97", + "\xeb\x05\xbd\xa9\x37\xfa\xee\xd2\x7f\x88\x33\x29\x5d\x4b\xa5\x59", 1, + 0, 128 }, + { 128, 192, 128, 139, + "\x99\x87\x50\xba\x78\x48\x41\xe4\x0a\x7c\x5b\x03\x98\x57\x32\xb6\x39" + "\x7e\x54\x59\xa3\x84\x39\x54", + "\x1d\x3d\x62\xec\xcd\x8a\xc5\xe8\x96\xf2\x65\x4a\x7f\x60\x6f\xc9", + "", + "\x2f\x60\xca\x34\x94\xa9\x58\xdc\x3e\x6e\xbe\xb5\xd0\xb4\xe6\xdd\xa0" + "\xd0\xc4\x33\x1a\xb9\xc9\x57\xf6\x42\x2a\x51\x00\x87\x8e\xbf", + "\x34\x4c\x2c\xea\x17\xb0\x6c\xb3\xda\x27\x2e\x22\xa2\x2a\x3a\x71\xee" + "\x0e\xaa\x19\x59\xa7\xfa\xcf\xff\x46\x46\x60\xdd\xcc\xed\xd1", + "\xba\xb7\xfb\xf4\x99\xff\x06\xaa\xd5\xf7\x57\xb1\xc1\xa4\xfc\xc0", 1, + 0, 256 }, + { 96, 192, 128, 140, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "", + "\x0b\x4d\xbb\xba\x89\x82\xe0\xf6\x49\xf8\xba\x85\xf3\xaa\x06\x1b", + "\x3f\x87\x5c\x9b\xd7\xd8\x51\x14\x48\x45\x94\x68\xe3\x98\xc3\xb2", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 96, 192, 128, 141, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "", + "\x1a\xe9\x36\x88\xef\x7e\x26\x50\xa9\x34\x2a\xd4\x71\x8b\x27\x80", + "\x21\x0d\xab\xea\x43\x64\xc6\xd5\xb3\x42\x9e\x77\x43\x32\x29\x36", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 96, 192, 128, 142, + "\x50\x19\xeb\x9f\xef\x82\xe5\x75\x0b\x63\x17\x58\xf0\x21\x3e\x3e\x5f" + "\xcc\xa1\x27\x48\xb4\x0e\xb4", + "\xff\x0d\xdb\x0a\x0d\x7b\x36\xd2\x19\xda\x12\xb5", "", "", "", + "\x79\x71\x28\x4e\x6c\x9e\x6a\xac\x34\x6f\xe2\xb7\xa0\xa0\x64\xc2", 1, + 0, 0 }, + { 96, 192, 128, 143, + "\x21\x21\x8a\xf7\x90\x42\x8f\x80\x24\xd3\xe7\xe1\x42\x8c\x9f\xcf\x57" + "\x8c\x21\x66\x36\xd6\x0e\x73", + "\x34\x04\x7b\xc3\x9b\x9c\x60\x83\x84\xdf\xf5\xb8", "", "\xe3", + "\xfe", + "\x2e\x98\x2e\x24\xb8\x1c\xd1\x20\xd3\x5a\x70\xfe\x69\x35\xe6\x65", 1, + 0, 8 }, + { 96, 192, 128, 144, + "\x3a\x8b\xf5\x43\xc4\x80\x92\x56\x32\x11\x82\x45\xbc\xbf\x5d\x01\x52" + "\x2b\x98\x7a\x31\xa3\x3d\xa3", + "\x4e\xbc\x13\xcf\x46\x36\xcc\x7c\x45\xe5\x60\xa7", "", + "\x53\xfc\x72\xe7\x1b\x59\xee\xb3", + "\x99\xf2\xff\x1c\x8a\x44\xe5\xf2", + "\x68\x70\xf1\x04\xdd\xc5\x14\x47\x7b\x40\x03\x36\xfb\x01\x86\x0e", 1, + 0, 64 }, + { 96, 192, 128, 145, + "\x92\xf4\xd2\x67\x2f\xce\xec\x43\x96\x3c\xcf\xfb\x17\xe6\xea\x75\x78" + "\xb1\x14\x18\xb0\x6a\x3b\x82", + "\x6e\x7f\xf7\xf0\x79\x76\x85\xcf\xc4\x4b\x05\xff", "", + "\xc3\xec\x16\xad\xb1\x84\xaf\xfa\x8a\xe9\x73\x8b\xff\xb9\x16", + "\xaf\xe8\xef\x41\x59\x1b\xfc\xc0\x0d\xb3\xc8\x80\xce\xb1\x86", + "\x29\xff\xf7\xf2\x85\x76\x86\x45\xc9\xc8\xbf\x7a\x47\x1c\x93\x93", 1, + 0, 120 }, + { 96, 192, 128, 146, + "\xbc\xb6\xbc\x5e\xe6\x74\x3d\xf1\x39\x6a\x34\x63\x93\x27\xb2\x58\x09" + "\xec\x9c\x81\xdd\x6a\x0c\x0e", + "\xbe\x03\x26\xd2\x3b\xdc\x2c\x64\x64\x8d\x13\xf4", "", + "\x80\x47\x4a\x3a\x3b\x80\x95\x60\xee\xe2\xce\x7a\x7a\x33\xea\x07", + "\x90\x33\x9d\xca\x02\xef\x71\x7f\x16\x03\x99\x4a\xee\x6c\xf6\xd2", + "\xe3\xd3\x3e\x01\xce\x64\xf2\x71\x78\x31\x47\xde\x22\x62\x28\xbc", 1, + 0, 128 }, + { 96, 192, 128, 147, + "\x5e\x1d\x28\x21\x3e\x09\x25\x36\x52\x5b\xba\xe0\x9e\x21\x4a\xf4\xc8" + "\x91\xe2\x02\xb2\xb4\xfa\x4f", + "\xb6\xbe\x6c\xd0\x68\x12\x35\xd8\x26\xaa\x28\xea", "", + "\x53\xd5\x94\x33\xa7\xdb\x7f\x41\xb3\x1c\xcb\x6d\x4a\x2d\x78\x99" + "\x65", + "\xb9\x8e\xd6\x32\x16\x79\x94\x1a\x3e\x52\x18\x34\x29\x66\x86\xad" + "\x98", + "\x9f\x50\xc0\x3e\x05\x5e\x51\x97\x12\xc5\x82\xec\x9d\xb3\x23\x5b", 1, + 0, 136 }, + { 96, 192, 128, 148, + "\x7f\x67\x2d\x85\xe1\x51\xaa\x49\x0b\xc0\xee\xc8\xf6\x6b\x5e\x5b\xee" + "\x74\xaf\x11\x64\x2b\xe3\xff", + "\xb0\x22\x06\x70\x48\x50\x5b\x20\x94\x62\x16\xef", "", + "\xef\x64\x12\xc7\x2b\x03\xc6\x43\xfa\x02\x56\x5a\x0a\xe2\x37\x8a\x93" + "\x11\xc1\x1a\x84\x06\x5f\x80", + "\xad\xdd\x30\x36\x51\x11\x9e\x52\xf6\x17\x0d\xfc\x7a\x91\x50\x64\x25" + "\x3d\x57\x53\x29\x87\xb9\xab", + "\xfa\x04\x84\xf8\xba\xa9\x5f\x5b\x7a\x31\xc5\x6d\x1b\x34\xc5\x8b", 1, + 0, 192 }, + { 96, 192, 128, 149, + "\x96\x9f\xed\x50\x68\x54\x1d\x65\x41\x8c\x2c\x1d\xe8\xfe\x1f\x84\x5e" + "\x03\x60\x30\x49\x6e\x12\x72", + "\x81\x7f\xe5\x1c\x31\xf2\x87\x91\x41\xa3\x43\x35", "\xcb", + "\x3d\x82\x33\x19\x1a\x28\x23\xbf\x76\x7e\x99\x16\x7b\x1d\x4a\xf4\xf4" + "\x84\x84\x58", + "\x0d\x2c\x3a\x3c\x0c\xc4\xb4\x0e\x70\xed\x45\xe1\x88\xe3\x56\xa0\xe1" + "\x53\x3b\x31", + "\x92\x90\x9a\x80\xe9\x05\x40\xe1\x87\x8a\xb5\x9e\xf3\x00\x07\x2b", 1, + 8, 160 }, + { 96, 192, 128, 150, + "\xfa\x5b\x9b\x41\xf9\x3f\x8b\x68\x2c\x04\xba\x81\x6c\x3f\xec\xc2\x4e" + "\xec\x09\x5b\x04\xdd\x74\x97", + "\x62\xb9\xcf\x1e\x92\x3b\xc1\x13\x8d\x05\xd2\x05", + "\x2e\xd8\x48\x71\x53\xe2\x1b\x12", + "\x18\x15\x98\x41\x81\x3a\x69\xfc\x0f\x8f\x42\x29\xe1\x67\x8d\xa7\xc9" + "\x01\x67\x11", + "\xc7\xc1\xcb\xb8\x5c\xe2\xa0\xa3\xf3\x2c\xb9\xef\x01\xad\x45\xec\x11" + "\x18\xb6\x6d", + "\x25\x33\x17\xf9\x8b\xda\xb8\x75\x31\xec\xe2\x04\x75\xcd\x9e\xbb", 1, + 64, 160 }, + { 96, 192, 128, 151, + "\xfb\xfb\x39\x56\x62\x78\x7e\x2d\x25\xa2\xe7\x51\x0f\x81\x8e\x82\x59" + "\x36\xa3\x51\x14\xe2\x37\xc9", + "\x3f\x1a\x1e\x02\xe9\x0a\x4b\xa7\xa1\xdb\x9d\xf2", + "\x74\x31\x8d\x88\x76\x52\x82\x43\xf1\x94\x4b\x73\xeb\x77\xe9\x6e", + "\x29\x52\xa3\xd6\x41\x07\xd5\xcb\xb9\x60\x22\x39\xd0\x5a\x5c\x5c\x22" + "\x2c\xf7\x2b", + "\xec\xf5\xe4\x03\xf1\x9c\x00\x7c\x8d\xa7\xa4\x56\xca\xf0\xa6\xd7\x57" + "\x62\x82\x9b", + "\xe0\x87\x7a\x10\x0f\x9d\xd9\xd6\x79\x5f\x0e\x74\xc5\x6a\x9f\xab", 1, + 128, 160 }, + { 96, 192, 128, 152, + "\x5d\x8e\x9c\x22\x22\x31\x6c\x9e\xd5\xff\x94\x51\x3c\xc9\x57\x43\x6a" + "\xe4\x47\xa6\xe1\xa7\x3a\x29", + "\x08\x02\xae\x86\xc7\x5a\x73\xbf\x79\x56\x15\x21", + "\x5c\xa3\x54\xa4\xcb\x8e\x4f\xc9\x79\x8a\xa2\x09\xad\x4f\x73\x9d\xc7" + "\xc2\x32\xfd\xd1\xf2\x25\x84", + "\x42\xb4\x43\x9e\x1d\x21\x16\xf8\x34\xb9\x1c\x51\x6a\x26\x29\x9d\xf2" + "\x79\x95\x6b", + "\x94\xd8\x44\xd9\x8b\x94\x67\xda\xa7\xe8\xdd\xe7\xf4\x29\x00\x37\x35" + "\x4d\x7f\xb2", + "\x62\x19\x66\x38\x59\x0c\xef\x42\x9d\x6b\x1d\x1a\x59\x83\x9c\x02", 1, + 192, 160 }, + { 96, 192, 128, 153, + "\xcc\xbd\x0f\x50\x98\x25\xa5\xf3\x58\xa1\x4a\xac\x04\x4a\xe2\x82\x6b" + "\xb2\xc9\xea\xaa\xaa\x07\x7f", + "\x91\x89\xa7\x1a\xc3\x59\xb7\x3c\x8c\x08\xdf\x22", "", + "\xa1\xed\x10\x07\xb5\x2e\x36\xec\x0f\x70\x10\x9c\x68\xda\x72\xee\x7b" + "\x67\x5c\x85\x5e\x3e\x49\x56\xd2\xdc\xf9\xd1\x2f\x67\x5d\x69\x33\xf6" + "\x77\xdd\xcc\x58\xfa\xce\x85\x76\x99\xd2\xe3\xd9\x0a\xdc\xb8\xc6\xc5" + "\x7c\x9d\x88\xb5\xdf\xcf\x35\x6d\xe4\xc0\xb6\x3f\x0e", + "\xe9\x91\x5b\xc5\xae\xa6\x3c\x8b\xc0\x14\xf2\xae\x6a\x49\x86\xb0\x31" + "\x15\xff\x1f\x34\xad\x6c\x0a\xcd\x74\xff\xca\x07\xc4\x53\xec\x3f\x3c" + "\xe6\x90\x2d\x5f\xf3\x38\xc5\x88\xa3\x4a\x1c\x3b\x30\xef\x75\x3e\xc7" + "\x00\x15\x72\xcb\xfe\xaf\xe6\x90\xfd\x00\xf5\x9b\x02", + "\xfb\xf1\x9b\x6b\x90\xe2\xd9\xdf\x7e\xad\x0c\x3b\xc6\xe3\x75\xa2", 1, + 0, 512 }, + { 96, 192, 128, 154, + "\xd0\x45\xc6\xeb\x17\x3f\x44\x08\x43\xfa\xec\x3e\x93\x74\x60\x2a\x94" + "\xee\x3f\x71\x76\x31\x22\x08", + "\x98\xe9\x15\x3d\xac\xa2\x52\x2e\x31\x62\xcb\x15", "", + "\x3f\x0b\x30\xdc\x96\x3a\x82\xd1\x82\xc0\x35\xb5\xa8\x23\x06\x0f\x07" + "\xc4\x12\x37\x92\xe6\xce\xe6\xbf\x91\xfe\xa3\xc5\x2f\xa6\x6b\xb6\xa9" + "\x3e\xa6\xcc\xe9\xf4\x81\x3e\xb9\x5b\xf1\x8f\x81\x6c\x00\xad\x4f\xb5" + "\x69\x32\x82\x7a\x39\xef\xb2\xfe\x56\x80\x4e\x60\x4a\x60\x67\x74\xee" + "\x92\xad\x46\xcd\x8c\x17\x2a\x0d\x2b\xde\xa2\xfc\x99\xf6\x7c\xd8\x2c" + "\x60\x24\xc3\x15\xcf\xee\x6d\xbb\x8d\x27\xf7\x45\xc9\xd0\xce\x9b\xf5" + "\xd0\x97\x24\xf4\xbe\xd0\x03\xcf\x39\x47\x83\x48\xb3\x30\x4b\xaa\x4e" + "\xcc\x99\x74\xfc\x4f\x3f\xf9\x3f\x95", + "\x96\x63\xe6\xf9\x8b\x27\x68\x44\x8e\x6d\xd0\xdd\x78\x0e\x14\x56\x68" + "\xaf\x5b\x00\x22\x57\xe3\x53\x21\x38\x68\xc9\xcd\x9f\xd3\xa1\xe9\x42" + "\x75\x30\x32\x75\x41\x77\x5a\x09\x31\x23\x07\x6d\x34\x98\x5d\xb3\xaa" + "\x24\x8c\xd5\x5e\x53\x26\x09\xd1\xa3\x92\x74\xc4\x92\x16\xea\x20\xfb" + "\xab\x71\x9b\x9c\x7e\x31\x0b\x27\x87\x7b\x9a\x33\xd1\xb6\x9a\xb7\x47" + "\xaf\xac\x94\x4d\x1e\x97\xea\x78\x93\x67\x82\x1c\x33\x1f\x00\xb5\xd6" + "\x18\x40\x2b\xfc\x57\x88\x4d\x18\xed\xbd\x60\xc4\xdf\xe2\x18\xc0\x80" + "\x80\xb8\xe3\x47\x9f\xf8\x4b\xdf\xb5", + "\xfc\x2f\xf6\x2a\x41\xbd\xb7\x9a\xfc\x36\x98\x42\xe4\xec\xca\xbf", 1, + 0, 1024 }, + { 96, 192, 128, 155, + "\xe6\x02\x18\x8a\xbf\x6a\x91\xf3\xe2\x58\x83\x8c\xea\x6b\xef\xef\xfc" + "\xf6\x25\x7a\x50\x9c\x3e\x95", + "\x9e\x35\xd3\xef\x18\x97\xc5\xfe\x3f\x64\x72\x04", "", + "\x3b\x9a\x6e\xdc\x44\x84\x8c\x07\x23\x41\xfd\x4a\xf5\x1e\xc1\x16\xac" + "\x32\x8f\x69\xcc\x5a\x33\x54\xe4\x92\x99\xfb\x2e\x5d\x22\xfa\x00\x84" + "\xe3\x0b\x36\xec\xaf\x54\x30\x93\x97\xb2\xb4\x98\xd6\x86\x08\x7f\x34" + "\x57\x69\x8c\x36\x39\xe7\x3c\xa1\x8c\x78\xc3\xe0\x21\xd6\x73\x98\x6c" + "\xfc\x2c\xeb\x4d\x07\xe6\x69\x71\xe9\x76\xf5\x8f\x03\x36\xf8\x2c\x7f" + "\xc0\xd5\x2d\x66\x61\x0f\x26\xca\x3b\xfe\x53\xc0\xb0\x1c\xf7\xc2\x07" + "\x30\x6d\xb9\x04\xc1\xad\x30\x0a\xb9\x5c\x56\xfd\xe8\x20\xa8\xed\xd2" + "\x56\xf2\xb9\x90\x6b\x31\x2b\xf7\xaf\x5e\xf4\xa8\x06\xf6\x18\xdd\xfc" + "\xb6\x71\x79\xb0\x3f\xff\x80\xa2\x45\xc3\x8d\x8f\x4c\xff\x28\x75\xb7" + "\x1a\x0b\xf6\x91\x29\xca\xf9\x71\x21\x46\x2e\x05\x01\xec\x65\x74\xed" + "\xe9\x47\x06\xf4\xa0\x4d\x2f\xb3\x01\xd4\x15\xc2\x2e\xa1\x21\x57\xd2" + "\xe9\x19\xbc\x7a\x01\x69\xa5\xad\x5c\x7b\xb5\x76\x1a\x85\x31\xab\xbe" + "\x77\xd6\x6a\x48\x71\xb3\xf2\x7a\x71\x70\xf0\x99\x04\x4b\x9f\xdc\x50" + "\xa8\xcb\x3b\x89\x42\x52\xa5\x01\xcc\x89\x6a\xc4\x79\x3b\xdb\x47\x8b" + "\xb1\xcb\x99\xc0\x23\x41\xd7\x23\x8d\xd8\xd5\x93\xcf\xda\x02\xf7\xd5" + "\x20\xd7", + "\x16\x71\x83\x66\x16\x75\x67\x76\x25\xbe\xd2\xd5\xf5\x5f\x72\x8d\xab" + "\x80\xd7\xf0\x6f\x62\x9d\x99\xe5\x8b\x45\x06\x9f\xe9\xd7\x42\x8e\x89" + "\x61\x56\x1b\x11\x24\x5c\x70\x9a\xc9\xeb\xb5\xc5\x9a\xc2\xa8\x9d\x83" + "\x75\xd8\xa0\x1d\x84\x9c\x77\x33\xa1\xb4\x82\x52\x99\x27\xe3\xf1\xa1" + "\xa5\x3f\x63\xa4\xbe\x08\xa1\x1c\x94\x1c\x63\x4c\xd4\x03\x73\xc4\x2f" + "\xfb\x24\x49\xc6\x41\xbc\x9e\x39\xea\xfb\xcf\x9c\x0f\xba\x67\x7e\x36" + "\x49\x6f\x73\xfc\x70\xaa\x09\x72\x22\x49\x01\xab\x04\xb0\xa1\x96\xab" + "\x74\x52\x62\x02\x1b\x23\x13\xa8\x46\x41\x87\xfe\xce\xc4\x3a\xdb\x40" + "\x62\x58\xbd\xdc\xd8\xc9\xd0\x4d\xc2\xae\x29\xe6\x5d\x54\xa8\x9d\xd0" + "\xf1\x75\x2d\x6d\x95\x0d\xbf\x7d\xa4\xde\xa0\xa7\xb9\x46\x55\x79\x50" + "\x3f\xc8\xec\x44\x51\xf4\xb3\x98\x78\xac\x47\x54\xa1\xaa\xf7\xb0\xb7" + "\x3f\xee\x11\x21\x3c\xb8\xe6\x01\xfc\x60\x39\x39\x3f\x72\xe0\xe0\x79" + "\xee\x97\xec\xc6\x10\x24\x17\x57\xda\x2d\xb2\xf5\x1d\x5e\xd1\x21\x48" + "\x15\x40\xef\xf4\x72\x87\x74\x4d\xac\x43\x37\x5c\x4f\x48\xa4\x6a\xf7" + "\x01\x90\x45\x3a\x17\xc3\xc7\x8d\x73\x5b\xa1\xd1\xfc\x76\xa3\x30\xe6" + "\xcb\xed", + "\xc7\x20\x35\x31\x4f\x43\xd2\x56\xf8\xd8\x45\xeb\x69\x6b\xd9\x43", 1, + 0, 2056 }, + { 96, 192, 128, 156, + "\x55\xa4\xca\x52\x64\x43\x35\x7a\xc7\xc8\x96\xd9\xa6\x7c\xf7\xd4\x67" + "\xf6\x92\x1d\x69\x00\x2d\x3a", + "\xdb\xa2\x33\xcc\xbc\x79\x92\xe6\x4e\x82\xcf\xa3", + "\xdf\x73\x7c\xd7\x7d\x31\xeb\x90\x97\xa1\x7c\x31\xb4\xc9\x28\x89\xef" + "\x1f\x32\xb7\x46\x4e\x26\x20\xe9\x00\x71\x92\xea\x67\x5b\x9a\xd6\x91" + "\x05\x27\xff\xec\xee\x24\x52\xbe\x02\x48\xfa\xb7\x56\x08\xc7\xfd\xca" + "\x08\xe8\x65\x80\x32\x2a\xac\x1d\x6a\x11\xb9\x6e\xcf", + "\x4e\x56\xd1\xea\x53\x8c\xf4\x9c\xad\x49\x95\x9e\x88\x4e\xb5\x40\xc8" + "\x46\x55\x6c", + "\x3f\x57\xec\x1b\x41\x4f\x74\x81\x8f\xea\xd9\xf3\x5a\xa1\x67\x94\x02" + "\xc3\xe7\x50", + "\x97\xb8\x9b\x29\x14\x19\xe3\x2c\xf6\x54\xea\x63\x0a\x3a\xd0\x14", 1, + 512, 160 }, + { 96, 192, 128, 157, + "\xf3\x81\xd0\xff\xd3\x37\x3a\x1a\xa0\x2e\xdd\x1d\x7f\xa7\x48\xe9\x19" + "\x08\xfe\x53\x4b\xef\x73\xd2", + "\x10\xaa\xec\x0d\xe4\xad\x75\x37\x6b\xe9\xfd\x41", + "\x77\x39\xaa\xd7\x39\x9d\x9c\x0f\x0a\x3c\x95\xb4\x03\x88\x8f\x00\x72" + "\xd9\x4a\xcb\x76\xff\x57\x6e\x05\xf4\xa0\x63\x12\x0b\x84\xe7\x22\xb4" + "\xd5\xcd\x43\xa5\x8e\x4a\xba\xb4\x44\xcb\x8c\xed\x11\x2f\x3d\xbd\x89" + "\x93\xb8\x31\xc3\x9b\x4e\xdb\x76\xe9\x2e\xb3\x3e\xe2\x4c\x59\x22\xb5" + "\x65\x52\x68\x5f\x3b\x0f\x4c\xf2\x2e\x0e\x11\x62\x8f\x6a\x3d\x33\xef" + "\xf9\xde\xf7\xec\x52\x71\x12\xdf\xaf\xcf\x12\x28\x14\xe3\xd1\xaa\xf6" + "\x6c\x3f\x97\x05\x26\x51\x10\x88\xbf\xfe\xf8\x10\x1d\x1c\xef\x83\x32" + "\x68\xff\x80\x38\x7d\xf3\x05\x57\xf7", + "\x65\x3a\x3f\x03\x3c\x27\x75\xe0\x8f\xef\x73\xcf\x80\xf5\xe2\x69\x9f" + "\xb3\x60\xcb", + "\x55\x65\xc6\xd0\x9c\x4c\x92\x4d\x61\xc0\xef\x80\x8f\xb0\xea\x14\x4f" + "\xfb\x47\x38", + "\x12\xb7\x2e\xc1\xd9\xc3\x2f\xb2\x2c\x13\xc4\x0b\x33\x79\x6f\xa9", 1, + 1024, 160 }, + { 96, 192, 128, 158, + "\x8f\x27\xb1\xc3\xb3\xd7\x02\x3c\x76\xee\x66\xc7\x68\xa3\xe9\x2d\x49" + "\x71\xe2\x5f\x72\x9d\x87\x88", + "\x12\x44\x40\x40\xca\xed\xe6\x72\x85\xe4\x90\xd7", + "\x58\xfd\x02\xac\x23\xec\x7f\xa5\xb9\x46\x0f\x60\xbf\xc8\x5b\x4b\xeb" + "\xba\x70\x03\x9a\x8f\x83\x26\x1d\x6c\xc4\xf5\x60\x10\x7c\x10\xbc\x69" + "\x54\x8a\x5d\x61\x52\x88\x2f\xb4\x65\xfd\x59\xfb\x81\x64\xd7\xc9\x45" + "\x23\xc3\xdd\x42\x06\xd3\x30\x64\xf5\x19\x1b\xd3\x1f\x0c\x48\xfe\x03" + "\xd7\x46\x0e\x99\x5c\x93\x17\x5b\x57\xcb\x03\xf5\x87\x11\xad\xc9\x46" + "\x32\x03\x1c\x43\x05\x27\x23\x67\xb4\x28\x9c\x72\x5d\x9c\xb7\xae\x9b" + "\xa9\x96\xb3\xa0\x79\x17\x45\x08\xc1\xea\xe8\x16\x2a\x0b\xac\x44\x6c" + "\x1e\x53\xfe\x0c\x40\x2b\x69\x12\xdf\xd6\x70\x2a\xdd\xcc\xad\xa3\x0a" + "\x5c\x01\x0f\xc2\x2c\x2c\x75\xe4\x32\x26\x37\x8e\xc7\xf4\xb3\xb7\x1c" + "\xcc\x71\xf3\x2a\xb1\xad\xc8\x77\xcc\x7b\x0a\x18\x0c\x75\xd3\x85\xc0" + "\xf7\x1a\x0b\x29\x1a\x1c\xcc\xf4\xbe\x47\xe2\x72\x24\x9d\x61\xff\xbf" + "\x05\x9c\x4f\x7b\xe7\x4e\xba\x07\xd5\xe1\xbe\x3a\x74\x38\x45\x8a\x61" + "\x1f\xe5\x8c\xee\x4f\x94\x6e\x25\xde\xe0\x3e\x64\x85\x23\x55\x66\xf2" + "\x0e\xd5\x55\xbe\x32\xcd\x57\xa9\x4e\x52\x2d\x21\x68\xea\xe2\x3c\x45" + "\x87\x37\x1a\x2d\x14\x5f\x41\x8c\x59\xe7\xbb\xc4\x64\xa3\xbd\x88\xb8" + "\x91\x9b", + "\x0d\xf6\xe7\x50\x09\x2b\x9a\xc5\x76\xdd\xe6\x60\x06\xa4\xca\xb2\x11" + "\x6e\xee\x21", + "\xc6\x87\x7b\x03\x55\x2e\x97\xd9\xa1\xe6\x55\x7f\x90\xdc\x7a\xdd\xe1" + "\x5a\x2f\x43", + "\x25\x36\x27\x2b\xee\x74\x46\x82\x00\x41\x85\x4e\x10\xb4\x9a\x03", 1, + 2056, 160 }, + { 96, 192, 128, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb5\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb6\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\x34\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe5\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 163, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\xdb\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 164, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2e\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 165, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2d\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 166, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\xcc\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 167, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x79\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 168, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\xf8\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 169, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xd3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 170, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x59\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 171, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0c\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 172, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0f\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 173, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x8d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 174, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x65", 0, + 0, 128 }, + { 96, 192, 128, 175, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x66", 0, + 0, 128 }, + { 96, 192, 128, 176, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\x24", 0, + 0, 128 }, + { 96, 192, 128, 177, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x78\xf3\x58\xda\x0d\x99\xcb\xe4", 0, + 0, 128 }, + { 96, 192, 128, 178, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb5\xe4\x4c\x5b\x2f\xe9\x0e\x4c\x79\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 179, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\xdb\x2f\xe9\x0e\xcc\x78\xf3\x58\xda\x0d\x99\xcb\x64", 0, + 0, 128 }, + { 96, 192, 128, 180, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb4\xe4\x4c\x5b\x2f\xe9\x0e\xcc\x78\xf3\x58\xda\x0d\x99\xcb\xe4", 0, + 0, 128 }, + { 96, 192, 128, 181, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\x4b\x1b\xb3\xa4\xd0\x16\xf1\xb3\x87\x0c\xa7\x25\xf2\x66\x34\x9b", 0, + 0, 128 }, + { 96, 192, 128, 182, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, 128 }, + { 96, 192, 128, 183, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, 128 }, + { 96, 192, 128, 184, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\x34\x64\xcc\xdb\xaf\x69\x8e\xcc\xf8\x73\xd8\x5a\x8d\x19\x4b\xe4", 0, + 0, 128 }, + { 96, 192, 128, 185, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b", "", + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x45\x82\x56\x84\x2d\xfd\x29\x7f\x30\xbd\x2f\x8f\x15\xc9\x2d\xb0", + "\xb5\xe5\x4d\x5a\x2e\xe8\x0f\x4d\x79\xf2\x59\xdb\x0c\x98\xca\x65", 0, + 0, 128 }, + { 128, 256, 128, 186, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x5c\x2e\xa9\xb6\x95\xfc\xf6\xe2\x64\xb9\x60\x74\xd6\xbf\xa5\x72", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x28\xe1\xc5\x23\x2f\x4e\xe8\x16\x1d\xbe\x4c\x03\x63\x09\xe0\xb3\x25" + "\x4e\x92\x12\xbe\xf0\xa9\x34\x31\xce\x5e\x56\x04\xc8\xf6\xa7\x3c\x18" + "\xa3\x18\x30\x18\xb7\x70", + "\xd5\x80\x8a\x1b\xd1\x1a\x01\x12\x9b\xf3\xc6\x91\x9a\xff\x23\x39", 1, + 0, 320 }, + { 128, 256, 128, 187, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x57\xb3\xa8\x1f\x2c\x36\xb6\xb0\x65\x77\xca\x0f\xba\xb8\xfa\x8e", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xcc\xee\xbe\xb4\xfe\x4c\xd9\x0c\x51\x4e\x52\xd2\x32\x7a\x2e\xcd\x75" + "\x39\x36\x61\x00\x6c\xf2\x47\x6d\x86\x20\x14\x9a\xef\x3d\x1c\xdc\xe4" + "\x91\xff\xf3\xe7\xa7\xa3", + "\x81\x32\xe8\x65\xb6\x9d\x64\xef\x37\xdb\x26\x1f\x80\xcb\xbe\x24", 1, + 0, 320 }, + { 128, 256, 128, 188, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xce\x20\xa7\xe8\x70\x69\x6a\x5e\x68\x53\x3c\x46\x5b\xad\x2b\xa1", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x4f\x43\x50\x56\x5d\x91\xd9\xaa\x8c\x5f\x40\x48\x55\x04\x92\xad\x6d" + "\x6f\xda\xbf\x66\xda\x5d\x1e\x2a\xf7\xbf\xe1\xa8\xaa\xda\xa0\xba\xa3" + "\xde\x38\xa4\x1d\x97\x13", + "\x15\x5d\xa6\x44\x1e\xc0\x71\xef\x2d\x8e\x6c\xff\xba\xcc\x1c\x7c", 1, + 0, 320 }, + { 128, 256, 128, 189, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x91\x8e\x3c\x19\xdb\xdf\xee\x2d\xb1\x81\x56\xc5\xb9\x3f\x3d\x75", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x83\x16\xa5\x31\x67\xb6\xde\x1a\x75\x75\x70\x06\x93\xff\xef\x27\x4f" + "\x43\x50\x56\x5d\x91\xd9\xaa\x8c\x5f\x40\x48\x55\x04\x92\xad\x6d\x6f" + "\xda\xbf\x66\xda\x5d\x1e", + "\x6c\x57\x4a\xa6\xa2\x49\x0c\xc3\xb2\xf2\xf8\xf0\xff\xbc\x56\xc4", 1, + 0, 320 }, + { 128, 256, 128, 190, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x71\x7d\x90\x0b\x27\x04\x62\xb9\xdb\xf7\xe9\x41\x9e\x89\x06\x09", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x51\x75\x92\x75\x13\xe7\x51\xeb\x30\x9f\x45\xbc\x2e\xf2\x25\xf2\x83" + "\x16\xa5\x31\x67\xb6\xde\x1a\x75\x75\x70\x06\x93\xff\xef\x27\x4f\x43" + "\x50\x56\x5d\x91\xd9\xaa", + "\x80\x82\xa7\x61\xe1\xd7\x55\x34\x4b\xf2\x96\x22\x14\x4e\x7d\x39", 1, + 0, 320 }, + { 128, 256, 128, 191, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xec\xd5\x21\x20\xaf\x24\x0e\x9b\x4b\xf3\xb9\xd1\xee\xb4\x94\x34", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x36\xb3\xfb\xec\xd0\x91\x78\xd0\x45\x27\xfb\x37\x54\x4f\x55\x79\xd2" + "\x0d\x60\xa4\x12\x66\xf6\x85\xc4\x80\x98\xe1\xa5\x28\x04\xca\x38\x7d" + "\x90\x70\x9d\x32\x68\xdd", + "\x03\x3e\x0e\xf2\x95\x3e\xbf\xd8\x42\x57\x37\xc7\xd3\x93\xf8\x9a", 1, + 0, 320 }, + { 128, 256, 128, 192, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xb3\x7b\xba\xd1\x04\x92\x8a\xe8\x92\x21\xd3\x52\x0c\x26\x82\xe0", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x16\x92\x9b\x77\x30\x51\xf1\x2b\x0a\xda\xc9\x5f\x65\xe2\x1a\x7f\x36" + "\xb3\xfb\xec\xd0\x91\x78\xd0\x45\x27\xfb\x37\x54\x4f\x55\x79\xd2\x0d" + "\x60\xa4\x12\x66\xf6\x85", + "\xca\x44\x8b\xb7\xe5\x2e\x89\x7e\xca\x23\x4e\xf3\x43\xd0\x57\xd0", 1, + 0, 320 }, + { 128, 256, 128, 193, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x53\x88\x16\xc3\xf8\x49\x06\x7c\xf8\x57\x6c\xd6\x2b\x90\xb9\x9c", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x6d\x3f\xae\xfa\xf6\x91\xd5\x81\x63\x84\x6f\x8d\x4b\x9f\xfd\x59\x16" + "\x92\x9b\x77\x30\x51\xf1\x2b\x0a\xda\xc9\x5f\x65\xe2\x1a\x7f\x36\xb3" + "\xfb\xec\xd0\x91\x78\xd0", + "\x84\xf4\x97\x40\xe6\x75\x7f\x63\xdd\x0d\xf7\xcb\x76\x56\xd0\xef", 1, + 0, 320 }, + { 128, 256, 128, 194, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xd1\x0e\x63\x19\x43\xcd\x3b\xda\xba\xba\xb2\xbb\xd1\x39\x51\xc0", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xd6\x01\x96\xc2\xd1\x4f\xcf\x30\xc0\x99\x1d\x27\x21\xdd\xc5\x2d\x38" + "\x5f\x40\x7a\x16\x69\x1d\xad\xe8\x2c\x90\x23\xc8\x55\xfd\x8e\x2e\x8f" + "\xbb\x56\x21\x02\xf0\x18", + "\x87\x7e\x15\xd9\x88\x9e\x69\xa9\x9f\xcc\x6d\x72\x74\x65\xc3\x91", 1, + 0, 320 }, + { 128, 256, 128, 195, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x8e\xa0\xf8\xe8\xe8\x7b\xbf\xa9\x63\x68\xd8\x38\x33\xab\x47\x14", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x94\x8f\xbc\xec\xa1\x2a\x6e\x4f\xab\xb7\x9b\x6d\x96\x5e\x33\x6f\xd6" + "\x01\x96\xc2\xd1\x4f\xcf\x30\xc0\x99\x1d\x27\x21\xdd\xc5\x2d\x38\x5f" + "\x40\x7a\x16\x69\x1d\xad", + "\xcd\x57\x57\x62\x69\x45\x97\x6b\xa9\xf0\x26\x4b\xd6\xbe\xe8\x94", 1, + 0, 320 }, + { 128, 256, 128, 196, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x7b\x2d\xf4\xfb\xed\x1d\xe2\x72\x7e\xb2\x48\x98\xe5\xde\xab\xb9", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\xa1\xa0\x12\x06\x60\xff\x52\xe6\xb1\x70\x0b\x12\xc5\x4d\x2d\x33\xb9" + "\x4b\x00\xcd\x78\x82\xd8\x85\x7d\x84\xe6\xe1\x83\xa1\xde\xa6\xee\x85" + "\xa7\xda\x84\xfb\xc3\x5d", + "\xb0\x15\xd7\x2d\xa6\x2c\x81\xcb\x4d\x26\x72\x53\xb2\x0d\xb9\xe5", 1, + 0, 320 }, + { 128, 256, 128, 197, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x24\x83\x6f\x0a\x46\xab\x66\x01\xa7\x60\x22\x1b\x07\x4c\xbd\x6d", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00", + "\x5e\x34\x34\xb4\x5e\xdb\xf0\xd1\xf6\xe0\x2d\x11\x44\xdb\xf8\x67\xa1" + "\xa0\x12\x06\x60\xff\x52\xe6\xb1\x70\x0b\x12\xc5\x4d\x2d\x33\xb9\x4b" + "\x00\xcd\x78\x82\xd8\x85", + "\xee\x74\xcc\xb3\x0d\x64\x9e\xbf\x69\x16\xd0\x5a\x7d\xbe\x56\x96", 1, + 0, 320 }, + { 128, 256, 128, 198, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "", + "\x8d\x74\xf1\xc9\x72\x43\xd3\x62\x57\x7f\xf3\x76\xc3\x93\xd2\xdc", + "\x26\x5c\x42\xe2\xb9\x6e\xa1\xde\x9c\x24\xf7\x18\x2e\x33\x73\x90", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 0, 128 }, + { 128, 256, 128, 199, + "\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff\x10" + "\x21\x32\x43\x54\x65\x76\x87\x98\xa9\xba\xcb\xdc\xed\xfe\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "", + "\x88\x4d\xf0\xe7\x6f\x3c\xe2\x27\xbf\x95\x95\xd1\x03\x82\x5a\x46", + "\x98\x8f\x47\x66\x8e\xa6\x50\xcb\xaa\x67\x14\x71\x1a\xbe\x26\x8d", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 0, 128 }, + { 128, 256, 128, 200, + "\xb4\xcd\x11\xdb\x0b\x3e\x0b\x9b\x34\xea\xfd\x9f\xe0\x27\x74\x69\x76" + "\x37\x91\x55\xe7\x61\x16\xaf\xde\x1b\x96\xd2\x12\x98\xe3\x4f", + "\x00\xc4\x9f\x4e\xbb\x07\x39\x3f\x07\xeb\xc3\x82\x5f\x7b\x08\x30", + "", "", "", + "\x30\x6f\xe8\xc9\x64\x5c\xc8\x49\x82\x3e\x33\x3a\x68\x5b\x90\xb2", 1, + 0, 0 }, + { 128, 256, 128, 201, + "\xb7\x79\x7e\xb0\xc1\xa6\x08\x9a\xd5\x45\x2d\x81\xfd\xb1\x48\x28\xc0" + "\x40\xdd\xc4\x58\x9c\x32\xb5\x65\xaa\xd8\xcb\x4d\xe3\xe4\xa0", + "\x0a\xd5\x70\xd8\x86\x39\x18\xfe\x89\x12\x4e\x09\xd1\x25\xa2\x71", + "", "\xed", "\x3f", + "\xfd\x8f\x59\x3b\x83\x31\x4e\x33\xc5\xa7\x2e\xfb\xeb\x70\x95\xe8", 1, + 0, 8 }, + { 128, 256, 128, 202, + "\x4c\x01\x0d\x95\x61\xc7\x23\x4c\x30\x8c\x01\xce\xa3\x04\x0c\x92\x5a" + "\x9f\x32\x4d\xc9\x58\xff\x90\x4a\xe3\x9b\x37\xe6\x0e\x1e\x03", + "\x2a\x55\xca\xa1\x37\xc5\xb0\xb6\x6c\xf3\x80\x9e\xb8\xf7\x30\xc4", + "", "\x2a\x09\x3c\x9e\xd7\x2b\x8f\xf4\x99\x42\x01\xe9\xf9\xe0\x10", + "\x04\x13\x41\x07\x8f\x04\x39\xe5\x0b\x43\xc9\x91\x63\x51\x17", + "\x5b\x8a\x2f\x2d\xa2\x0e\xf6\x57\xc9\x03\xda\x88\xef\x5f\x57\xbb", 1, + 0, 120 }, + { 128, 256, 128, 203, + "\xe7\xf7\xa4\x8d\xf9\x9e\xdd\x92\xb8\x1f\x50\x86\x18\xaa\x96\x52\x6b" + "\x27\x9d\xeb\xd9\xdd\xb2\x92\xd3\x85\xdd\xba\xe8\x0b\x22\x59", + "\x7e\xe3\x76\x91\x0f\x08\xf4\x97\xaa\x6c\x3a\xa7\x11\x36\x97\xfd", + "", + "\x5e\x51\xdb\xbb\x86\x1b\x5e\xc6\x07\x51\xc0\x99\x6e\x00\x52\x7f", + "\x46\x94\x78\xd4\x48\xf7\xe9\x7d\x75\x55\x41\xaa\x09\xad\x95\xb0", + "\x25\x4a\xda\x5c\xf6\x62\xd9\x0c\x5e\x11\xb2\xbd\x9c\x4d\xb4\xc4", 1, + 0, 128 }, + { 128, 256, 128, 204, + "\x4f\x84\x78\x2b\xfb\xb6\x4a\x97\x3c\x3d\xe3\xdc\xfa\x34\x30\x36\x7f" + "\xd6\x8b\xc0\xb4\xc3\xb3\x1e\x5d\x7c\x81\x41\xba\x3e\x6a\x67", + "\x5d\x1b\xde\x6f\xa0\x99\x4b\x33\xef\xd8\xf2\x3f\x53\x12\x48\xa7", + "", + "\x78\xcb\x66\x50\xa1\x90\x8a\x84\x21\x01\xea\x85\x80\x4f\xed\x00\xcc" + "\x56\xfb\xda\xfa\xfb\xa0\xef\x4d\x1c\xa6\x07\xdc\xae\x57\xb6", + "\xcb\x96\x02\x01\xfa\x5a\xd4\x1d\x41\xd1\xc2\xc8\x03\x7c\x71\xd5\x2b" + "\x72\xe7\x6b\x16\xb5\x89\xd7\x1b\x97\x66\x27\xc9\x73\x4c\x9d", + "\x8d\xfc\xe1\x64\x67\xc3\xa6\xeb\xb3\xe7\x24\x2c\x9a\x55\x19\x62", 1, + 0, 256 }, + { 120, 128, 128, 205, + "\x34\xc7\x4e\x28\x18\x29\x48\xe0\x3a\xf0\x2a\x01\xf4\x6e\xb4\xf7", + "\xb0\xa7\x31\x19\xa9\x7d\x62\x38\x06\xb4\x9d\x45\xdd\xf4\xc7", "", + "\xfe\x82\xba\x66\xcf\x2e\x26\x57\x41\xf2\xc8\x6c", + "\x2b\xc3\xef\x8e\x74\x02\xb4\x63\x1f\x48\xe9\xbe", + "\x4b\x6f\x6f\x5b\xe2\x91\xa9\x0b\x9e\x93\xa8\xa8\x2d\xdb\xc8\xd8", 1, + 0, 96 }, + { 160, 128, 128, 206, + "\x55\xcb\x7c\xac\x77\xef\xe1\x8a\x1e\xa3\xb3\x0c\x65\xf3\xf3\x46", + "\xe2\x2b\x6b\x14\x4a\xb2\x6b\x57\x81\x31\x6e\x7a\x42\xa7\x62\x02\xac" + "\x4b\x22\x78", + "", "\x2f\x3d\x11\xea\x32\xbf\x5b\xc7\x2c\xbe\x2b\x8d", + "\x4f\xe1\x3e\xf2\x9f\x11\x8f\x85\xa6\x31\x88\xf8", + "\x05\x97\x5b\x17\x53\x16\xdf\x80\x45\x88\x9f\x43\xe0\xc8\x57\xe0", 1, + 0, 96 }, + { 120, 192, 128, 207, + "\x66\xf7\x5a\xcb\xd8\xd3\xac\xf7\xaf\x47\xd1\x3e\x83\x84\xc2\x80\x9d" + "\x6b\x91\x50\x3a\x7f\x29\x4b", + "\xed\xf9\x3e\x16\x29\x4f\x15\xed\xed\x83\x80\x8f\x09\x32\x0e", "", + "\xa9\x00\xc8\x6b\x6b\x7e\x0e\x55\x63\xf8\xf8\x26", + "\x9a\xf1\xa0\x22\xc6\x1c\x43\x15\xaa\x0e\x92\x3e", + "\x20\x52\x9b\xff\x3c\x59\x22\x2e\xc3\x33\x53\xaf\x33\x7b\x1d\x40", 1, + 0, 96 }, + { 160, 192, 128, 208, + "\xef\x2e\x29\x9d\xd4\xec\xd7\xe3\xb9\xcc\x62\x78\x09\x22\xcc\x2c\x89" + "\xf7\x88\x40\x56\x4d\x12\x76", + "\x13\x0c\x14\xc8\x39\xe3\x5b\x7d\x56\xb3\x35\x0b\x19\x4b\x0d\xa3\x42" + "\xe6\xb6\x5d", + "", "\x03\xf5\x95\x79\xb1\x44\x37\x19\x95\x83\x27\x0e", + "\x07\x3a\x52\x91\xb1\x1d\xf3\x79\xf3\x1b\x4f\x16", + "\x17\x20\x59\x99\x49\x1b\xd4\xc1\xd6\xc7\xec\x3e\x56\x77\x9c\x32", 1, + 0, 96 }, + { 120, 256, 128, 209, + "\xe9\x8b\x06\x69\xa6\x45\xeb\x14\xcd\x06\xdf\x69\x68\xfc\x5f\x10\xed" + "\xc9\xf5\x4f\xee\xd2\x64\xe3\xd4\x10\xcd\xc6\x1b\x72\xef\x51", + "\x17\xca\x25\x0f\xb7\x33\x87\x75\x56\x26\x32\x23\xea\xdd\xe1", "", + "\xf3\x84\xb3\xed\x7b\x27\x46\x41\xf5\xdb\x60\xcf", + "\xfc\x21\x36\x02\xaa\x42\x3b\x87\xd7\xc2\xa8\x74", + "\x36\xb1\x5b\xab\x69\x23\xb1\x72\x18\xfe\x1c\x24\x04\x8e\x23\x91", 1, + 0, 96 }, + { 160, 256, 128, 210, + "\x84\x9b\x3e\x6b\x8c\xdd\x85\xbd\xcf\xb8\xeb\x70\x1a\xa5\x52\x2a\xe2" + "\x34\x0f\xbe\x52\x14\xe3\x89\x62\x2c\xef\x76\x97\x92\x25\xc4", + "\x0f\x9d\x6e\xd7\xee\xf3\x62\xdf\xa4\xa7\xdf\xa5\xc0\xf7\x4c\x5b\x27" + "\xbd\x4e\xbf", + "", "\x8c\x55\x64\xe5\x30\x51\xc0\xde\x27\x31\x99\xb4", + "\xc1\xd7\x62\x33\xe8\xc5\x04\x2e\x92\xbf\x8d\x32", + "\x7c\xf0\x36\xd2\x35\xd3\xb2\xdd\x34\x9a\x8c\x80\x4b\x65\x14\x4a", 1, + 0, 96 }, + { 256, 128, 128, 211, + "\x59\x27\xba\xe7\x48\xbb\x69\xd8\x1b\x5a\x72\x4e\x0a\x16\x56\x52", + "\x36\x5e\x0b\x96\x93\x2b\x13\x30\x6f\x92\xe9\xbb\x23\x84\x71\x65\xbc" + "\xbf\x5d\x35\xe4\x5a\x83\xd7\x5c\x86\xec\xca\x70\x13\x1f\x4c", + "", "\x31\x6b\xf9\x9b\xfa\xfc\x76\xf1\xbf\xc0\xb0\x3c", + "\x53\x48\xaf\x57\xfa\xfe\x24\x85\xb4\x3f\x2b\xc4", + "\x01\x9a\x96\xc5\x37\x3c\x03\x16\x26\xb6\xc0\x30\x0d\x4c\xf7\x8b", 1, + 0, 96 }, + { 512, 128, 128, 212, + "\xdb\xd3\x67\x6f\x29\x34\x09\x27\x3f\x27\xb3\x75\xe0\x37\x93\xa3", + "\x96\x7f\xa7\xc9\x90\xeb\x2b\xec\xbd\x45\x08\x35\xe2\x8e\xa3\xa9\x00" + "\x0c\x72\x16\x28\x5c\xfa\x76\x96\xe8\xc3\xda\xc3\xce\x95\x2a\x1f\xe6" + "\x38\xd7\xc8\xc7\x3e\x1d\x70\x8d\xce\x01\xb5\xa2\x0f\xcc\x9a\xa0\x11" + "\x94\x9d\x2a\x83\x5f\x77\x74\x23\xc1\x72\xfa\x3a\xa0", + "", "\x62\x5e\xfe\xdb\x8b\x7f\x1a\xa6\x22\x38\xa8\xf2", + "\xf5\x59\xb7\x0f\xe1\x14\x9c\xb3\x44\x06\xa2\xc7", + "\x94\x18\x0d\xdb\x7b\xb1\x99\x5a\xbe\x02\x19\xea\xb5\xce\x23\x2f", 1, + 0, 96 }, + { 1024, 128, 128, 213, + "\x7e\x5a\x39\xdc\xda\x7e\x06\x69\x88\xf1\x9a\xdf\x4d\xe4\xd5\x01", + "\x49\x43\x56\xc3\x45\x9d\x60\xe3\xa8\x34\x33\xc9\xbc\xf2\xc0\x45\x4a" + "\x76\x3e\x49\x6e\x4e\xc9\x9b\xfb\xe4\xbb\xb8\x3a\x4f\xda\x76\xb5\x42" + "\x21\x38\x99\xdc\xf5\x52\x1c\xd9\xbb\xbe\x5d\x11\x54\x5b\xda\x44\xa3" + "\xf4\xa6\x81\xce\x28\x43\xac\xea\x73\x0d\x83\xd3\x93\x0e\xa3\x09\x91" + "\xee\x1a\x68\xeb\xf6\xd1\xa5\xa4\x0f\x9b\x02\xa1\xaa\xb0\x91\x29\x8d" + "\xf8\xdd\x68\x9d\xc7\x61\x3b\xcb\xff\x94\xd3\x5f\x2c\xa4\x33\x77\xd8" + "\x16\x18\x56\x2b\xcf\x65\x73\x41\x1e\xc9\xbc\x97\xc5\xa6\x27\x6b\x55" + "\x40\x54\xc0\xfa\x78\x70\x73\xd0\x67", + "", "\xb0\x47\x29\xb4\xad\xba\xac\x63\xc2\xaa\xf8\xd8", + "\x52\x91\xdd\x4d\xa9\x1c\xcc\x2e\x77\x30\x6d\x83", + "\xa7\xf7\xb2\x1a\x3b\x7e\xce\x50\x9e\x92\x26\x47\xfd\x90\x5f\x06", 1, + 0, 96 }, + { 2056, 128, 128, 214, + "\xea\xc3\xf2\x8c\xd9\x37\xff\x29\xeb\x61\x58\xa3\x72\x1b\x51\x45", + "\x6f\xd2\x60\xbb\xa8\x73\x39\x53\x9c\x37\xdc\x68\xfd\xc3\x65\x6f\x63" + "\xc8\x30\x28\xcb\x8a\xdc\xb5\x31\x08\x5e\x98\xbd\x57\x0c\x6b\x73\x5d" + "\x0c\xc4\xb4\xb9\x24\x69\x60\x00\xa2\xd8\x93\x62\x1a\xe6\x4d\xcc\xe9" + "\x92\xb5\x62\xb8\x9a\x52\x85\x64\x3a\x08\xfe\xbc\xcb\xc5\x22\x43\xcb" + "\xfc\x8d\x45\x21\x2e\x04\x7b\x00\xc8\x7c\x6b\x6b\xf1\x75\xf8\xbb\x67" + "\x8e\xc5\x5c\x10\x91\x31\x5c\xbe\xcb\x8b\x85\x70\x0f\x4a\x46\x53\x62" + "\x3f\xb7\x8e\x63\xcf\xff\x7d\x62\x35\xe4\x8e\x98\x32\xc9\xf0\x71\x6d" + "\x10\x99\x2f\xc5\xb0\xad\x4e\x69\x72\xbb\xee\xb1\xad\x67\x0c\xd7\xec" + "\x8f\xac\x82\xe0\x7e\xa5\xa6\x4f\x97\x61\xa3\x97\x14\xaa\xa7\x3a\xff" + "\xd2\xcb\x19\x0a\x7a\xc2\xdf\x5e\x5d\xce\xa6\x81\x2a\xe2\xc8\x72\xc7" + "\xac\x70\x45\x3c\x5e\x7e\xc4\xd0\xb5\xb1\x8c\x6f\xf3\xbf\xb9\xae\x15" + "\xfe\xa4\x4c\xf3\x92\x61\x5b\x80\x03\x4e\xda\xe5\x96\xb8\x82\x1f\x97" + "\xfc\xa5\x8d\x16\x7f\xb4\x4a\x09\x3b\x0c\x00\x9a\x0b\xd5\x63\x13\x55" + "\xb0\xcb\x25\xd9\x3b\xa9\xb7\x9b\x00\x63\x01\xd9\x9d\xb6\x57\xe8\x01" + "\x93\x3f\xc2\x76\x4a\x0c\xe6\x50\xea\xf5\xa1\x29\x9e\xfe\x60\xcb\x53" + "\xb6\x34", + "", "\x09\x89\x12\xa3\x02\x77\x33\x77\xb9\xc2\x6a\xc3", + "\xe3\xbe\x94\x71\x53\xa2\x6a\x3a\x54\xe3\x01\x5c", + "\xfd\x04\x2b\xdd\xe2\x2f\x67\xc4\xfd\x29\x8d\x5d\xc0\x86\x76\x06", 1, + 0, 96 }, + { 256, 192, 128, 215, + "\x8f\x9e\xbc\x67\xa9\xa6\x43\x0c\x2b\x0c\xee\xaf\x98\x3e\x13\x56\x96" + "\x4b\xb9\x28\x63\x5b\x9c\xa4", + "\x36\xe4\xb3\x81\x57\x4d\x17\x1c\x77\x69\xa7\x88\xcb\xc1\x47\x22\x4f" + "\xab\xd8\xb7\x73\xf1\x6b\x8a\xe8\x4d\x8f\x26\x03\xaa\xa4\x40", + "", "\xa3\xa9\x6e\xe9\x4f\x94\xca\xa8\x1e\xbc\xd6\x6d", + "\x8c\x2a\x98\x23\xa3\xb3\xd4\x13\xbe\x69\x63\x87", + "\xfa\xaf\x01\xce\xb4\x0a\x7e\x14\x5e\x8f\xe6\x5a\xa9\xaf\x58\xc0", 1, + 0, 96 }, + { 512, 192, 128, 216, + "\xf4\xbb\xdf\xd0\x6f\x7f\xb1\x43\x48\x80\xe4\x16\x6d\x38\xd5\x6e\x02" + "\xa3\xf0\xdf\x0d\x53\x01\xce", + "\x90\x74\x3b\xd5\xd7\x94\xd5\x2a\xc8\x48\xb7\xe2\x38\x45\x45\xa2\x58" + "\x46\xac\xf1\x43\xbe\x84\xc0\xea\xd0\x43\x2f\xcf\x31\x72\x63\x1c\xf5" + "\x8d\x0c\xa7\x85\x71\xc0\x30\x53\xc1\xe1\xb8\x5e\xd7\x9c\xb5\x30\x3d" + "\x0e\x3a\x98\xff\x4f\x56\xc4\xf0\xa5\xeb\x4f\x0e\xac", + "", "\x39\xd2\xab\xe6\x69\x7f\x17\xec\x27\xf2\xa3\x9c", + "\xa6\x60\xea\x5b\xf0\x7a\x78\xfe\xa0\x12\x01\x73", + "\x74\x04\xfc\x7b\x73\x54\x69\x44\x28\x23\x6f\x20\x3c\x13\x02\x44", 1, + 0, 96 }, + { 1024, 192, 128, 217, + "\x17\x61\xc7\x77\x98\xef\x9c\xdf\xa4\x05\x53\xf3\x46\x14\xfe\x74\x02" + "\x21\x20\x87\xf0\x50\x94\x11", + "\xfb\xb3\xea\xb3\x79\xc9\xb8\x68\x9d\xc3\x0b\x07\x13\x69\x0e\x55\xd5" + "\x1c\x95\x6c\xa3\x6f\xbc\xc7\x3e\xee\xee\x16\xa4\x6d\x7c\x41\xa7\xa9" + "\x62\x6e\x68\xe2\x5d\x68\x5c\x00\x8c\x19\xd3\xb2\xb1\x79\x2b\xdc\x99" + "\xc3\x54\x41\xa6\xfc\xac\x35\xe0\xd6\x44\x6d\xd9\x14\xf5\x43\xab\xd9" + "\xec\xd6\xb0\xcb\x52\x01\xc2\x43\x02\x6c\x4f\x13\x64\x1d\x67\xc8\xd8" + "\xcd\x51\x14\xb6\xe1\x1e\xbb\xc6\xb1\xde\xe2\xa1\x8d\xb2\x15\x0a\x5a" + "\x57\x5d\xcd\x21\x64\x8e\x03\x37\xda\xdb\xcc\xd3\xde\xff\xd6\xd9\x79" + "\xe0\x3e\x6b\x9d\xdf\xee\x0a\xbd\xc2", + "", "\x35\xca\x4e\xb4\x63\xa2\x00\x01\x38\x21\x0b\x4d", + "\xf4\x00\x13\x2f\xf3\x8c\x04\xed\x74\x7d\xde\x34", + "\xca\x15\x34\xe7\xdd\x03\x36\xbb\xb3\x2a\x79\x83\x0c\x71\xa4\x47", 1, + 0, 96 }, + { 2056, 192, 128, 218, + "\xf7\x95\xec\xe7\xde\x18\x81\xfb\xc6\x84\x3e\xb7\x40\xf8\x12\xe4\x1e" + "\x3f\xc4\x9f\xf6\xc7\xb9\x40", + "\x35\x69\xfc\xa7\xc9\xd0\x6e\x2a\x03\xfe\xd1\xaa\xc2\x48\x4f\xd4\x41" + "\x6c\xa0\x7d\x55\xec\xbb\x33\x3e\xc6\x74\xf0\xea\x5c\x6e\x75\xa1\x0d" + "\xfb\x9c\x73\x8b\x69\xda\xb2\xed\xa1\x0a\xda\x72\x1a\x61\xc7\xf0\x2b" + "\x7e\x7f\x79\xe8\xa9\xe2\xdc\x36\xb3\xfd\xf6\x09\xe4\x36\x05\x4c\x82" + "\xa7\x74\xec\x61\x7d\xce\xec\x84\xa5\x77\x03\x7f\xf1\xa3\xf1\x20\xd9" + "\x81\x8d\x04\x20\x63\xac\xb3\x6c\x95\x84\xe8\x1e\xc9\x4f\x11\xf1\xee" + "\x24\x0f\x2e\x45\xe9\x44\x69\x4a\x9c\x8e\x53\x5a\xcb\xb0\x1d\x93\x95" + "\x84\x11\xcf\xf6\x8e\x3d\x32\xf8\x93\x17\x46\xa4\xa0\xce\xce\x65\xe9" + "\x3c\x51\xc7\x0b\x31\x11\x03\x4b\x68\x67\xb4\x07\xe0\x14\x7f\x97\xc5" + "\x76\xd3\xed\x8c\xec\x7e\x8e\xc2\x6e\x95\x64\x3e\x46\xe9\x7e\xa3\x59" + "\x5c\x9c\x31\x72\xb4\x85\x6f\x2d\x2b\x6d\xc8\x56\x46\x66\xdd\xac\x92" + "\xc7\x94\xff\xb2\xd4\xdc\x7f\x46\x17\x61\xf0\xe3\x26\x65\x0f\x48\xd3" + "\x27\x60\x4e\x09\x5b\xd8\x75\x40\x72\x11\x6c\x96\x36\x0d\x09\xf0\x10" + "\xac\x2f\x39\xeb\x96\xb2\x27\xf3\xd7\x38\xde\xb7\x56\xc8\x69\x94\x60" + "\xd8\x8c\xf7\x16\x17\x0a\xe1\x52\x67\xb1\x4f\x4a\x89\x16\x47\x20\xf1" + "\xc6\x02", + "", "\x22\xdb\xd8\x03\x7a\xa0\x5b\x14\xcf\x81\xdd\x23", + "\x13\xa9\x5a\x06\xc1\xbe\xd4\x84\x5a\xf9\xc7\x01", + "\x03\x37\x98\x36\xb0\xc8\x2f\x64\xa1\xbc\xcd\xcd\x76\x3a\xcb\xbc", 1, + 0, 96 }, + { 256, 256, 128, 219, + "\xee\x41\x71\x91\x7d\x23\x37\x49\x68\x12\xa2\x78\x4d\x6a\x71\x30\x0e" + "\x6b\x8c\x1a\xc3\xb1\xef\x58\xce\xe7\x7c\x22\x9a\xea\xf2\xc5", + "\xe8\x26\xa7\x93\x61\xf9\xd5\x82\xb6\x44\x50\xe3\xed\xc8\x25\x89\x48" + "\x78\x53\xd5\xb2\x2f\xea\xa0\xc8\x89\x87\x5b\xd0\xd8\x7c\xd4", + "", "\x94\xd2\xf8\x69\x7f\xac\xaa\xa1\x91\xba\x61\x7a", + "\xa2\x95\xc2\xcb\x27\xce\x23\xd2\x68\x74\xad\xe1", + "\x04\x65\x0a\x78\xbb\xb6\x1d\xb3\x37\xc9\xc3\x2a\xa3\xe7\xb6\xfa", 1, + 0, 96 }, + { 512, 256, 128, 220, + "\x13\x2c\x59\xb4\xbc\xb8\xaf\xb3\x16\x37\x73\x4a\x81\x10\x5b\xb2\xc9" + "\x87\x8f\x32\x0a\xce\x90\x76\xd5\xfd\x7c\x5d\x21\x6c\x8d\x12", + "\xec\x51\xee\x18\xcf\xb4\x68\x97\xd3\x66\x6c\x7d\xf3\x5c\x29\xca\x5d" + "\x89\x82\x41\xc4\xa3\x4f\x89\x3e\xb1\xdb\x5d\x5c\x6b\x76\xe2\x46\x17" + "\x45\x9d\x11\x53\x86\x81\x54\x43\x7a\x0e\x95\xaa\x3c\x26\xe9\x56\xb4" + "\x94\xa5\x2d\xd5\xac\x3b\x93\x31\x11\x6c\x7c\x77\x5f", + "", "\x12\xc7\xbe\x00\xfa\xcd\xa4\x95\x96\xe1\x91\x34", + "\x9c\xdc\xfc\x3a\xaa\x8d\x46\x6f\x25\x58\x8e\x4b", + "\x7e\x80\xf5\x1e\x71\x80\xf1\xcd\x3b\xa8\x43\x49\x88\x8f\xcd\x5c", 1, + 0, 96 }, + { 1024, 256, 128, 221, + "\x7b\x0b\x12\x49\x19\x01\xd6\x2d\x09\x7f\xa2\x6d\xc7\x1e\x15\xcf\xac" + "\xaf\xa3\x22\x67\x19\xe4\x71\x26\xd9\x9c\x79\xd9\x8e\xc2\x22", + "\x7d\x08\xb2\x26\xb4\xa5\xd0\x3f\x6f\x8c\xb3\xa3\xcb\x8d\x1c\xe3\x1b" + "\x05\x9d\xc5\x11\x23\x85\x27\x5e\x38\xa1\x5c\x97\xe0\xf2\x40\x22\xb2" + "\x49\xa5\xf7\x01\x9e\xa5\x77\x19\x8c\xb2\x6a\xc6\x4e\x82\xb2\xb0\x46" + "\x81\x53\x7c\x41\x98\x77\x5a\x52\x3b\x0e\x64\x94\xb8\x4f\xeb\xae\xf3" + "\x39\x9b\x35\xc2\x7b\x09\x69\xfa\x43\x57\x2b\xf5\x82\x7a\x76\x3a\xac" + "\x1a\xf6\x95\x26\xf3\x7e\x38\xac\xb5\xd3\x54\xf2\xb6\x84\x87\xf2\x75" + "\xf4\x36\x1e\xd3\x90\x73\xf7\xdd\x66\x53\xac\x17\xc0\x79\x41\x18\xa0" + "\xcf\x14\x32\x93\xac\x0b\xe6\x62\x29", + "", "\xc8\x03\x12\x59\x07\x00\xc3\xbb\xfa\xcd\x1a\x40", + "\x3f\x3c\x15\x1e\x98\x4d\x05\x94\x62\xf9\xe5\xa0", + "\xe5\x59\xf5\xf7\x55\xaa\x29\x21\x71\xcc\x35\xfb\xf9\x11\xa6\x4f", 1, + 0, 96 }, + { 2056, 256, 128, 222, + "\x3b\xc3\xbf\x39\xd0\xd5\xff\xd9\x4c\xca\x2b\x45\xc6\x78\xa2\xd0\x49" + "\x15\x1e\xd2\xba\xbc\x71\x3b\xe5\x3c\xb6\x6f\x54\xa1\x63\x37", + "\x92\xc2\xce\xe7\xe9\x13\x8b\x18\x6d\xa5\x1f\x14\x6f\xb2\x1f\xd5\xb4" + "\x91\xf1\xa1\x9e\xef\x61\xd4\xed\x14\xce\x6b\x21\xb0\x4f\xdb\x6f\xf8" + "\xeb\xb6\x0f\xdd\xc5\x59\x26\xe7\xbd\xa2\xa8\xf3\x5c\x61\x0b\xb7\x95" + "\x23\x24\x12\x73\x9d\x6c\x2d\x74\x45\x8e\xf5\xa1\xa1\xcd\xe9\xbf\x17" + "\xe4\x7e\x3b\x00\xdb\x0b\x05\x04\xd5\x6d\xc8\xb8\xd3\xde\x23\xf7\xc3" + "\xa5\xd5\x2e\x8d\x0a\xab\x1e\x64\x40\x5a\xaa\x85\x2e\xc2\xdd\x66\x7e" + "\xd9\xc1\xfd\x8d\xc1\xfd\xbb\xc8\x71\x2c\x7a\x38\xf3\x0f\xae\xab\x59" + "\x4f\x33\x89\x7b\x41\xb1\x72\x0f\x3c\x2f\x95\x4e\xd9\x1c\xa4\x50\xd8" + "\x2c\x3d\xcd\x35\x85\x8c\x60\x8a\xd4\x2f\x36\x83\x2e\x56\xb0\x48\x21" + "\xa1\x32\xf7\x2e\x0d\xa7\xb6\x2c\xbd\x39\x25\x25\x0f\x64\xfb\xb3\xf5" + "\xc4\x78\x34\x95\x89\x30\x97\xad\xc0\x9a\x32\xd7\x76\xe0\x4b\xf7\x25" + "\x58\xd3\x78\x30\xb3\x72\x34\x1f\x65\x36\xd8\xee\x9d\xf4\xa8\x2e\x40" + "\x74\xe7\x77\x4a\xb6\x91\x7a\x04\xfa\x8c\x49\x9e\xb4\xb4\x6a\x92\xde" + "\xf3\x65\xda\x8b\x5e\xb1\xe0\xb4\x38\x77\x95\x07\xd1\xf5\x27\x2a\x6e" + "\x86\x29\xa3\xf9\xc7\xbd\x48\x62\xc5\x69\x1e\xe8\xb5\x6b\xfe\x29\x2d" + "\xeb\x4e", + "", "\x81\x25\xee\x76\x37\xd7\xd0\xe0\x3b\xba\xcf\x35", + "\x54\x96\xae\x94\xc3\x32\x2e\xbf\x95\x9e\xa9\xa9", + "\x70\x71\x7c\xc0\x0f\xd1\xff\xa5\x9b\xb0\x43\x29\x22\x6a\x0c\x0a", 1, + 0, 96 }, + { 0, 128, 128, 223, + "\x8f\x3f\x52\xe3\xc7\x5c\x58\xf5\xcb\x26\x1f\x51\x8f\x4a\xd3\x0a", + "", "", "", "", + "\xcf\x71\x97\x8f\xfc\xc7\x78\xf3\xc8\x5a\xc9\xc3\x1b\x6f\xe1\x91", 0, + 0, 0 }, + { 0, 128, 128, 224, + "\x2a\x4b\xf9\x0e\x56\xb7\x0f\xdd\x86\x49\xd7\x75\xc0\x89\xde\x3b", + "", "", + "\x32\x4c\xed\x6c\xd1\x5e\xcc\x5b\x37\x41\x54\x1e\x22\xc1\x8a\xd9", + "\x00\xa2\x9f\x0a\x5e\x2e\x74\x90\x27\x9d\x1f\xaf\x8b\x88\x1c\x7b", + "\xa2\xc7\xe8\xd7\xa1\x9b\x88\x4f\x74\x2d\xfe\xc3\xe7\x6c\x75\xee", 0, + 0, 128 }, + { 0, 192, 128, 225, + "\x0b\x18\xd2\x13\x37\x03\x5c\x7b\xaa\x08\x21\x1b\x70\x2f\xa7\x80\xac" + "\x7c\x09\xbe\x8f\x9e\xd1\x1f", + "", "", "", "", + "\xca\x69\xa2\xeb\x3a\x09\x6e\xa3\x6b\x10\x15\xd5\xdf\xff\xf5\x32", 0, + 0, 0 }, + { 0, 192, 128, 226, + "\xba\x76\xd5\x94\xa6\xdf\x91\x5b\xb7\xab\x7e\x6d\x1a\x8d\x02\x4b\x27" + "\x96\x33\x6c\x1b\x83\x28\xa9", + "", "", + "\xd6\x2f\x30\x27\x42\xd6\x1d\x82\x3e\xa9\x91\xb9\x34\x30\xd5\x89", + "\x50\x9b\x06\x58\xd0\x9f\x7a\x5b\xb9\xdb\x43\xb7\x0c\x83\x87\xf7", + "\x2c\x94\x88\xd5\x3a\x0b\x2b\x53\x08\xc2\x75\x7d\xfa\xc7\x21\x9f", 0, + 0, 128 }, + { 0, 256, 128, 227, + "\x3f\x8c\xa4\x7b\x9a\x94\x05\x82\x64\x4e\x8e\xcf\x9c\x2d\x44\xe8\x13" + "\x83\x77\xa8\x37\x9c\x5c\x11\xaa\xfe\x7f\xec\x19\x85\x6c\xf1", + "", "", "", "", + "\x17\x26\xaa\x69\x5f\xba\xa2\x1a\x1d\xb8\x84\x55\xc6\x70\xa4\xb0", 0, + 0, 0 }, + { 0, 256, 128, 228, + "\x76\x60\xd1\x09\x66\xc6\x50\x39\x03\xa5\x52\xdd\xe2\xa8\x09\xed\xe9" + "\xda\x49\x0e\x5e\x5c\xc3\xe3\x49\xda\x99\x96\x71\x80\x98\x83", + "", "", + "\xc3\x14\x23\x53\x41\xde\xbf\xaf\xa1\x52\x6b\xb6\x10\x44\xa7\xf1", + "\x77\x72\xea\x35\x89\x01\xf5\x71\xd3\xd3\x5c\x19\x49\x76\x39\xd9", + "\x8f\xe0\x52\x0a\xd7\x44\xa1\x1f\x0c\xcf\xd2\x28\x45\x43\x63\xfa", 0, + 0, 128 }, + { 8, 128, 128, 229, + "\x59\xa2\x84\xf5\x0a\xed\xd8\xd3\xe2\xa9\x16\x37\xd3\x81\x55\x79", + "\x80", "", "", "", + "\xaf\x49\x8f\x70\x1d\x24\x70\x69\x5f\x6e\x7c\x83\x27\xa2\x39\x8b", 1, + 0, 0 }, + { 8, 128, 128, 230, + "\xfe\xc5\x8a\xa8\xcf\x06\xbf\xe0\x5d\xe8\x29\xf2\x7e\xc7\x76\x93", + "\x9d", "", + "\xf2\xd9\x9a\x9f\x89\x33\x78\xe0\x75\x7d\x27\xc2\xe3\xa3\x10\x1b", + "\x0a\x24\x61\x2a\x9d\x1c\xbe\x96\x7d\xbf\xe8\x04\xbf\x84\x40\xe5", + "\x96\xe6\xfd\x2c\xdc\x70\x7e\x3e\xe0\xa1\xc9\x0d\x34\xc9\xc3\x6c", 1, + 0, 128 }, + { 16, 128, 128, 231, + "\x88\xa9\x72\xcc\xe9\xea\xf5\xa7\x81\x3c\xe8\x14\x9d\x0c\x1d\x0e", + "\x0f\x2f", "", "", "", + "\x4c\xcf\x1e\xfb\x4d\xa0\x5b\x4a\xe4\x45\x2a\xea\x42\xf5\x42\x4b", 1, + 0, 0 }, + { 16, 128, 128, 232, + "\xb4\x39\x67\xee\x93\x3e\x46\x32\xbd\x65\x62\xba\x12\x01\xbf\x83", + "\x87\x60", "", + "\x5a\x6a\xd6\xdb\x70\x59\x1d\x1e\x52\x0b\x01\x22\xf0\x50\x21\xa0", + "\xba\x3e\x7f\x8b\x29\x99\x99\x5c\x7f\xc4\x00\x6c\xa4\xf4\x75\xff", + "\x98\xf4\x7a\x52\x79\xce\xbb\xca\xc2\x14\x51\x57\x10\xf6\xcd\x8a", 1, + 0, 128 }, + { 32, 128, 128, 233, + "\x4e\x9a\x97\xd3\xed\x54\xc7\xb5\x46\x10\x79\x3a\xb0\x50\x52\xe1", + "\xcc\x85\x19\x57", "", "", "", + "\xe5\x74\xb3\x55\xbd\xa2\x98\x0e\x04\x7e\x58\x4f\xeb\x16\x76\xca", 1, + 0, 0 }, + { 32, 128, 128, 234, + "\xd8\x3c\x1d\x7a\x97\xc4\x3f\x18\x24\x09\xa4\xaa\x56\x09\xc1\xb1", + "\x7b\x5f\xae\xb2", "", + "\xc8\xf0\x7b\xa1\xd6\x55\x54\xa9\xbd\x40\x39\x0c\x30\xc5\x52\x9c", + "\x1b\x84\xba\xea\x9d\xf1\xe6\x5b\xee\x7b\x49\xe4\xa8\xcd\xa1\xec", + "\x5c\x0b\xb7\x9d\x82\x40\x04\x1e\xdc\xe0\xf9\x4b\xd4\xbb\x38\x4f", 1, + 0, 128 }, + { 48, 128, 128, 235, + "\xc6\xa7\x05\x67\x7a\xff\xb4\x9e\x27\x6d\x95\x11\xca\xa4\x61\x45", + "\x4a\xd8\x0c\x28\x54\xfb", "", "", "", + "\x1e\x2e\xd7\x2a\xf5\x90\xca\xfb\x86\x47\xd1\x85\x86\x5f\x54\x63", 1, + 0, 0 }, + { 48, 128, 128, 236, + "\xeb\xa7\x69\x9b\x56\xcc\x0a\xa2\xf6\x6a\x2a\x5b\xe9\x94\x44\x13", + "\xd1\xda\xfc\x8d\xe3\xe3", "", + "\xd0\x21\xe5\x3d\x90\x98\xa2\xdf\x3d\x6b\x90\x3c\xda\xd0\xcd\x9c", + "\x18\x29\x1a\xa8\xdc\x7b\x07\x44\x8a\xa8\xf7\x1b\xb8\xe3\x80\xbf", + "\x9c\x0e\x22\xe5\xc4\x1b\x10\x39\xff\x56\x61\xff\xae\xfa\x8e\x0f", 1, + 0, 128 }, + { 8, 192, 128, 237, + "\xc7\x0c\xe3\x8e\x84\xe5\xf5\x3e\xd4\x1c\x3f\x0d\x2c\xa4\x93\x41\x2a" + "\xd3\x2c\xb0\x4c\x6e\x2e\xfa", + "\xcb", "", "", "", + "\x08\xd9\x6e\xdb\x5e\x22\x87\x4c\xd1\x0c\xb2\x25\x6c\xa0\x4b\xc6", 1, + 0, 0 }, + { 8, 192, 128, 238, + "\x74\xc8\x16\xb8\x3d\xfd\x28\x72\x10\xa3\xe2\xc6\xda\x8d\x30\x53\xbb" + "\xfb\xd9\xb1\x56\xd3\xfd\xd8", + "\x0f", "", + "\xf2\xb7\xb2\xc9\xb3\x12\xcf\x2a\xf7\x8f\x00\x3d\xf1\x5c\x8e\x19", + "\x6c\x5e\x79\x6b\xa9\xa3\xdd\xc6\x4f\x40\x1e\x68\xd1\x35\x10\x1d", + "\x96\xa1\x32\xed\x43\x92\x4e\x98\xfe\xb8\x88\xff\x68\x2b\xda\xef", 1, + 0, 128 }, + { 16, 192, 128, 239, + "\xcb\xf4\x5b\xa4\x88\x93\x2a\xea\x1a\x10\xe5\x86\x2f\x92\xe4\xa7\xe2" + "\x77\xbd\xa9\xf3\x4a\xf6\xd0", + "\x75\xe5", "", "", "", + "\x1f\x0d\x23\x07\x0f\xcd\x74\x8e\x25\xbf\x64\x54\xf5\xc9\x13\x6e", 1, + 0, 0 }, + { 16, 192, 128, 240, + "\xe1\xc0\x44\x6f\x11\xae\x6a\xa4\xfa\x25\x4f\x9a\x84\x6f\xc6\xe1\x3e" + "\x45\xe5\x37\xe4\x7f\x20\x42", + "\x89\x89", "", + "\x3a\x2f\x5a\xd0\xeb\x21\x6e\x54\x6e\x0b\xca\xa3\x77\xb6\xcb\xc7", + "\x55\x0b\x48\xa4\x3e\x82\x1f\xd7\x6f\x49\xf0\xf1\xa8\x97\xae\xad", + "\xf6\xe0\xa9\x79\x48\x1f\x99\x57\xdd\xad\x0f\x21\xa7\x77\xa7\x3a", 1, + 0, 128 }, + { 32, 192, 128, 241, + "\x56\x75\x63\xbf\x4c\xf1\x54\x90\x22\x75\xa5\x3b\xc5\x7c\xd6\xdd\x7b" + "\x37\x0d\x27\x01\x1b\xda\xc8", + "\x68\xd7\xfc\x38", "", "", "", + "\x14\x75\x56\x3e\x32\x12\xf3\xb5\xe4\x00\x62\x56\x9a\xfd\x71\xe3", 1, + 0, 0 }, + { 32, 192, 128, 242, + "\x83\x4d\x0b\xb6\x01\x17\x08\x65\xa7\x81\x39\x42\x8a\x15\x03\x69\x5a" + "\x6a\x29\x1e\xbd\x74\x7c\xd1", + "\xbb\x9d\x2a\xa3", "", + "\x6f\x79\xe1\x8b\x4a\xcd\x5a\x03\xd3\xa5\xf7\xe1\xa8\xd0\xf1\x83", + "\x30\x91\x33\xe7\x61\x59\xfe\x8a\x41\xb2\x08\x43\x48\x65\x11\xab", + "\x03\xab\x26\x99\x3b\x70\x19\x10\xa2\xe8\xec\xcc\xd2\xba\x9e\x52", 1, + 0, 128 }, + { 48, 192, 128, 243, + "\x99\xfb\x18\xf5\xba\x43\x0b\xb9\xea\x94\x29\x68\xec\xb7\x99\xb4\x34" + "\x06\xe1\xaf\x4b\x64\x25\xa1", + "\xa9\x84\xbd\xcd\xca\xe2", "", "", "", + "\xd7\xb9\xa6\xb5\x8a\x97\x98\x29\x16\xe8\x32\x19\xfb\xf7\x1b\x1e", 1, + 0, 0 }, + { 48, 192, 128, 244, + "\xb7\x7b\x24\x2a\xa0\xd5\x1c\x92\xfd\xa0\x13\xe0\xcb\x0e\xf2\x43\x73" + "\x99\xac\xe5\xd3\xf5\x07\xe4", + "\x52\xaa\x01\xe0\xd0\xd6", "", + "\x4b\xa5\x41\xa9\x91\x47\x29\x21\x61\x53\x80\x13\x40\xab\x17\x79", + "\xe0\x82\x61\xe4\x6e\xaf\x90\xd9\x78\xea\x8f\x78\x89\xbc\xcd\x4f", + "\xc0\x52\xa5\x5d\xf3\x92\x6a\x50\x99\x0a\x53\x2e\xfe\x3d\x80\xec", 1, + 0, 128 }, + { 64, 192, 128, 245, + "\xd7\x45\x99\xb3\xd2\xdb\x81\x65\x3d\xe4\x3b\x52\xfc\x99\x4c\x50\xd0" + "\xbe\x75\x9f\xab\x87\xc3\x3a", + "\xd1\xc6\x1c\xf8\x53\x25\x31\xb5", "", "", "", + "\xf9\x4f\x20\x49\xa6\x56\x0c\x47\x0b\x3a\x7c\xa7\xbb\xc3\x1a\x3d", 1, + 0, 0 }, + { 64, 192, 128, 246, + "\x0b\x17\x71\x98\xc8\xb4\x19\xbf\x74\xac\xc3\xbc\x65\xb5\xfb\x3d\x09" + "\xa9\x15\xff\x71\xad\xd7\x54", + "\x8f\x07\x5c\xbc\xda\x98\x31\xc3", "", + "\xc4\xb1\xe0\x5c\xa3\xd5\x91\xf9\x54\x3e\x64\xde\x3f\xc6\x82\xac", + "\x3c\x6e\xc0\xab\x1b\x82\x7b\xf2\x38\xa5\x38\x4f\xb7\xe2\x12\xce", + "\x7d\xb7\x40\x22\x24\xfd\x58\x3e\x31\x2b\xc0\xe6\x1c\xf1\x13\x66", 1, + 0, 128 }, + { 8, 256, 128, 247, + "\x8f\x9a\x38\xc1\x01\x49\x66\xe4\xd9\xae\x73\x61\x39\xc5\xe7\x9b\x99" + "\x34\x58\x74\xf4\x2d\x4c\x7d\x2c\x81\xaa\x67\x97\xc4\x17\xc0", + "\xa9", "", "", "", + "\x2a\x26\x8b\xf3\xa7\x5f\xd7\xb0\x0b\xa2\x30\xb9\x04\xbb\xb0\x14", 1, + 0, 0 }, + { 8, 256, 128, 248, + "\x14\x4c\xd8\x27\x92\x29\xe8\xbb\x2d\xe9\x9d\x24\xe6\x15\x30\x66\x63" + "\x91\x3f\xe9\x17\x7f\xcd\x27\x0f\xaf\xec\x49\x3d\x43\xbc\xa1", + "\xb3", "", + "\x97\x62\x29\xf5\x53\x8f\x96\x36\x47\x6d\x69\xf0\xc3\x28\xe2\x9d", + "\x7b\xea\x30\xec\xc2\xf7\x3f\x8e\x12\x12\x63\xb3\x79\x66\x95\x4c", + "\x8b\xba\xd4\xad\xc5\x4b\x37\xa2\xb2\xf0\xf6\xe8\x61\x75\x48\xc9", 1, + 0, 128 }, + { 16, 256, 128, 249, + "\x7d\x31\x86\x1f\x9d\x35\x36\xe1\x40\x16\xa3\x21\x6b\x10\x42\xe0\xd2" + "\xf7\xd4\x61\x43\x14\x26\x8b\x6f\x83\x4e\xc7\xf3\x8b\xbb\x65", + "\xc3\x32", "", "", "", + "\x1d\x97\x8a\x69\x31\x20\xc1\x1f\x6d\x51\xa3\xed\x88\xcd\x4a\xce", 1, + 0, 0 }, + { 16, 256, 128, 250, + "\x22\xb3\x5f\xe9\x62\x3e\xe1\x1f\x8b\x60\xb6\xd2\x2d\xb3\x76\x5b\x66" + "\x6e\xd9\x72\xfa\x7c\xcd\x92\xb4\x5f\x22\xde\xee\x02\xca\xb1", + "\xda\x6c", "", + "\x53\x41\xc7\x8e\x4c\xe5\xbf\x8f\xbc\x3e\x07\x7d\x19\x90\xdd\x5d", + "\x9c\x39\xf5\xb1\x10\x36\x1e\x9a\x77\x0c\xc5\xe8\xb0\xf4\x44\xbb", + "\xb6\x3f\xf4\x3c\x12\x07\x3e\xc5\x57\x2b\x1b\xe7\x0f\x17\xe2\x31", 1, + 0, 128 }, + { 32, 256, 128, 251, + "\xc2\x24\xe0\xbb\xa3\xd7\xa9\x91\x65\xf7\x99\x6b\x67\xa0\xfc\xe3\xe1" + "\x2f\x2c\x01\x17\x9b\x19\x7b\x69\xb7\xe6\x28\xbc\xa9\x20\x96", + "\x6b\x30\x14\x5e", "", "", "", + "\xae\x6f\x7c\x9a\x29\xf0\xd8\x20\x4c\xa5\x0b\x14\xa1\xe0\xdc\xf2", 1, + 0, 0 }, + { 32, 256, 128, 252, + "\x09\x3e\xb1\x23\x43\x53\x7e\xe8\xe9\x1c\x1f\x71\x5b\x86\x26\x03\xf8" + "\xda\xf9\xd4\xe1\xd7\xd6\x72\x12\xa9\xd6\x8e\x5a\xac\x93\x58", + "\x51\x10\x60\x4c", "", + "\x33\xef\xb5\x8c\x91\xe8\xc7\x02\x71\x87\x0e\xc0\x0f\xe2\xe2\x02", + "\xf7\x3f\x72\xf9\x76\xa2\x96\xba\x3c\xa9\x4b\xc6\xeb\x08\xcd\x46", + "\xb8\x24\xc3\x3c\x13\xf2\x89\x42\x96\x59\xaa\x01\x7c\x63\x2f\x71", 1, + 0, 128 }, + { 48, 256, 128, 253, + "\x98\xe6\xf8\xab\x67\x3e\x80\x4e\x86\x5e\x32\x40\x3a\x65\x51\xbf\x80" + "\x7a\x95\x93\x43\xc6\x0d\x34\x55\x93\x60\xbc\x29\x5e\xcb\x5b", + "\xd4\xd8\x57\x51\x08\x88", "", "", "", + "\x3d\xb1\x67\x25\xfa\xfc\x82\x8d\x41\x4a\xb6\x1c\x16\xa6\xc3\x8f", 1, + 0, 0 }, + { 48, 256, 128, 254, + "\x0b\xd0\xe8\xe7\x78\x11\x66\xe1\xd8\x76\xde\xc8\xfa\xd3\x4b\xa9\x5b" + "\x03\x2a\x27\xca\xc0\x55\x15\x95\x11\x60\x91\x00\x59\x47\xb7", + "\x1b\xdc\xd4\x4b\x66\x3e", "", + "\x91\x22\x22\x63\xb1\x2c\xf5\x61\x6a\x04\x9c\xbe\x29\xab\x9b\x5b", + "\xed\x46\x3f\x4f\x43\x33\x6a\xf3\xf4\xd7\xe0\x87\x70\x20\x11\x45", + "\xc8\xfc\x39\x90\x6a\xca\x0c\x64\xe1\x4a\x43\xff\x75\x0a\xbd\x8a", 1, + 0, 128 }, + { 64, 256, 128, 255, + "\x61\xba\x69\x48\x97\x92\x5d\x1b\x41\x74\xd4\x04\x01\x46\x9c\x3e\xf2" + "\x67\xcd\xb9\xf8\x29\xed\xb1\xa1\x06\x18\xc1\x6d\x66\x60\x59", + "\x0d\x10\xc5\xc8\x4b\x88\xd6\x88", "", "", "", + "\x13\x11\xf9\xf8\x30\xd7\x29\xc1\x89\xb7\x4e\xc4\xf9\x08\x0f\xa1", 1, + 0, 0 }, + { 64, 256, 128, 256, + "\x11\x58\x84\xf6\x93\xb1\x55\x56\x3e\x9b\xfb\x3b\x07\xca\xcb\x2f\x7f" + "\x7c\xaa\x9b\xfe\x51\xf8\x9e\x23\xfe\xb5\xa9\x46\x8b\xfd\xd0", + "\x04\x10\x21\x99\xef\x21\xe1\xdf", "", + "\x82\xe3\xe6\x04\xd2\xbe\x8f\xca\xb7\x4f\x63\x8d\x1e\x70\xf2\x4c", + "\x7e\x0d\xd6\xc7\x2a\xec\x49\xf8\x9c\xc6\xa8\x00\x60\xc0\xb1\x70", + "\xaf\x68\xa3\x7c\xfe\xfe\xcc\x4a\xb9\x9b\xa5\x0a\x53\x53\xed\xca", 1, + 0, 128 }, + { 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0 } +}; diff --git a/test/wycheproof/chacha20_poly1305_test.json.c b/test/wycheproof/chacha20_poly1305_test.json.c new file mode 100644 index 00000000..9a30de04 --- /dev/null +++ b/test/wycheproof/chacha20_poly1305_test.json.c @@ -0,0 +1,4058 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* CHACHA20-POLY1305, 0.8r12 */ +#include "aead_test.h" +const struct aead_test chacha20_poly1305_test_json[] = { + { 96, 256, 128, 1, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x07\x00\x00\x00\x40\x41\x42\x43\x44\x45\x46\x47", + "\x50\x51\x52\x53\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7", + "\x4c\x61\x64\x69\x65\x73\x20\x61\x6e\x64\x20\x47\x65\x6e\x74\x6c\x65" + "\x6d\x65\x6e\x20\x6f\x66\x20\x74\x68\x65\x20\x63\x6c\x61\x73\x73\x20" + "\x6f\x66\x20\x27\x39\x39\x3a\x20\x49\x66\x20\x49\x20\x63\x6f\x75\x6c" + "\x64\x20\x6f\x66\x66\x65\x72\x20\x79\x6f\x75\x20\x6f\x6e\x6c\x79\x20" + "\x6f\x6e\x65\x20\x74\x69\x70\x20\x66\x6f\x72\x20\x74\x68\x65\x20\x66" + "\x75\x74\x75\x72\x65\x2c\x20\x73\x75\x6e\x73\x63\x72\x65\x65\x6e\x20" + "\x77\x6f\x75\x6c\x64\x20\x62\x65\x20\x69\x74\x2e", + "\xd3\x1a\x8d\x34\x64\x8e\x60\xdb\x7b\x86\xaf\xbc\x53\xef\x7e\xc2\xa4" + "\xad\xed\x51\x29\x6e\x08\xfe\xa9\xe2\xb5\xa7\x36\xee\x62\xd6\x3d\xbe" + "\xa4\x5e\x8c\xa9\x67\x12\x82\xfa\xfb\x69\xda\x92\x72\x8b\x1a\x71\xde" + "\x0a\x9e\x06\x0b\x29\x05\xd6\xa5\xb6\x7e\xcd\x3b\x36\x92\xdd\xbd\x7f" + "\x2d\x77\x8b\x8c\x98\x03\xae\xe3\x28\x09\x1b\x58\xfa\xb3\x24\xe4\xfa" + "\xd6\x75\x94\x55\x85\x80\x8b\x48\x31\xd7\xbc\x3f\xf4\xde\xf0\x8e\x4b" + "\x7a\x9d\xe5\x76\xd2\x65\x86\xce\xc6\x4b\x61\x16", + "\x1a\xe1\x0b\x59\x4f\x09\xe2\x6a\x7e\x90\x2e\xcb\xd0\x60\x06\x91", 1, + 96, 912 }, + { 96, 256, 128, 2, + "\x80\xba\x31\x92\xc8\x03\xce\x96\x5e\xa3\x71\xd5\xff\x07\x3c\xf0\xf4" + "\x3b\x6a\x2a\xb5\x76\xb2\x08\x42\x6e\x11\x40\x9c\x09\xb9\xb0", + "\x4d\xa5\xbf\x8d\xfd\x58\x52\xc1\xea\x12\x37\x9d", "", "", "", + "\x76\xac\xb3\x42\xcf\x31\x66\xa5\xb6\x3c\x0c\x0e\xa1\x38\x3c\x8d", 1, + 0, 0 }, + { 96, 256, 128, 3, + "\x7a\x4c\xd7\x59\x17\x2e\x02\xeb\x20\x4d\xb2\xc3\xf5\xc7\x46\x22\x7d" + "\xf5\x84\xfc\x13\x45\x19\x63\x91\xdb\xb9\x57\x7a\x25\x07\x42", + "\xa9\x2e\xf0\xac\x99\x1d\xd5\x16\xa3\xc6\xf6\x89", + "\xbd\x50\x67\x64\xf2\xd2\xc4\x10", "", "", + "\x90\x6f\xa6\x28\x4b\x52\xf8\x7b\x73\x59\xcb\xaa\x75\x63\xc7\x09", 1, + 64, 0 }, + { 96, 256, 128, 4, + "\xcc\x56\xb6\x80\x55\x2e\xb7\x50\x08\xf5\x48\x4b\x4c\xb8\x03\xfa\x50" + "\x63\xeb\xd6\xea\xb9\x1f\x6a\xb6\xae\xf4\x91\x6a\x76\x62\x73", + "\x99\xe2\x3e\xc4\x89\x85\xbc\xcd\xee\xab\x60\xf1", "", "\x2a", + "\x3a", + "\xca\xc2\x7d\xec\x09\x68\x80\x1e\x9f\x6e\xde\xd6\x9d\x80\x75\x22", 1, + 0, 8 }, + { 96, 256, 128, 5, + "\x46\xf0\x25\x49\x65\xf7\x69\xd5\x2b\xdb\x4a\x70\xb4\x43\x19\x9f\x8e" + "\xf2\x07\x52\x0d\x12\x20\xc5\x5e\x4b\x70\xf0\xfd\xa6\x20\xee", + "\xab\x0d\xca\x71\x6e\xe0\x51\xd2\x78\x2f\x44\x03", + "\x91\xca\x6c\x59\x2c\xbc\xca\x53", "\x51", "\xc4", + "\x16\x83\x10\xca\x45\xb1\xf7\xc6\x6c\xad\x4e\x99\xe4\x3f\x72\xb9", 1, + 64, 8 }, + { 96, 256, 128, 6, + "\x2f\x7f\x7e\x4f\x59\x2b\xb3\x89\x19\x49\x89\x74\x35\x07\xbf\x3e\xe9" + "\xcb\xde\x17\x86\xb6\x69\x5f\xe6\xc0\x25\xfd\x9b\xa4\xc1\x00", + "\x46\x1a\xf1\x22\xe9\xf2\xe0\x34\x7e\x03\xf2\xdb", "", "\x5c\x60", + "\x4d\x13", + "\x91\xe8\xb6\x1e\xfb\x39\xc1\x22\x19\x54\x53\x07\x7b\x22\xe5\xe2", 1, + 0, 16 }, + { 96, 256, 128, 7, + "\xc8\x83\x3d\xce\x5e\xa9\xf2\x48\xaa\x20\x30\xea\xcf\xe7\x2b\xff\xe6" + "\x9a\x62\x0c\xaf\x79\x33\x44\xe5\x71\x8f\xe0\xd7\xab\x1a\x58", + "\x61\x54\x6b\xa5\xf1\x72\x05\x90\xb6\x04\x0a\xc6", + "\x88\x36\x4f\xc8\x06\x05\x18\xbf", "\xdd\xf2", "\xb6\x0d", + "\xea\xd0\xfd\x46\x97\xec\x2e\x55\x58\x23\x77\x19\xd0\x24\x37\xa2", 1, + 64, 16 }, + { 96, 256, 128, 8, + "\xbd\x8e\xd7\xfb\x0d\x60\x75\x22\xf0\x4d\x0b\x12\xd4\x2c\x92\x57\x0b" + "\xcc\xc5\xba\x24\x86\x95\x3d\x70\xba\x2e\x81\x93\xf6\x22\x5a", + "\xd2\xab\x0a\xbb\x50\xa8\xe9\xfb\xa2\x54\x29\xe1", "", + "\x20\x12\x21", "\x3c\xf4\x70", + "\xa2\x7a\x69\xc9\xd7\xee\x84\x58\x6f\x11\x38\x8c\x68\x84\xe6\x3a", 1, + 0, 24 }, + { 96, 256, 128, 9, + "\x1c\x8b\x59\xb1\x7a\x5c\xec\xed\x31\xbd\xe9\x7d\x4c\xef\xd9\xaa\xaa" + "\x63\x36\x2e\x09\x6e\x86\x3e\xc1\xc8\x95\x80\xbc\xa7\x9b\x7a", + "\x94\xf3\x2a\x6d\xff\x58\x8f\x2b\x5a\x2e\xad\x45", + "\x6c\x8c\xf2\xab\x38\x20\xb6\x95", "\x45\x3f\x95", "\x61\x09\x25", + "\xa8\xa7\x88\x3e\xb7\xe4\x0b\xc4\x0e\x2e\x59\x22\xae\x95\xdd\xc3", 1, + 64, 24 }, + { 96, 256, 128, 10, + "\xe4\x91\x2c\xb7\x5a\x11\x74\x34\x5f\x1a\x45\x73\x66\xf1\x88\x85\xfe" + "\x84\x60\xb0\x64\x78\xe0\x4b\xe2\xf7\xfb\x4e\xc9\xc1\x13\xe5", + "\x7a\xa5\xad\x8b\xf5\x25\x47\x62\x17\x1e\xc8\x69", "", + "\x9e\x4c\x1d\x03", "\xfe\x68\x49\xaa", + "\x99\xad\x07\x87\x1b\x25\xc2\x7d\xef\xc3\x1a\x54\x1b\xd5\xc4\x18", 1, + 0, 32 }, + { 96, 256, 128, 11, + "\xe0\x57\x77\xef\x3d\x98\x9a\xce\x7d\x2a\xbf\xba\x45\x2b\xfd\xed\x54" + "\x80\x1d\xbd\x5c\x66\xe9\x1c\x0c\x2e\xf0\x04\x79\xd8\x55\x72", + "\xb7\xf5\x26\xe3\xfd\x71\xcf\x57\x20\x96\x1a\xec", + "\x15\xd9\x3a\x96\xd0\xe6\xc5\xa9", "\x17\xbf\xda\x03", + "\xf4\x71\x0e\x51", + "\xb9\x57\xc6\xa3\x7b\x6a\x4c\x94\x99\x6c\x00\x21\x86\xd6\x3b\x2b", 1, + 64, 32 }, + { 96, 256, 128, 12, + "\x1a\x4c\x4f\x39\xab\xe8\x90\xe6\x23\x45\xc9\x47\xbc\xf7\xde\x7c\x2e" + "\x33\xbd\x5c\xee\xda\x0a\x0a\xbf\x0e\x7e\xf9\x35\xdd\xf3\xee", + "\x94\x47\xbf\x85\xd5\xb9\x7d\x8a\xee\x0f\x8e\x51", "", + "\xc1\x5a\x59\x3b\xd0", "\xf7\x11\x64\x7f\xf1", + "\x22\xb1\x2d\xc3\x8c\xb7\x96\x29\xf8\x4c\xdb\xdc\x24\x25\xc0\x9d", 1, + 0, 40 }, + { 96, 256, 128, 13, + "\x80\x0e\x9a\x24\x79\x17\x00\xc9\x60\x97\x36\x69\x5b\xa2\xa8\xb9\x9b" + "\x2d\x57\xf1\xc3\xbf\xb6\x1e\xd4\x9d\xb1\xc6\xc5\x21\x95\x83", + "\x3d\xbe\x87\x6b\xd8\x80\xec\x8e\xa2\x01\x70\x43", + "\x96\x22\x48\x35\x61\x0b\x78\x2b", "\xa7\xbf\xd0\x41\xe3", + "\xd1\x71\xf0\x46\xea", + "\xd1\x79\xb1\xb9\xc4\x18\x43\x78\xdf\x00\x90\x19\xdb\xb8\xc2\x49", 1, + 64, 40 }, + { 96, 256, 128, 14, + "\x20\x8c\x2c\x37\x6c\x94\x30\x43\x3d\xb2\x0e\x1a\x6b\x7b\xa8\x17\xf8" + "\xff\xbf\xa6\x82\x7f\x26\x75\x9c\xce\xde\x42\xe5\x91\xd3\xec", + "\x27\xfb\x58\xec\x6a\x21\xe8\x46\x96\xcb\x88\x30", "", + "\xaf\x10\x4b\x5c\xcd\x0e", "\x93\x51\xb1\xb1\xb0\x82", + "\x56\x07\x85\x50\x9f\x60\xf2\x6b\x68\x19\x33\xd9\xcd\xbf\xd2\x9f", 1, + 0, 48 }, + { 96, 256, 128, 15, + "\x2e\xb1\x68\xe5\x3b\x07\xab\x04\x35\x5e\xa7\x92\xfe\x11\xa6\xbe\x2c" + "\xe9\xc3\x9c\xfe\x15\xa9\x97\x07\x6b\x1e\x38\xc1\x7a\xd6\x20", + "\xb5\x96\x54\x70\xc3\x83\xfd\x29\xfe\x7e\xae\xe7", + "\x6d\x52\xfe\xb2\x50\x9f\x7f\xbf", "\x6f\xdf\x29\x27\xe1\x69", + "\x41\xab\xff\x7b\x71\xcc", + "\x9b\x51\x74\x29\x7c\x03\xcf\x89\x02\xd1\xf7\x06\xfd\x00\x89\x02", 1, + 64, 48 }, + { 96, 256, 128, 16, + "\x55\x56\x81\x58\xd3\xa6\x48\x3f\x1f\x70\x21\xea\xb6\x9b\x70\x3f\x61" + "\x42\x51\xca\xdc\x1a\xf5\xd3\x4a\x37\x4f\xdb\xfc\x5a\xda\xc7", + "\x3c\x4e\x65\x4d\x66\x3f\xa4\x59\x6d\xc5\x5b\xb7", "", + "\xab\x85\xe9\xc1\x57\x17\x31", "\x5d\xfe\x34\x40\xdb\xb3\xc3", + "\xed\x7a\x43\x4e\x26\x02\xd3\x94\x28\x1e\x0a\xfa\x9f\xb7\xaa\x42", 1, + 0, 56 }, + { 96, 256, 128, 17, + "\xe3\xc0\x9e\x7f\xab\x1a\xef\xb5\x16\xda\x6a\x33\x02\x2a\x1d\xd4\xeb" + "\x27\x2c\x80\xd5\x40\xc5\xda\x52\xa7\x30\xf3\x4d\x84\x0d\x7f", + "\x58\x38\x93\x75\xc6\x9e\xe3\x98\xde\x94\x83\x96", + "\x84\xe4\x6b\xe8\xc0\x91\x90\x53", "\x4e\xe5\xcd\xa2\x0d\x42\x90", + "\x4b\xd4\x72\x12\x94\x1c\xe3", + "\x18\x5f\x14\x08\xee\x7f\xbf\x18\xf5\xab\xad\x6e\x22\x53\xa1\xba", 1, + 64, 56 }, + { 96, 256, 128, 18, + "\x51\xe4\xbf\x2b\xad\x92\xb7\xaf\xf1\xa4\xbc\x05\x55\x0b\xa8\x1d\xf4" + "\xb9\x6f\xab\xf4\x1c\x12\xc7\xb0\x0e\x60\xe4\x8d\xb7\xe1\x52", + "\x4f\x07\xaf\xed\xfd\xc3\xb6\xc2\x36\x18\x23\xd3", "", + "\xbe\x33\x08\xf7\x2a\x2c\x6a\xed", + "\x8e\x94\x39\xa5\x6e\xee\xc8\x17", + "\xfb\xe8\xa6\xed\x8f\xab\xb1\x93\x75\x39\xdd\x6c\x00\xe9\x00\x21", 1, + 0, 64 }, + { 96, 256, 128, 19, + "\x11\x31\xc1\x41\x85\x77\xa0\x54\xde\x7a\x4a\xc5\x51\x95\x0f\x1a\x05" + "\x3f\x9a\xe4\x6e\x5b\x75\xfe\x4a\xbd\x56\x08\xd7\xcd\xda\xdd", + "\xb4\xea\x66\x6e\xe1\x19\x56\x33\x66\x48\x4a\x78", + "\x66\xc0\xae\x70\x07\x6c\xb1\x4d", + "\xa4\xc9\xc2\x80\x1b\x71\xf7\xdf", + "\xb9\xb9\x10\x43\x3a\xf0\x52\xb0", + "\x45\x30\xf5\x1a\xee\xe0\x24\xe0\xa4\x45\xa6\x32\x8f\xa6\x7a\x18", 1, + 64, 64 }, + { 96, 256, 128, 20, + "\xe1\x09\x49\x67\xf8\x6d\x89\x3c\xdf\xe2\xe2\xe6\xd5\xc7\xee\x4d\xfe" + "\xf6\x7d\xa3\xc9\xc5\xd6\x4e\x6a\xd7\xc1\x57\x7d\xcb\x38\xc5", + "\x80\x92\xfc\x24\x5b\x33\x26\xcd\xdb\xd1\x42\x4c", "", + "\xc3\x7a\xa7\x91\xdd\xd6\xac\xcf\x91", + "\xd9\xd8\x97\xa9\xc1\xc5\xbb\x9f\x01", + "\x08\x5a\x43\x03\x73\x05\x8f\x1a\x12\xa0\xd5\x89\xfd\x5b\xe6\x8b", 1, + 0, 72 }, + { 96, 256, 128, 21, + "\x23\x6f\x9b\xae\xe4\xf9\xda\x15\xbe\xec\xa4\x0f\xf4\xaf\x7c\x76\x0f" + "\x25\x4a\x64\xbc\x3a\x3d\x7f\x4f\xad\x55\x7e\x61\xb6\x85\x86", + "\xf1\xca\x81\x33\x86\x29\x58\x7a\xcf\x93\x72\xbf", + "\x8c\x32\xf4\x7a\x38\x61\x52\xec", + "\xd7\xf2\x6d\x52\x52\xe1\x76\x5f\x5b", + "\x8f\xdb\x42\x9d\x47\x76\x1c\xbf\x8e", + "\x8e\xf6\x47\xed\x33\x4f\xde\xbb\xc2\xbe\xf8\x0b\xe0\x28\x84\xe0", 1, + 64, 72 }, + { 96, 256, 128, 22, + "\x4d\xe2\x07\xa3\xb7\x0c\x51\xe5\xf2\x30\x48\xee\xd5\xa5\xda\x9b\xb6" + "\x5e\x91\x7a\x69\xaa\x93\xe7\xc8\xb4\xa8\x15\xcd\x97\x24\xde", + "\x4c\x15\xa7\x1d\xc6\x79\x1a\x8c\x00\x5a\xd5\x02", "", + "\xf2\xc5\x4b\x6b\x5e\x49\x0d\xa1\x86\x59", + "\x70\x0d\x35\xad\xf5\x10\x0a\x22\xa1\xde", + "\x10\x2d\x99\x2f\xfa\xff\x59\x9b\x5b\xdd\xdd\xeb\x2d\xfb\x39\x9b", 1, + 0, 80 }, + { 96, 256, 128, 23, + "\x6d\x66\x7f\xd7\x9e\x5f\xb7\x25\xf5\x03\x43\xdc\xcc\x48\x63\x22\x7c" + "\x75\xee\x3f\x7a\x57\x84\x76\xe3\xe9\xf3\x25\x98\xd8\x15\x59", + "\x62\x20\x52\x7a\xba\x88\xe2\x7f\x76\x66\x58\xb2", + "\xe1\xe2\x7c\xcd\xdb\x3c\xb4\x07", + "\x0c\x8c\x5a\x25\x26\x81\xf2\xb5\xb4\xc0", + "\x04\xaa\xd6\x6c\x60\xe0\xbf\x8e\xbb\xa9", + "\xc1\x5f\x69\xa4\xd2\xae\xf9\x7d\x77\x48\x75\x6f\xf4\x9d\x89\x4b", 1, + 64, 80 }, + { 96, 256, 128, 24, + "\x8f\x4b\xd9\x4e\xf7\x3e\x75\xd1\xe0\x68\xc3\x0b\x37\xea\xd5\x76\xc5" + "\x34\x4e\x09\x3e\xce\x13\x30\xe9\x10\x1c\x82\xf7\x93\xcf\x05", + "\xec\x1e\x29\x67\xf0\xf6\x97\x9e\x5f\x5b\x07\xfb", "", + "\xb8\x98\x12\xb3\x4d\x9b\xce\xd4\xa0\xba\x07", + "\x1c\x3d\x53\xba\xaa\x36\xea\xa1\xd8\xec\x4d", + "\x4d\x94\xeb\xf9\x60\xf1\x24\x33\xbe\xc4\x3a\xa8\x6d\x7e\x6e\x6d", 1, + 0, 88 }, + { 96, 256, 128, 25, + "\x2a\xa3\xbc\x70\x33\x35\x1c\xac\x51\x36\x4c\xda\xf6\xff\xac\x2c\x20" + "\xf6\x40\x46\xe1\x55\x0a\x7b\x1c\x65\xf4\x18\x00\x59\x90\x19", + "\x28\xcc\xe5\x7a\x5d\xb2\xcd\x20\x63\x21\xe3\x40", + "\xa9\xbc\x35\x0e\xaf\x2e\x6e\x3d", + "\x83\x01\x68\x23\x12\x34\x84\xb5\x60\x95\xb0", + "\x1c\x85\x78\xf8\xe7\x52\x03\xd0\x33\x6a\x52", + "\x59\x10\xf7\xa9\xd5\xe4\xdf\x05\xd7\x24\x8b\xd7\xa8\xd6\x5e\x63", 1, + 64, 88 }, + { 96, 256, 128, 26, + "\x99\xb6\x2b\xd5\xaf\xbe\x3f\xb0\x15\xbd\xe9\x3f\x0a\xbf\x48\x39\x57" + "\xa1\xc3\xeb\x3c\xa5\x9c\xb5\x0b\x39\xf7\xf8\xa9\xcc\x51\xbe", + "\x9a\x59\xfc\xe2\x6d\xf0\x00\x5e\x07\x53\x86\x56", "", + "\x42\xba\xae\x59\x78\xfe\xaf\x5c\x36\x8d\x14\xe0", + "\xff\x7d\xc2\x03\xb2\x6c\x46\x7a\x6b\x50\xdb\x33", + "\x57\x8c\x0f\x27\x58\xc2\xe1\x4e\x36\xd4\xfc\x10\x6d\xcb\x29\xb4", 1, + 0, 96 }, + { 96, 256, 128, 27, + "\x85\xf3\x5b\x62\x82\xcf\xf4\x40\xbc\x10\x20\xc8\x13\x6f\xf2\x70\x31" + "\x11\x0f\xa6\x3e\xc1\x6f\x1e\x82\x51\x18\xb0\x06\xb9\x12\x57", + "\x58\xdb\xd4\xad\x2c\x4a\xd3\x5d\xd9\x06\xe9\xce", + "\xa5\x06\xe1\xa5\xc6\x90\x93\xf9", + "\xfd\xc8\x5b\x94\xa4\xb2\xa6\xb7\x59\xb1\xa0\xda", + "\x9f\x88\x16\xde\x09\x94\xe9\x38\xd9\xe5\x3f\x95", + "\xd0\x86\xfc\x6c\x9d\x8f\xa9\x15\xfd\x84\x23\xa7\xcf\x05\x07\x2f", 1, + 64, 96 }, + { 96, 256, 128, 28, + "\xfa\xf4\xbf\xe8\x01\x9a\x89\x1c\x74\x90\x1b\x17\xf4\xf4\x8c\xee\x5c" + "\xd0\x65\xd5\x5f\xde\xa6\x01\x18\xaa\xf6\xc4\x31\x9a\x0e\xa5", + "\xb7\x76\xc3\xfd\xdb\xa7\xc8\x13\x62\xce\x6e\x1b", "", + "\x8d\xad\xff\x8d\x60\xc8\xe8\x8f\x60\x4f\x27\x48\x33", + "\xe6\xb3\x3a\x74\xa4\xac\x44\x3b\xd9\x3f\x9c\x1b\x94", + "\x0c\x11\x51\x72\xbd\xb0\x2b\xba\xd3\x13\x0f\xff\x22\x79\x0d\x60", 1, + 0, 104 }, + { 96, 256, 128, 29, + "\x84\x10\x20\xd1\x60\x6e\xdc\xfc\x53\x6a\xbf\xb1\xa6\x38\xa7\xb9\x58" + "\xe2\x1e\xfc\x10\xc3\x86\xac\x45\xa1\x84\x93\x45\x0a\xfd\x5f", + "\x6d\x62\xf1\x59\x73\x1b\x14\x0e\xb1\x8c\xe0\x74", + "\x5a\x8e\x1c\x7a\xa3\x98\x10\xd5", + "\xd6\xaf\x13\x8f\x70\x1b\x80\x1e\x60\xc8\x5f\xfd\x5c", + "\xb0\xa7\x50\x0a\xca\x45\xbb\x15\xf0\x1e\xce\x43\x89", + "\x01\x60\xe8\x3a\xdb\xec\x7f\x6a\x2e\xe2\xff\x02\x15\xf9\xef\x00", 1, + 64, 104 }, + { 96, 256, 128, 30, + "\x47\x0f\x9c\xe3\xd2\x25\x0b\xd6\x0c\xbb\xef\xdb\x2e\x6a\x11\x78\xc0" + "\x12\x29\x9b\x55\x90\x63\x9c\x77\x97\xb6\x02\x4f\xa7\x03\xd8", + "\xa9\xea\x4d\x61\x9f\xe4\x05\xd0\x4c\xba\x7d\x7a", "", + "\x6c\xa6\x7d\xd0\x23\xfb\xa6\x50\x7b\x9f\x9a\x1f\x66\x7e", + "\xd3\x01\x7e\x0b\xb1\x70\x5b\x38\x0b\x34\xcc\x33\x34\x50", + "\x57\x08\xe7\x2c\xa2\xbd\x35\x4f\x48\x7f\x82\xf6\x7f\xbc\x3a\xcb", 1, + 0, 112 }, + { 96, 256, 128, 31, + "\xe4\xb9\x7e\x91\xe4\xc8\xe8\x5e\xb7\xce\x0a\x7f\x30\xbf\x8a\x0a\xbf" + "\x44\x68\x25\x1e\x4c\x63\x86\xc0\xe7\xaa\xcb\x8e\x87\x9a\xa8", + "\x0e\x23\xc9\x42\xa0\xc9\xfb\x52\x65\x86\xee\xad", + "\xea\xaa\xea\xb2\x69\x57\xf9\xa1", + "\xb8\x4b\x3f\x74\xcd\x23\x06\x4b\xb4\x26\xfe\x2c\xed\x2b", + "\x52\xe9\x67\x2b\x41\x6d\x84\xd9\x70\x33\x79\x60\x72\xd0", + "\xe8\x38\x39\xdc\x1f\xd9\xb8\xb9\xd1\x44\x4c\x40\xe4\x88\xd4\x93", 1, + 64, 112 }, + { 96, 256, 128, 32, + "\x67\x11\x96\x27\xbd\x98\x8e\xda\x90\x62\x19\xe0\x8c\x0d\x0d\x77\x9a" + "\x07\xd2\x08\xce\x8a\x4f\xe0\x70\x9a\xf7\x55\xee\xec\x6d\xcb", + "\x68\xab\x7f\xdb\xf6\x19\x01\xda\xd4\x61\xd2\x3c", "", + "\x51\xf8\xc1\xf7\x31\xea\x14\xac\xdb\x21\x0a\x6d\x97\x3e\x07", + "\x0b\x29\x63\x8e\x1f\xbd\xd6\xdf\x53\x97\x0b\xe2\x21\x00\x42", + "\x2a\x91\x34\x08\x7d\x67\xa4\x6e\x79\x17\x8d\x0a\x93\xf5\xe1\xd2", 1, + 0, 120 }, + { 96, 256, 128, 33, + "\xe6\xf1\x11\x8d\x41\xe4\xb4\x3f\xb5\x82\x21\xb7\xed\x79\x67\x38\x34" + "\xe0\xd8\xac\x5c\x4f\xa6\x0b\xbc\x8b\xc4\x89\x3a\x58\x89\x4d", + "\xd9\x5b\x32\x43\xaf\xae\xf7\x14\xc5\x03\x5b\x6a", + "\x64\x53\xa5\x33\x84\x63\x22\x12", + "\x97\x46\x9d\xa6\x67\xd6\x11\x0f\x9c\xbd\xa1\xd1\xa2\x06\x73", + "\x32\xdb\x66\xc4\xa3\x81\x9d\x81\x55\x74\x55\xe5\x98\x0f\xed", + "\xfe\xae\x30\xde\xc9\x4e\x6a\xd3\xa9\xee\xa0\x6a\x0d\x70\x39\x17", 1, + 64, 120 }, + { 96, 256, 128, 34, + "\x59\xd4\xea\xfb\x4d\xe0\xcf\xc7\xd3\xdb\x99\xa8\xf5\x4b\x15\xd7\xb3" + "\x9f\x0a\xcc\x8d\xa6\x97\x63\xb0\x19\xc1\x69\x9f\x87\x67\x4a", + "\x2f\xcb\x1b\x38\xa9\x9e\x71\xb8\x47\x40\xad\x9b", "", + "\x54\x9b\x36\x5a\xf9\x13\xf3\xb0\x81\x13\x1c\xcb\x6b\x82\x55\x88", + "\xe9\x11\x0e\x9f\x56\xab\x3c\xa4\x83\x50\x0c\xea\xba\xb6\x7a\x13", + "\x83\x6c\xca\xbf\x15\xa6\xa2\x2a\x51\xc1\x07\x1c\xfa\x68\xfa\x0c", 1, + 0, 128 }, + { 96, 256, 128, 35, + "\xb9\x07\xa4\x50\x75\x51\x3f\xe8\xa8\x01\x9e\xde\xe3\xf2\x59\x14\x87" + "\xb2\xa0\x30\xb0\x3c\x6e\x1d\x77\x1c\x86\x25\x71\xd2\xea\x1e", + "\x11\x8a\x69\x64\xc2\xd3\xe3\x80\x07\x1f\x52\x66", + "\x03\x45\x85\x62\x1a\xf8\xd7\xff", + "\x55\xa4\x65\x64\x4f\x5b\x65\x09\x28\xcb\xee\x7c\x06\x32\x14\xd6", + "\xe4\xb1\x13\xcb\x77\x59\x45\xf3\xd3\xa8\xae\x9e\xc1\x41\xc0\x0c", + "\x7c\x43\xf1\x6c\xe0\x96\xd0\xdc\x27\xc9\x58\x49\xdc\x38\x3b\x7d", 1, + 64, 128 }, + { 96, 256, 128, 36, + "\x3b\x24\x58\xd8\x17\x6e\x16\x21\xc0\xcc\x24\xc0\xc0\xe2\x4c\x1e\x80" + "\xd7\x2f\x7e\xe9\x14\x9a\x4b\x16\x61\x76\x62\x96\x16\xd0\x11", + "\x45\xaa\xa3\xe5\xd1\x6d\x2d\x42\xdc\x03\x44\x5d", "", + "\x3f\xf1\x51\x4b\x1c\x50\x39\x15\x91\x8f\x0c\x0c\x31\x09\x4a\x6e" + "\x1f", + "\x02\xcc\x3a\xcb\x5e\xe1\xfc\xdd\x12\xa0\x3b\xb8\x57\x97\x64\x74" + "\xd3", + "\xd8\x3b\x74\x63\xa2\xc3\x80\x0f\xe9\x58\xc2\x8e\xaa\x29\x08\x13", 1, + 0, 136 }, + { 96, 256, 128, 37, + "\xf6\x0c\x6a\x1b\x62\x57\x25\xf7\x6c\x70\x37\xb4\x8f\xe3\x57\x7f\xa7" + "\xf7\xb8\x7b\x1b\xd5\xa9\x82\x17\x6d\x18\x23\x06\xff\xb8\x70", + "\xf0\x38\x4f\xb8\x76\x12\x14\x10\x63\x3d\x99\x3d", + "\x9a\xaf\x29\x9e\xee\xa7\x8f\x79", + "\x63\x85\x8c\xa3\xe2\xce\x69\x88\x7b\x57\x8a\x3c\x16\x7b\x42\x1c" + "\x9c", + "\x35\x76\x64\x88\xd2\xbc\x7c\x2b\x8d\x17\xcb\xbb\x9a\xbf\xad\x9e" + "\x6d", + "\x1f\x39\x1e\x65\x7b\x27\x38\xdd\xa0\x84\x48\xcb\xa2\x81\x1c\xeb", 1, + 64, 136 }, + { 96, 256, 128, 38, + "\x37\xce\xb5\x74\xcc\xb0\xb7\x01\xdd\x11\x36\x93\x88\xca\x27\x10\x17" + "\x32\x33\x9f\x49\xd8\xd9\x08\xac\xe4\xb2\x3a\xf0\xb7\xce\x89", + "\x37\x27\x0b\x36\x8f\x6b\x1e\x3e\x2c\xa5\x17\x44", "", + "\xf2\x69\x91\x53\x72\x57\x37\x81\x51\xf4\x77\x6a\xad\x28\xae\x8b\xd1" + "\x6b", + "\xb6\x21\xd7\x6a\x8d\xac\xff\x00\xb3\xf8\x40\xcd\xf2\x6c\x89\x4c\xc5" + "\xd1", + "\xe0\xa2\x17\x16\xed\x94\xc0\x38\x2f\xa9\xb0\x90\x3d\x15\xbb\x68", 1, + 0, 144 }, + { 96, 256, 128, 39, + "\x68\x88\x83\x61\x91\x9b\xc1\x06\x22\xf4\x5d\xf1\x68\xe5\xf6\xa0\x3b" + "\xd8\xe8\x84\xc0\x61\x1b\xea\x2f\x34\xc1\x88\x2e\xd9\x83\x2b", + "\xbf\xd6\xff\x40\xf2\xdf\x8c\xa7\x84\x59\x80\xcc", + "\xb8\x37\x34\x38\xdd\xb2\xd6\xc3", + "\xff\x97\xf2\xee\xfb\x34\x01\xac\x31\xfc\x8d\xc1\x59\x0d\x1a\x92\xcb" + "\xc1", + "\xe0\xa7\x45\x18\x6c\x1a\x7b\x14\x7f\x74\xfa\xff\x2a\x71\x5d\xf5\xc1" + "\x9d", + "\x91\x7b\xaf\x70\x3e\x35\x5d\x4d\x95\x0e\x6c\x05\xfe\x8f\x34\x9f", 1, + 64, 144 }, + { 96, 256, 128, 40, + "\x1b\x35\xb8\x56\xb5\xa8\x6d\x34\x03\xd2\x8f\xc2\x10\x3a\x63\x1d\x42" + "\xde\xca\x51\x75\xcd\xb0\x66\x9a\x5e\x5d\x90\xb2\xca\xaf\xc5", + "\x23\x43\xde\x88\xbe\x6c\x71\x96\xd3\x3b\x86\x94", "", + "\x21\xef\x18\x5c\x3a\xe9\xa9\x6f\xa5\xeb\x47\x38\x78\xf4\xd0\xb2\x42" + "\x78\x1d", + "\xd6\xe0\xed\x54\xfc\xce\xf3\x0b\xd6\x05\xd7\x2d\xa3\x32\x0e\x24\x9a" + "\x9c\xb5", + "\xc6\x8b\xc6\x72\x4e\xc8\x03\xc4\x39\x84\xce\x42\xf6\xbd\x09\xff", 1, + 0, 152 }, + { 96, 256, 128, 41, + "\xd6\x48\x4e\x39\x73\xf6\xbe\x8c\x83\xed\x32\x08\xd5\xbe\x5c\xfa\x06" + "\xfd\xa7\x2f\xbf\xdc\x5b\x19\xd0\x9b\xe3\xf4\xe4\xeb\xa2\x9d", + "\x1a\xf1\xd9\x0e\x87\x7e\x11\xa4\x96\xef\xa3\xdf", + "\xcc\x4e\xfd\x83\x64\xfb\x11\x4a", + "\x73\x35\xab\x04\xb0\x3e\x70\x61\x09\xec\x3e\xe8\x35\xdb\x9a\x24\x6e" + "\xa0\xad", + "\x29\xe5\x4d\x60\x82\x37\xc3\xc3\x60\x9d\xba\x16\xe6\xed\xf4\x38\x42" + "\xd7\x2f", + "\xd3\x36\x5f\xdc\xd5\x06\xaa\xaa\x53\x68\x66\x1e\x80\xe9\xd9\x9b", 1, + 64, 152 }, + { 96, 256, 128, 42, + "\x42\x2a\xdd\x37\x84\x9d\x6e\x4c\x3d\xfd\x80\x20\xdc\x6a\x07\xe8\xa2" + "\x49\x78\x8f\x3d\x6a\x83\xb9\xcb\x4d\x80\x23\x62\xc9\x75\x42", + "\x1e\x7e\x67\xbe\x94\x8d\xe7\x35\x2f\xfd\xb7\x27", "", + "\xd7\xf5\xe6\x11\xdd\x3a\x27\x50\xfb\x84\x3f\xc1\xb6\xb9\x30\x87\x31" + "\x0d\xc8\x7d", + "\x7f\xe6\x06\x65\x2d\x85\x8f\x59\x5e\xc2\xe7\x06\x75\x4f\xa3\xd9\x33" + "\xfc\xc8\x34", + "\x78\xd5\x92\x35\xaa\x5d\x03\xa4\xc3\x25\x90\xe5\x90\xc0\x4d\x22", 1, + 0, 160 }, + { 96, 256, 128, 43, + "\xcd\xcc\xfe\x3f\x46\xd7\x82\xef\x47\xdf\x4e\x72\xf0\xc0\x2d\x9c\x7f" + "\x77\x4d\xef\x97\x0d\x23\x48\x6f\x11\xa5\x7f\x54\x24\x7f\x17", + "\x37\x61\x87\x89\x46\x05\xa8\xd4\x5e\x30\xde\x51", + "\x95\x68\x46\xa2\x09\xe0\x87\xed", + "\xe2\x8e\x0e\x9f\x9d\x22\x46\x3a\xc0\xe4\x26\x39\xb5\x30\xf4\x21\x02" + "\xfd\xed\x75", + "\x14\xf7\x07\xc4\x46\x98\x8a\x49\x03\x77\x5e\xc7\xac\xec\x6d\xa1\x14" + "\xd4\x31\x12", + "\x98\x7d\x4b\x14\x7c\x49\x0d\x43\xd3\x76\xa1\x98\xca\xb3\x83\xf0", 1, + 64, 160 }, + { 96, 256, 128, 44, + "\xe7\x9d\xfc\x6d\x2f\xc4\x65\xb8\x43\x9e\x1c\x5b\xac\xcb\x5d\x8e\xf2" + "\x85\x38\x99\xfc\x19\x75\x3b\x39\x7e\x6c\x25\xb3\x5e\x97\x7e", + "\xf9\xd6\x32\x0d\x7c\xe5\x1d\x8e\xd0\x67\x7d\x3a", "", + "\x4f\x54\x3e\x79\x38\xd1\xb8\x78\xda\xca\xee\xc8\x1d\xce\x48\x99\x97" + "\x48\x16\x81\x3b", + "\x10\x03\xf1\x3e\xa1\x32\x9c\xbb\x18\x73\x16\xf6\x4c\x3f\xf3\xa8\x7c" + "\xf5\xb9\x66\x61", + "\xd2\x32\x3a\xd6\x25\x09\x4b\xec\x84\x79\x0d\x79\x58\xd5\x58\x3f", 1, + 0, 168 }, + { 96, 256, 128, 45, + "\x1d\x7b\x8f\x1d\x96\xa1\x42\x49\x23\xae\xf8\xa9\x84\x86\x9d\x4a\x77" + "\x7a\x11\x09\x90\xba\x46\x56\x27\xac\xf8\x03\x96\xc7\xf3\x76", + "\x50\xba\x19\x62\xcd\xc3\x2a\x5a\x2d\x36\xe6\x40", + "\x09\x30\x53\xe2\x02\x61\xda\xab", + "\x5d\x3e\xfd\x57\x67\xf3\xc1\x2e\xfd\x08\xaf\x9a\x44\xe0\x28\xae\x68" + "\xc9\xef\xf8\x43", + "\x2d\x48\xb0\x83\x4e\x9f\xfe\x30\x46\x10\x3e\xf7\xa2\x14\xf0\x2e\x8e" + "\x4d\x33\x36\x0e", + "\xd5\x33\xad\x08\x9b\xe2\x29\xea\x60\x6e\xc0\xf3\xfa\x22\xeb\x33", 1, + 64, 168 }, + { 96, 256, 128, 46, + "\xdd\x43\x3e\x28\xcf\xbc\xb5\xde\x4a\xb3\x6a\x02\xbf\x38\x68\x6d\x83" + "\x20\x87\x71\xa0\xe6\x3d\xcd\x08\xb4\xdf\x1a\x07\xac\x47\xa1", + "\xc9\xcc\x0a\x1a\xfc\x38\xec\x6c\x30\xc3\x8c\x68", "", + "\x8a\x3e\x17\xab\xa9\x60\x6d\xd4\x9e\x3b\x1a\x4d\x9e\x5e\x42\xf1\x74" + "\x23\x73\x63\x24\x89", + "\xe9\x91\x7f\xf3\xe6\x4b\xbe\x17\x83\x57\x93\x75\xe7\x5e\xa8\x23\x97" + "\x6b\x35\x53\x99\x49", + "\x07\x4a\x89\x06\x69\xb2\x51\x05\x43\x4c\x75\xbe\xed\x32\x48\xdb", 1, + 0, 176 }, + { 96, 256, 128, 47, + "\xa6\x09\x24\x10\x1b\x42\xac\x24\x15\x4a\x88\xde\x42\x14\x2b\x23\x34" + "\xcf\x59\x91\x76\xca\xf4\xd1\x22\x6f\x71\x2d\xd9\x17\x29\x30", + "\x8b\xa7\x76\x44\xb0\x8d\x65\xd5\xe9\xf3\x19\x42", + "\xb2\xa4\xe1\x2a\x19\xa6\x1c\x75", + "\xc9\x49\x95\x7e\x66\x43\x9d\xee\xe4\xb2\xac\x1d\x4a\x6c\x98\xa6\xc5" + "\x27\xb9\x0f\x52\xab", + "\xdb\x4c\x70\x05\x13\x81\x89\x72\xb0\xdc\x0e\x53\x1b\x1c\x28\x1c\xa0" + "\x3e\x40\xc6\x0d\xea", + "\x63\xf4\x47\x8b\xba\x2a\xf4\x69\xa7\xa4\xdc\x3b\x4f\x14\x13\x60", 1, + 64, 176 }, + { 96, 256, 128, 48, + "\x1a\xa4\x20\x27\x83\x69\x65\xb1\xe6\x08\x6f\xa1\x37\xf9\xcf\x7f\x1f" + "\xf4\x86\x76\x69\x68\x29\xbd\x28\x1f\xf8\x1c\x8e\xa0\xa4\xa9", + "\x4b\x3d\xca\x84\xec\xc4\x07\xf4\x24\xf2\x81\xa9", "", + "\x37\x25\x2a\x3e\xb5\xc8\x96\x0f\x05\x67\xe5\x03\xa9\x03\x57\x83\xb3" + "\xd0\xa1\x9a\x4b\x9a\x47", + "\xb5\xf1\x46\x17\x49\x1f\xc9\x23\xb6\x83\xe2\xcc\x95\x62\xd0\x43\xdd" + "\x59\x86\xb9\x7d\xbd\xbd", + "\x97\x2c\xe5\x47\x13\xc0\x5c\x4b\xb4\xd0\x88\xc0\xa3\x0c\xac\xd3", 1, + 0, 184 }, + { 96, 256, 128, 49, + "\x5d\x40\xdb\x0c\xc1\x8e\xf2\xe4\x28\x15\xd3\xb6\x24\x5a\x46\x6a\x0b" + "\x30\xa0\xf9\x3e\x31\x8a\xc1\x0e\xdd\xe3\xbf\x8a\xd9\x81\x60", + "\xac\xad\x61\x80\x39\xb3\x17\x47\x0d\x21\x62\x1b", + "\x41\x30\x36\x41\x1a\xf7\x57\x45", + "\x95\x9d\xde\x1e\xf3\x12\x9b\x27\x70\x2c\x55\x88\x49\xe4\x66\xf2\xba" + "\xca\x1a\x45\xbd\xf4\xb2", + "\xb7\xca\x38\x79\xf9\x51\x40\xbf\x6a\x97\xb3\x21\x22\x18\xb7\xbf\x86" + "\x4a\x51\xe5\xbb\x0b\x3e", + "\xfe\x55\x8f\xb5\x70\x14\x54\x70\xea\x69\x3e\xb7\x6e\xb7\x31\x71", 1, + 64, 184 }, + { 96, 256, 128, 50, + "\x02\x12\xa8\xde\x50\x07\xed\x87\xb3\x3f\x1a\x70\x90\xb6\x11\x4f\x9e" + "\x08\xce\xfd\x96\x07\xf2\xc2\x76\xbd\xcf\xdb\xc5\xce\x9c\xd7", + "\xe6\xb1\xad\xf2\xfd\x58\xa8\x76\x2c\x65\xf3\x1b", "", + "\x10\xf1\xec\xf9\xc6\x05\x84\x66\x5d\x9a\xe5\xef\xe2\x79\xe7\xf7\x37" + "\x7e\xea\x69\x16\xd2\xb1\x11", + "\x42\xf2\x6c\x56\xcb\x4b\xe2\x1d\x9d\x8d\x0c\x80\xfc\x99\xdd\xe0\x0d" + "\x75\xf3\x80\x74\xbf\xe7\x64", + "\x54\xaa\x7e\x13\xd4\x8f\xff\x7d\x75\x57\x03\x94\x57\x04\x0a\x3a", 1, + 0, 192 }, + { 96, 256, 128, 51, + "\xc5\xbc\x09\x56\x56\x46\xe7\xed\xda\x95\x4f\x1f\x73\x92\x23\xda\xda" + "\x20\xb9\x5c\x44\xab\x03\x3d\x0f\xae\x4b\x02\x83\xd1\x8b\xe3", + "\x6b\x28\x2e\xbe\xcc\x54\x1b\xcd\x78\x34\xed\x55", + "\x3e\x8b\xc5\xad\xe1\x82\xff\x08", + "\x92\x22\xf9\x01\x8e\x54\xfd\x6d\xe1\x20\x08\x06\xa9\xee\x8e\x4c\xc9" + "\x04\xd2\x9f\x25\xcb\xa1\x93", + "\x12\x30\x32\x43\x7b\x4b\xfd\x69\x20\xe8\xf7\xe7\xe0\x08\x7a\xe4\x88" + "\x9e\xbe\x7a\x0a\xd0\xe9\x00", + "\x3c\xf6\x8f\x17\x95\x50\xda\x63\xd3\xb9\x6c\x2d\x55\x41\x18\x65", 1, + 64, 192 }, + { 96, 256, 128, 52, + "\x94\x60\xb3\xc4\x4e\xd8\x6e\x70\xf3\xbd\xa6\x63\x85\xe1\xca\x10\xb0" + "\xc1\x67\x7e\xf4\xf1\x36\x05\x32\x83\x0d\x17\x53\x5f\x99\x6f", + "\xab\xfa\xf4\x2e\x0d\xba\x88\x4e\xfc\xf0\x78\x23", "", + "\x5c\x5c\xce\x88\x1b\x93\xfb\x7a\x1b\x79\x39\xaf\x1f\xfc\x5f\x84\xd3" + "\x28\x0a\xda\x77\x8c\xca\x09\x53", + "\x1d\x21\x8c\x9f\x1f\x9f\x02\xf2\x48\xa6\xf9\x76\xa7\x55\x70\x57\xf3" + "\x7d\x93\x93\xd9\xf2\x13\xc1\xf3", + "\xbc\x88\x34\x4c\x6f\xdc\x89\x8f\xee\xd3\x94\xfb\x28\x51\x13\x16", 1, + 0, 200 }, + { 96, 256, 128, 53, + "\xc1\x11\xd6\xd5\xd7\x8a\x07\x1b\x15\xab\x37\xcc\x8c\x38\x19\x19\x93" + "\x87\xab\x7c\x19\x33\xaa\x97\xb1\x48\x9f\x65\x84\xba\x8e\x2a", + "\x85\xf1\x8a\xd8\xff\x72\xca\xfe\xe2\x45\x2a\xb8", + "\x84\xcd\xff\x93\x93\x91\xc0\x22", + "\x69\x89\xc6\x46\xa1\x0b\x7c\x76\xf4\xd9\xf7\xd5\x74\xda\x40\xe1\x52" + "\x01\x3c\xf0\xdd\x78\xf5\xaa\x8a", + "\x97\x15\xd3\x44\xe8\xd3\xf3\xa3\xea\xa9\x8a\x9c\xea\x57\xc0\xcd\x71" + "\x7c\x6e\xf5\x07\x60\x27\xc9\xec", + "\x30\x56\xff\x5e\xe0\xaa\x86\x36\xbb\x63\x99\x84\xed\xb5\x23\x6b", 1, + 64, 200 }, + { 96, 256, 128, 54, + "\x8a\x1b\x1e\x69\x9a\x0c\x4a\x3e\x61\x0b\x10\x90\x2d\xae\xda\xb1\xbf" + "\x1e\xa0\xd5\x05\xc4\x7d\x78\x42\xcb\xce\xe0\xd3\xb1\xb6\xe6", + "\xa6\xf9\xa8\xd3\x35\xfa\x84\xc3\xb2\x7d\xcd\x2a", "", + "\xee\x6a\x15\xfc\x18\x31\x08\xf0\x87\x7e\x7f\x2b\x8a\x96\x15\xf4\xb3" + "\xfc\x36\xe1\xc8\x34\x40\xf6\x6a\xad", + "\x90\x89\xbb\xdb\x8b\xcf\xd1\x24\xe2\x27\xbf\x75\xc4\xbf\xe1\xcb\xa2" + "\x00\x4a\x27\x4f\xc3\x1a\xa3\x23\x58", + "\xfd\x2e\x21\xc6\x4a\x01\x96\x21\xc6\x85\x94\x82\x6c\xd7\xb1\xcd", 1, + 0, 208 }, + { 96, 256, 128, 55, + "\x74\xb3\x84\xe6\xe0\x13\xec\x41\x72\xed\x7a\x28\xa1\x0f\xb9\xbb\x79" + "\xb4\xbe\x2a\x24\xf6\x99\x9e\x3d\x3c\xaa\x28\xe6\x4a\x86\x56", + "\xeb\xc1\x9f\xc9\xec\xb2\x33\x99\x08\xea\x38\x36", + "\x85\x07\x3f\x2e\xdc\x13\xd3\xa1", + "\x3a\xa9\xf7\x37\x2f\x05\x6e\x5a\x07\x29\x75\x2d\x9a\x37\x13\x2d\x6d" + "\xd0\x7c\x56\x79\x2e\x1c\x75\x82\xa9", + "\x79\x6f\xfb\x70\xab\x43\xe7\xfa\x79\xf9\x55\x83\xe3\x84\x52\x47\x27" + "\xbb\x3e\x47\xfc\x45\xb9\x69\xf7\x14", + "\xc3\x32\x2b\x44\x45\xde\x5f\x3c\x9f\x18\xdc\xc8\x47\xcc\x94\xc3", 1, + 64, 208 }, + { 96, 256, 128, 56, + "\x77\xd8\x24\x79\x5d\x20\x29\xf0\xeb\x0e\x0b\xaa\xb5\xcf\xeb\x32\xf7" + "\xe9\x34\x74\x91\x3a\x7f\x95\xc7\x37\xa6\x67\xa3\xc3\x33\x14", + "\xf3\x30\x74\x30\xf4\x92\xd2\xb8\xa7\x2d\x3a\x81", "", + "\x0c\x41\x79\xa4\x97\xd8\xfd\xd7\x27\x96\xfb\x72\x56\x92\xb8\x05\xd6" + "\x3b\x7c\x71\x83\x59\xcf\x10\x51\x8a\xee", + "\x49\xc8\x1d\x17\xd6\x7d\x7b\xa9\x95\x4f\x49\x7d\x0b\x0d\xdc\x21\xf3" + "\xf8\x39\xc9\xd2\xcc\x19\x8d\x30\xbc\x2c", + "\x50\x00\x98\x99\xe5\xb2\xa9\x72\x6c\x8f\x35\x56\xca\xdf\xbe\x84", 1, + 0, 216 }, + { 96, 256, 128, 57, + "\xbe\xc5\xea\xc6\x8f\x89\x39\x51\xcb\xd7\xd1\xec\xd3\xee\x66\x11\x13" + "\x0d\xd9\xc3\xf8\x0c\xdd\xf9\x51\x11\xd0\x7d\x5e\xdd\x76\xd1", + "\x34\x2a\xda\x4f\x0c\x11\x51\x24\xb2\x22\xdf\x80", + "\x73\x36\x5f\x6d\x80\xed\xb1\xd8", + "\x48\x14\x33\xd8\xb1\xcd\x38\xaf\x4a\x75\x0e\x13\xa6\x4b\x7a\x4e\x85" + "\x07\x68\x2b\x35\x17\x59\x59\x38\xa2\x0e", + "\x4c\x12\x9f\xc1\x3c\xbd\xd9\xd3\xfe\x81\xac\x75\x5b\xf4\xfb\xea\x2f" + "\xdd\x7e\x0a\xca\x05\x05\xa6\xee\x96\x37", + "\x9c\xed\xe1\xd3\x0a\x03\xdb\x5d\x55\x26\x5d\x36\x48\xbc\x40\xd4", 1, + 64, 216 }, + { 96, 256, 128, 58, + "\xa5\x9c\x1e\x13\x06\x4d\xf8\xf2\xb8\xdf\x77\xa4\x92\xb0\xca\x2e\xae" + "\x92\x1b\x52\xa8\x4b\x30\x5a\x3a\x9a\x51\x40\x8a\x9e\xcb\x69", + "\x95\x44\xd4\x1e\xce\x0c\x92\xef\x01\xcf\xac\x2d", "", + "\x1c\x35\xb8\x98\x82\x1b\xa5\x5c\x26\x17\xc2\x5d\xf9\xe6\xdf\x2a\x80" + "\x02\xb3\x84\x90\x21\x86\xcd\x69\xdf\xd2\x0e", + "\xa6\xfa\x8f\x57\xdd\xc8\x1d\x60\x99\xf6\x67\xdd\x62\x40\x2b\x6a\x5d" + "\x5b\x7d\x05\xa3\x29\x29\x80\x29\x11\x31\x69", + "\xbb\x24\xe3\x8b\x31\xdb\xbc\x3e\x57\x5b\x9e\x3e\xe0\x76\xaf\x2a", 1, + 0, 224 }, + { 96, 256, 128, 59, + "\x08\x4b\x5d\x73\x65\xf1\xa8\xfe\xc6\x36\x59\x39\xed\x74\x1e\x6e\xa5" + "\x89\x3e\x03\x18\xd8\x2a\xb4\x75\x00\xa9\x7d\x77\xaa\xa0\x41", + "\x82\x9f\x00\x5e\x98\x0f\x0a\x6e\x2f\x98\x3e\xaa", + "\x77\x0f\x6e\x6e\x89\xa3\xfe\x8e", + "\x75\x10\x01\x6e\xfa\xdc\x38\x5a\x71\xed\x68\x9c\xeb\x59\x0c\x8e\xa9" + "\xcc\x1e\x81\xb7\x93\x33\x8b\xdd\xf5\xf1\x0c", + "\xfd\x42\xcb\x5c\xf8\x94\xf8\x79\xe3\xcf\x75\x16\x62\xaa\xa5\x8a\x22" + "\x88\xcc\x53\x54\x88\x02\xbe\xca\xf4\x23\x59", + "\x18\x83\x29\x43\x8a\xfe\x1c\xd7\x22\x5d\x04\x78\xaa\x90\xc7\x73", 1, + 64, 224 }, + { 96, 256, 128, 60, + "\x5a\x7f\x85\x0a\x1d\x9a\xaf\xa7\x7d\x59\xae\x1b\x73\x19\x65\xe8\xaa" + "\xec\x63\x52\x28\x0f\xc7\x6a\x7b\x5e\x23\xef\x36\x10\xcf\xe4", + "\x49\x46\xa0\xd6\xad\xea\x93\xb8\x2d\x43\x32\xe5", "", + "\x3c\x16\x1d\x79\x1f\x62\x4f\xb0\x38\x8e\x80\x8f\x0f\x69\xed\x79\x0d" + "\xbe\x4c\xbd\x08\x9e\xba\xc4\x66\x27\xbc\xf0\x1d", + "\x40\x23\x02\xb5\x61\x40\xc4\xdc\xc3\x97\x74\x73\x2c\x55\x88\x3d\xe1" + "\x24\xce\x4b\xf0\xa0\x26\x1c\xfa\x15\x69\xe2\xcf", + "\xe8\x30\xbf\xe9\x33\xa9\x67\x86\xcf\xf2\xdd\x72\xb8\x2c\x4b\xd5", 1, + 0, 232 }, + { 96, 256, 128, 61, + "\xe6\xd5\xa4\x24\x6f\x6f\x05\x61\x8b\x59\xc8\xf9\xec\x3a\xc8\x06\x8c" + "\xc0\xd3\xf3\x51\xc5\x71\xaa\x52\xb0\x9c\xb2\x51\xf9\xc2\xf6", + "\x2f\x90\xa6\x5e\x9e\x48\x72\x5d\xe6\xff\xc7\x27", + "\xf2\x41\x53\x77\xad\x28\x3f\xd8", + "\x96\x4f\xc9\xe0\xe8\x35\x59\x47\xaa\x1c\x2c\xaa\xdd\x7b\x3d\xbe\xf8" + "\x2a\x10\x24\xe6\x23\x60\x6f\xac\x43\x6e\xf5\x73", + "\xd0\x52\x93\x2b\xad\x6e\x6c\x4f\x83\x5f\x02\x01\x9e\x52\xd7\xff\x80" + "\x7d\xc2\xa5\xaa\xc2\x04\x08\x83\xc7\x9d\xd3\xd5", + "\x65\x5f\x93\x39\x6b\x4d\x75\x5d\xc4\x47\x57\x21\x66\x5f\xed\x91", 1, + 64, 232 }, + { 96, 256, 128, 62, + "\x09\xe8\x22\x12\x3a\xdb\xb1\xed\x89\xb7\x9a\x58\x61\x9c\x64\x85\x39" + "\x92\xf8\x37\x1d\x46\x33\x87\x12\xf6\xc9\x1a\xb1\x1a\x68\xbb", + "\xa7\x97\x20\x5a\x6c\xac\xdd\x7e\x47\xa4\x78\x9d", "", + "\x80\xb7\x1b\xbe\x83\x36\x29\x84\x1b\xd3\xae\xae\xb9\xdb\x61\x23\xe5" + "\x1d\x36\x7b\x43\x6f\xe9\xd2\xd3\x45\x4b\x62\xcf\xad", + "\x83\xf5\xc7\x73\x96\xca\xbd\x28\xdf\xcc\x00\x2c\xba\x07\x56\xd4\xea" + "\x54\x55\xe0\x26\x1d\x84\x7d\x57\x08\xaa\xc2\x1e\x8d", + "\x70\x5a\x05\x82\x0a\x21\xf3\x81\xd2\x44\xd4\x0e\x58\xd2\xf1\x6b", 1, + 0, 240 }, + { 96, 256, 128, 63, + "\x62\x57\x35\xfe\x7f\x8f\xc8\x1b\x0c\x1e\xdc\x3d\x08\xa7\x8b\x41\x26" + "\x8f\x87\xa3\xc6\x84\x88\xb6\x74\x22\x26\x30\xc1\xd5\x87\xa5", + "\x9d\x8c\xdf\x28\x9d\xdd\xd0\x9a\xfd\xc1\xb0\x2f", + "\x20\x0a\x9c\x95\x94\x6f\xf0\x5c", + "\x67\xae\x18\x82\xd0\xb1\xc1\xb2\x48\x5b\xec\x98\x11\x5e\xcf\x53\xb9" + "\xb4\x38\xde\xb1\xd0\x40\x05\x31\x70\x50\x38\x87\x3a", + "\x20\x9b\x75\x39\x38\x5c\x8b\x19\xec\xd0\xfd\x8b\x50\x11\xb2\x99\x6e" + "\x31\x6f\x19\x42\x06\x4e\x68\xed\xfa\x36\x3a\xcb\xcd", + "\xfa\x2f\x45\x4b\x9f\xa2\x60\x8f\x78\x0f\x7c\x6f\x9b\x78\x0f\xe1", 1, + 64, 240 }, + { 96, 256, 128, 64, + "\x2e\xb5\x1c\x46\x9a\xa8\xeb\x9e\x6c\x54\xa8\x34\x9b\xae\x50\xa2\x0f" + "\x0e\x38\x27\x11\xbb\xa1\x15\x2c\x42\x4f\x03\xb6\x67\x1d\x71", + "\x04\xa9\xbe\x03\x50\x8a\x5f\x31\x37\x1a\x6f\xd2", "", + "\xb0\x53\x99\x92\x86\xa2\x82\x4f\x42\xcc\x8c\x20\x3a\xb2\x4e\x2c\x97" + "\xa6\x85\xad\xcc\x2a\xd3\x26\x62\x55\x8e\x55\xa5\xc7\x29", + "\x45\xc7\xd6\xb5\x3a\xca\xd4\xab\xb6\x88\x76\xa6\xe9\x6a\x48\xfb\x59" + "\x52\x4d\x2c\x92\xc9\xd8\xa1\x89\xc9\xfd\x2d\xb9\x17\x46", + "\x56\x6d\x3c\xa1\x0e\x31\x1b\x69\x5f\x3e\xae\x15\x51\x65\x24\x93", 1, + 0, 248 }, + { 96, 256, 128, 65, + "\x7f\x5b\x74\xc0\x7e\xd1\xb4\x0f\xd1\x43\x58\xfe\x2f\xf2\xa7\x40\xc1" + "\x16\xc7\x70\x65\x10\xe6\xa4\x37\xf1\x9e\xa4\x99\x11\xce\xc4", + "\x47\x0a\x33\x9e\xcb\x32\x19\xb8\xb8\x1a\x1f\x8b", + "\x37\x46\x18\xa0\x6e\xa9\x8a\x48", + "\xf4\x52\x06\xab\xc2\x55\x52\xb2\xab\xc9\xab\x7f\xa2\x43\x03\x5f\xed" + "\xaa\xdd\xc3\xb2\x29\x39\x56\xf1\xea\x6e\x71\x56\xe7\xeb", + "\x46\xa8\x0c\x41\x87\x02\x47\x20\x08\x46\x27\x58\x00\x80\xdd\xe5\xa3" + "\xf4\xa1\x10\x93\xa7\x07\x6e\xd6\xf3\xd3\x26\xbc\x7b\x70", + "\x53\x4d\x4a\xa2\x83\x5a\x52\xe7\x2d\x14\xdf\x0e\x4f\x47\xf2\x5f", 1, + 64, 248 }, + { 96, 256, 128, 66, + "\xe1\x73\x1d\x58\x54\xe1\xb7\x0c\xb3\xff\xe8\xb7\x86\xa2\xb3\xeb\xf0" + "\x99\x43\x70\x95\x47\x57\xb9\xdc\x8c\x7b\xc5\x35\x46\x34\xa3", + "\x72\xcf\xd9\x0e\xf3\x02\x6c\xa2\x2b\x7e\x6e\x6a", "", + "\xb9\xc5\x54\xcb\xc3\x6a\xc1\x8a\xe8\x97\xdf\x7b\xee\xca\xc1\xdb\xeb" + "\x4e\xaf\xa1\x56\xbb\x60\xce\x2e\x5d\x48\xf0\x57\x15\xe6\x78", + "\xea\x29\xaf\xa4\x9d\x36\xe8\x76\x0f\x5f\xe1\x97\x23\xb9\x81\x1e\xd5" + "\xd5\x19\x93\x4a\x44\x0f\x50\x81\xac\x43\x0b\x95\x3b\x0e\x21", + "\x22\x25\x41\xaf\x46\xb8\x65\x33\xc6\xb6\x8d\x2f\xf1\x08\xa7\xea", 1, + 0, 256 }, + { 96, 256, 128, 67, + "\x27\xd8\x60\x63\x1b\x04\x85\xa4\x10\x70\x2f\xea\x61\xbc\x87\x3f\x34" + "\x42\x26\x0c\xad\xed\x4a\xbd\xe2\x5b\x78\x6a\x2d\x97\xf1\x45", + "\x26\x28\x80\xd4\x75\xf3\xda\xc5\x34\x0d\xd1\xb8", + "\x23\x33\xe5\xce\x0f\x93\xb0\x59", + "\x6b\x26\x04\x99\x6c\xd3\x0c\x14\xa1\x3a\x52\x57\xed\x6c\xff\xd3\xbc" + "\x5e\x29\xd6\xb9\x7e\xb1\x79\x9e\xb3\x35\xe2\x81\xea\x45\x1e", + "\x6d\xad\x63\x78\x97\x54\x4d\x8b\xf6\xbe\x95\x07\xed\x4d\x1b\xb2\xe9" + "\x54\xbc\x42\x7e\x5d\xe7\x29\xda\xf5\x07\x62\x84\x6f\xf2\xf4", + "\x7b\x99\x7d\x93\xc9\x82\x18\x9d\x70\x95\xdc\x79\x4c\x74\x62\x32", 1, + 64, 256 }, + { 96, 256, 128, 68, + "\x51\x55\xde\xe9\xaa\xde\x1c\xc6\x1e\xe7\xe3\xf9\x26\x60\xf7\x59\x0f" + "\x5e\x5b\xa8\x2f\x1b\x59\xb8\x50\xe3\xfa\x45\x3d\x2f\xa6\xb3", + "\xc2\x6c\x4b\x3b\xfd\xb9\x7e\xe6\xb0\xf6\x3c\xa1", "", + "\x27\x34\xe0\x8e\xff\x8f\x5c\x4f\x84\xfa\x0c\x20\x7f\x49\xc7\xfd\x78" + "\xaf\x1a\xd5\x12\x3f\xf8\x1f\x83\xf5\x00\xed\xf4\xed\xa0\x9e\xdf", + "\xf5\x98\x2b\x60\x1c\x7a\x18\xfc\x72\xa6\x5b\x21\x8c\x44\x97\x4d\xc5" + "\x64\xd8\x31\x4c\xbe\x6f\x87\xfc\xf6\xc6\xcf\xbe\x61\x8b\x34\xb1", + "\xc4\x36\x32\xf5\x57\x60\xb5\xd1\xed\x37\x55\x6a\x94\xd0\x49\xb5", 1, + 0, 264 }, + { 96, 256, 128, 69, + "\x57\x3f\x08\xeb\xbe\x0c\xce\x4a\xc9\x61\x8e\x8c\x3b\x22\x4b\xea\x0a" + "\x32\xf0\x55\xc6\x99\x68\x38\xa3\x2f\x52\x7c\xa3\xc3\xb6\x95", + "\xad\x80\x50\xdc\x6d\x12\x2d\xce\x3e\x56\x39\xed", + "\xe9\x96\x98\x24\x1c\x59\x9b\x5f", + "\x66\x8d\x5e\x3f\x95\xfe\x03\x0d\xaf\x43\x2a\x5f\xc5\x83\x7a\xf3\xa7" + "\x9c\x81\xe9\x4b\x28\xd8\x20\x4c\x5e\xe2\x62\xab\x3c\x99\x08\xa7", + "\xea\xf6\x81\x0e\x6e\xc1\xcb\x7a\x29\x18\x85\x62\x57\xd1\xaa\x3d\x51" + "\xa8\x27\x87\x91\x46\xc6\x33\x7e\xcf\x53\x5e\x9c\x89\xb1\x49\xc5", + "\xa2\x95\x0c\x2f\x39\x4a\x34\x66\xc3\x45\xf7\x96\x32\x3c\x1a\xa7", 1, + 64, 264 }, + { 96, 256, 128, 70, + "\xcf\x0d\x40\xa4\x64\x4e\x5f\x51\x81\x51\x65\xd5\x30\x1b\x22\x63\x1f" + "\x45\x44\xc4\x9a\x18\x78\xe3\xa0\xa5\xe8\xe1\xaa\xe0\xf2\x64", + "\xe7\x4a\x51\x5e\x7e\x21\x02\xb9\x0b\xef\x55\xd2", "", + "\x97\x3d\x0c\x75\x38\x26\xba\xe4\x66\xcf\x9a\xbb\x34\x93\x15\x2e\x9d" + "\xe7\x81\x9e\x2b\xd0\xc7\x11\x71\x34\x6b\x4d\x2c\xeb\xf8\x04\x1a\xa3" + "\xce\xdc\x0d\xfd\x7b\x46\x7e\x26\x22\x8b\xc8\x6c\x9a", + "\xfb\xa7\x8a\xe4\xf9\xd8\x08\xa6\x2e\x3d\xa4\x0b\xe2\xcb\x77\x00\xc3" + "\x61\x3d\x9e\xb2\xc5\x29\xc6\x52\xe7\x6a\x43\x2c\x65\x8d\x27\x09\x5f" + "\x0e\xb8\xf9\x40\xc3\x24\x98\x1e\xa9\x35\xe5\x07\xf9", + "\x8f\x04\x69\x56\xdb\x3a\x51\x29\x08\xbd\x7a\xfc\x8f\x2a\xb0\xa9", 1, + 0, 376 }, + { 96, 256, 128, 71, + "\x6c\xbf\xd7\x1c\x64\x5d\x18\x4c\xf5\xd2\x3c\x40\x2b\xdb\x0d\x25\xec" + "\x54\x89\x8c\x8a\x02\x73\xd4\x2e\xb5\xbe\x10\x9f\xdc\xb2\xac", + "\xd4\xd8\x07\x34\x16\x83\x82\x5b\x31\xcd\x4d\x95", + "\xb3\xe4\x06\x46\x83\xb0\x2d\x84", + "\xa9\x89\x95\x50\x4d\xf1\x6f\x74\x8b\xfb\x77\x85\xff\x91\xee\xb3\xb6" + "\x60\xea\x9e\xd3\x45\x0c\x3d\x5e\x7b\x0e\x79\xef\x65\x36\x59\xa9\x97" + "\x8d\x75\x54\x2e\xf9\x1c\x45\x67\x62\x21\x56\x40\xb9", + "\xa1\xff\xed\x80\x76\x18\x29\xec\xce\x24\x2e\x0e\x88\xb1\x38\x04\x90" + "\x16\xbc\xa0\x18\xda\x2b\x6e\x19\x98\x6b\x3e\x31\x8c\xae\x8d\x80\x61" + "\x98\xfb\x4c\x52\x7c\xc3\x93\x50\xeb\xdd\xea\xc5\x73", + "\xc4\xcb\xf0\xbe\xfd\xa0\xb7\x02\x42\xc6\x40\xd7\xcd\x02\xd7\xa3", 1, + 64, 376 }, + { 96, 256, 128, 72, + "\x5b\x1d\x10\x35\xc0\xb1\x7e\xe0\xb0\x44\x47\x67\xf8\x0a\x25\xb8\xc1" + "\xb7\x41\xf4\xb5\x0a\x4d\x30\x52\x22\x6b\xaa\x1c\x6f\xb7\x01", + "\xd6\x10\x40\xa3\x13\xed\x49\x28\x23\xcc\x06\x5b", "", + "\xd0\x96\x80\x31\x81\xbe\xef\x9e\x00\x8f\xf8\x5d\x5d\xdc\x38\xdd\xac" + "\xf0\xf0\x9e\xe5\xf7\xe0\x7f\x1e\x40\x79\xcb\x64\xd0\xdc\x8f\x5e\x67" + "\x11\xcd\x49\x21\xa7\x88\x7d\xe7\x6e\x26\x78\xfd\xc6\x76\x18\xf1\x18" + "\x55\x86\xbf\xea\x9d\x4c\x68\x5d\x50\xe4\xbb\x9a\x82", + "\x9a\x4e\xf2\x2b\x18\x16\x77\xb5\x75\x5c\x08\xf7\x47\xc0\xf8\xd8\xe8" + "\xd4\xc1\x8a\x9c\xc2\x40\x5c\x12\xbb\x51\xbb\x18\x72\xc8\xe8\xb8\x77" + "\x67\x8b\xec\x44\x2c\xfc\xbb\x0f\xf4\x64\xa6\x4b\x74\x33\x2c\xf0\x72" + "\x89\x8c\x7e\x0e\xdd\xf6\x23\x2e\xa6\xe2\x7e\xfe\x50", + "\x9f\xf3\x42\x7a\x0f\x32\xfa\x56\x6d\x9c\xa0\xa7\x8a\xef\xc0\x13", 1, + 0, 512 }, + { 96, 256, 128, 73, + "\x97\xd6\x35\xc4\xf4\x75\x74\xd9\x99\x8a\x90\x87\x5d\xa1\xd3\xa2\x84" + "\xb7\x55\xb2\xd3\x92\x97\xa5\x72\x52\x35\x19\x0e\x10\xa9\x7e", + "\xd3\x1c\x21\xab\xa1\x75\xb7\x0d\xe4\xeb\xb1\x9c", + "\x71\x93\xf6\x23\x66\x33\x21\xa2", + "\x94\xee\x16\x6d\x6d\x6e\xcf\x88\x32\x43\x71\x36\xb4\xae\x80\x5d\x42" + "\x88\x64\x35\x95\x86\xd9\x19\x3a\x25\x01\x62\x93\xed\xba\x44\x3c\x58" + "\xe0\x7e\x7b\x71\x95\xec\x5b\xd8\x45\x82\xa9\xd5\x6c\x8d\x4a\x10\x8c" + "\x7d\x7c\xe3\x4e\x6c\x6f\x8e\xa1\xbe\xc0\x56\x73\x17", + "\x5f\xbb\xde\xcc\x34\xbe\x20\x16\x14\xf6\x36\x03\x1e\xeb\x42\xf1\xca" + "\xce\x3c\x79\xa1\x2c\xff\xd8\x71\xee\x8e\x73\x82\x0c\x82\x97\x49\xf1" + "\xab\xb4\x29\x43\x67\x84\x9f\xb6\xc2\xaa\x56\xbd\xa8\xa3\x07\x8f\x72" + "\x3d\x7c\x1c\x85\x20\x24\xb0\x17\xb5\x89\x73\xfb\x1e", + "\x09\x26\x3d\xa7\xb4\xcb\x92\x14\x52\xf9\x7d\xca\x40\xf5\x80\xec", 1, + 64, 512 }, + { 96, 256, 128, 74, + "\xfe\x6e\x55\xbd\xae\xd1\xf7\x28\x4c\xa5\xfc\x0f\x8c\x5f\x2b\x8d\xf5" + "\x6d\xc0\xf4\x9e\x8c\xa6\x6a\x41\x99\x5e\x78\x33\x51\xf9\x01", + "\x17\xc8\x6a\x8a\xbb\xb7\xe0\x03\xac\xde\x27\x99", "", + "\xb4\x29\xeb\x80\xfb\x8f\xe8\xba\xed\xa0\xc8\x5b\x9c\x33\x34\x58\xe7" + "\xc2\x99\x2e\x55\x84\x75\x06\x9d\x12\xd4\x5c\x22\x21\x75\x64\x12\x15" + "\x88\x03\x22\x97\xef\xf5\x67\x83\x74\x2a\x5f\xc2\x2d\x74\x10\xff\xb2" + "\x9d\x66\x09\x86\x61\xd7\x6f\x12\x6c\x3c\x27\x68\x9e\x43\xb3\x72\x67" + "\xca\xc5\xa3\xa6\xd3\xab\x49\xe3\x91\xda\x29\xcd\x30\x54\xa5\x69\x2e" + "\x28\x07\xe4\xc3\xea\x46\xc8\x76\x1d\x50\xf5\x92", + "\xd0\x10\x2f\x6c\x25\x8b\xf4\x97\x42\xce\xc3\x4c\xf2\xd0\xfe\xdf\x23" + "\xd1\x05\xfb\x4c\x84\xcf\x98\x51\x5e\x1b\xc9\xa6\x4f\x8a\xd5\xbe\x8f" + "\x07\x21\xbd\xe5\x06\x45\xd0\x00\x83\xc3\xa2\x63\xa3\x10\x53\xb7\x60" + "\x24\x5f\x52\xae\x28\x66\xa5\xec\x83\xb1\x9f\x61\xbe\x1d\x30\xd5\xc5" + "\xd9\xfe\xcc\x4c\xbb\xe0\x8f\xd3\x85\x81\x3a\x2a\xa3\x9a\x00\xff\x9c" + "\x10\xf7\xf2\x37\x02\xad\xd1\xe4\xb2\xff\xa3\x1c", + "\x41\x86\x5f\xc7\x1d\xe1\x2b\x19\x61\x21\x27\xce\x49\x99\x3b\xb0", 1, + 0, 776 }, + { 96, 256, 128, 75, + "\xaa\xbc\x06\x34\x74\xe6\x5c\x4c\x3e\x9b\xdc\x48\x0d\xea\x97\xb4\x51" + "\x10\xc8\x61\x88\x46\xff\x6b\x15\xbd\xd2\xa4\xa5\x68\x2c\x4e", + "\x46\x36\x2f\x45\xd6\x37\x9e\x63\xe5\x22\x94\x60", + "\xa1\x1c\x40\xb6\x03\x76\x73\x30", + "\xce\xb5\x34\xce\x50\xdc\x23\xff\x63\x8a\xce\x3e\xf6\x3a\xb2\xcc\x29" + "\x73\xee\xad\xa8\x07\x85\xfc\x16\x5d\x06\xc2\xf5\x10\x0f\xf5\xe8\xab" + "\x28\x82\xc4\x75\xaf\xcd\x05\xcc\xd4\x9f\x2e\x7d\x8f\x55\xef\x3a\x72" + "\xe3\xdc\x51\xd6\x85\x2b\x8e\x6b\x9e\x7a\xec\xe5\x7b\xe6\x55\x6b\x0b" + "\x6d\x94\x13\xe3\x3f\xc5\xfc\x24\xa9\xa2\x05\xad\x59\x57\x4b\xb3\x9d" + "\x94\x4a\x92\xdc\x47\x97\x0d\x84\xa6\xad\x31\x76", + "\x75\x45\x39\x1b\x51\xde\x01\xd5\xc5\x3d\xfa\xca\x77\x79\x09\x06\x3e" + "\x58\xed\xee\x4b\xb1\x22\x7e\x71\x10\xac\x4d\x26\x20\xc2\xae\xc2\xf8" + "\x48\xf5\x6d\xee\xb0\x37\xa8\xdc\xed\x75\xaf\xa8\xa6\xc8\x90\xe2\xde" + "\xe4\x2f\x95\x0b\xb3\x3d\x9e\x24\x24\xd0\x8a\x50\x5d\x89\x95\x63\x97" + "\x3e\xd3\x88\x70\xf3\xde\x6e\xe2\xad\xc7\xfe\x07\x2c\x36\x6c\x14\xe2" + "\xcf\x7c\xa6\x2f\xb3\xd3\x6b\xee\x11\x68\x54\x61", + "\xb7\x0d\x44\xef\x8c\x66\xc5\xc7\xbb\xf1\x0d\xca\xdd\x7f\xac\xf6", 1, + 64, 776 }, + { 96, 256, 128, 76, + "\xd7\xad\xdd\x38\x89\xfa\xdf\x8c\x89\x3e\xee\x14\xba\x2b\x7e\xa5\xbf" + "\x56\xb4\x49\x90\x48\x69\x61\x5b\xd0\x5d\x5f\x11\x4c\xf3\x77", + "\x8a\x3a\xd2\x6b\x28\xcd\x13\xba\x65\x04\xe2\x60", "", + "\xc8\x77\xa7\x6b\xf5\x95\x56\x07\x72\x16\x7c\x6e\x3b\xcc\x70\x53\x05" + "\xdb\x9c\x6f\xcb\xeb\x90\xf4\xfe\xa8\x51\x16\x03\x8b\xc5\x3c\x3f\xa5" + "\xb4\xb4\xea\x0d\xe5\xcc\x53\x4f\xbe\x1c\xf9\xae\x44\x82\x4c\x6c\x2c" + "\x0a\x5c\x88\x5b\xd8\xc3\xcd\xc9\x06\xf1\x26\x75\x73\x7e\x43\x4b\x98" + "\x3e\x1e\x23\x1a\x52\xa2\x75\xdb\x5f\xb1\xa0\xca\xc6\xa0\x7b\x3b\x7d" + "\xcb\x19\x48\x2a\x5d\x3b\x06\xa9\x31\x7a\x54\x82\x6c\xea\x6b\x36\xfc" + "\xe4\x52\xfa\x9b\x54\x75\xe2\xaa\xf2\x54\x99\x49\x9d\x8a\x89\x32\xa1" + "\x9e\xb9\x87\xc9\x03\xbd\x85\x02\xfe", + "\x29\x4a\x76\x4c\x03\x35\x3f\x5f\x4f\x6e\x93\xcd\x7e\x97\x74\x80\xd6" + "\xc3\x43\x07\x1d\xb0\xb7\xc1\xf0\xdb\x1e\x95\xb8\x5e\x60\x53\xf0\x42" + "\x31\x68\xa9\xc7\x53\x32\x68\xdb\x9a\x19\x4e\x76\x65\x35\x9d\x14\x48" + "\x9b\xc4\x71\x72\xa9\xf2\x13\x70\xe8\x9b\x0b\xd0\xe5\xef\x96\x61\x73" + "\x8d\xe2\x82\x57\x2b\xcc\x3e\x54\x12\x47\x62\x6e\x57\xe7\x5d\xec\x0f" + "\x91\xac\x5c\x53\x0b\xd1\xa5\x32\x71\x84\x29\x96\xdc\xd0\x4d\x86\x53" + "\x21\xb1\xec\xb6\xe7\x63\x01\x14\xfe\x78\x02\x91\xb8\xdc\x3e\x5d\x0a" + "\xbc\x8e\x65\xb1\xc5\x49\x3e\x9a\xf0", + "\xf2\xb9\x74\xca\x0f\x14\xfb\x9f\x92\x01\x4b\xff\x18\x57\x3c\xff", 1, + 0, 1024 }, + { 96, 256, 128, 77, + "\x80\xbe\x86\xfb\x6f\xc4\x9b\xc7\x34\x28\xca\xb5\x76\xf6\xad\x72\xff" + "\x6a\xca\x04\x00\x1b\x8b\x1c\x57\xa7\x12\x8b\xe7\x39\x00\xaa", + "\x90\x31\x88\x43\x3c\x1c\xe8\x97\x1a\xa1\x9b\x9d", + "\x05\x87\xaf\x85\x30\xad\x05\x47", + "\x67\xce\x49\x9c\xd8\xed\x68\xbd\x71\x7d\xfe\x61\xc6\x0f\x27\xd2\x60" + "\xb1\xc1\x63\xa7\x2e\x8c\xc8\x59\x72\x53\xd3\xd9\x87\xc2\xdb\xe1\xbf" + "\xf2\xe4\x4d\x9b\xd4\x76\x5d\x3e\x53\xd9\xc3\xf8\xeb\x3b\x90\xe7\x51" + "\xf4\x7c\x71\x57\xbd\xc1\x14\x2b\xc3\x3f\x58\x33\xac\x1c\xd1\x26\x2c" + "\xbb\x23\x90\x66\xb3\x34\xa4\xed\x99\xae\x82\xc7\x4f\x2b\x49\x54\x0f" + "\x1a\x61\x4b\xc2\x39\xd8\xfc\x5a\xdd\x8c\x17\x81\x84\xe4\x12\x81\xf6" + "\xe6\x6c\x5c\x31\x17\xfd\x95\x35\x47\xf7\xc8\x29\x42\x5b\x50\x82\xaa" + "\x69\x68\x68\x47\xea\xf5\x78\x46\x92", + "\x2b\x90\xb4\xf3\xde\x28\x0c\x44\x91\x3d\x19\x84\xbd\xd5\xdf\xa0\x56" + "\x6c\x6a\x14\xa0\x58\x65\x9a\x9b\x62\x32\x77\xb0\xbb\x6e\x82\x10\x1e" + "\x79\x39\x5d\x12\xe6\x43\xf6\x2d\x9a\x82\x2b\xae\x49\x79\x07\x49\x3e" + "\x4f\x82\x13\xfc\xf9\x9d\xa8\xa7\x8f\xdf\x86\x7a\xf3\x6b\xc8\xb0\x93" + "\x1c\x18\x86\xb4\xf0\xae\x57\x29\x98\x64\x94\xdb\xd5\x97\x37\xe9\x56" + "\xcd\x8f\x22\x6c\x7c\x52\x26\x89\xd0\x82\xf0\x23\x89\x4d\x54\xac\xab" + "\x0c\x4d\x60\x9f\x37\x46\xa6\x73\x69\xbb\x88\x76\x00\x8f\x7f\xd3\xdc" + "\x66\x81\xc5\xfb\x9d\x72\x8c\x59\x11", + "\xf0\x05\xeb\xe1\xc1\xad\xa7\x5a\x9c\xee\x8d\x63\x08\x81\xd5\xb8", 1, + 64, 1024 }, + { 96, 256, 128, 78, + "\x7d\x00\xb4\x80\x95\xad\xfa\x32\x72\x05\x06\x07\xb2\x64\x18\x50\x02" + "\xba\x99\x95\x7c\x49\x8b\xe0\x22\x77\x0f\x2c\xe2\xf3\x14\x3c", + "\x87\x34\x5f\x10\x55\xfd\x9e\x21\x02\xd5\x06\x56", "\x02", + "\xe5\xcc\xaa\x44\x1b\xc8\x14\x68\x8f\x8f\x6e\x8f\x28\xb5\x00\xb2", + "\x7e\x72\xf5\xa1\x85\xaf\x16\xa6\x11\x92\x1b\x43\x8f\x74\x9f\x0b", + "\x12\x42\xc6\x70\x73\x23\x34\x02\x9a\xdf\xe1\xc5\x00\x16\x51\xe4", 1, + 8, 128 }, + { 96, 256, 128, 79, + "\x64\x32\x71\x7f\x1d\xb8\x5e\x41\xac\x78\x36\xbc\xe2\x51\x85\xa0\x80" + "\xd5\x76\x2b\x9e\x2b\x18\x44\x4b\x6e\xc7\x2c\x3b\xd8\xe4\xdc", + "\x87\xa3\x16\x3e\xc0\x59\x8a\xd9\x5b\x3a\xa7\x13", "\xb6\x48", + "\x02\xcd\xe1\x68\xfb\xa3\xf5\x44\xbb\xd0\x33\x2f\x7a\xde\xad\xa8", + "\x85\xf2\x9a\x71\x95\x57\xcd\xd1\x4d\x1f\x8f\xff\xab\x6d\x9e\x60", + "\x73\x2c\xa3\x2b\xec\xd5\x15\xa1\xed\x35\x3f\x54\x2e\x99\x98\x58", 1, + 16, 128 }, + { 96, 256, 128, 80, + "\x7a\xfa\x0f\x59\xdf\xcb\x5a\xd3\xa7\x64\x90\xc5\xc8\x04\x32\x7c\x8d" + "\x05\x2b\xe7\x37\xa6\x0f\xa8\xbc\xbf\x0a\x2c\x36\x63\x0a\x43", + "\x25\xb7\xbd\xf4\xa6\xdc\xbf\x7c\x9a\x3e\xc2\xb3", "\x8b\x71\xac", + "\x62\x3e\x6b\xa6\xd3\x16\x6a\x33\x8b\xfc\xc7\xaf\x90\xa2\x30\xc8", + "\xd4\x6e\x82\x65\xa8\xc6\xa2\x53\x93\xdd\x95\x6b\xb4\x43\x97\xad", + "\xe2\x8f\x3a\xd9\xe3\xef\x4a\x3d\x94\xee\x07\xbf\x53\x8e\xaa\xfb", 1, + 24, 128 }, + { 96, 256, 128, 81, + "\x2e\xc2\x5b\x0e\xc7\xac\x24\x42\x24\xe9\xc7\xfc\x2f\xa5\xd3\xef\x17" + "\x80\x9e\x19\xfd\x6e\x95\x41\x58\xdd\x0d\x72\x73\x8a\x4c\xc8", + "\x6f\xb0\xd1\x41\x7c\xdf\xff\x4d\xf3\x7d\xb0\x8c", + "\x3a\x5d\xdf\x40", + "\xa1\xc9\x33\x76\x8a\x6d\x57\x3e\xbf\x68\xa9\x9e\x5e\x18\xda\xe8", + "\x2d\x3c\xb2\xd9\x30\x34\x91\xe2\x64\xf2\x90\x4f\x0e\x07\x53\xf4", + "\x6c\x1d\xb9\x59\x36\x2d\x21\x7b\x23\x22\xb4\x66\x53\x6b\xfe\xa0", 1, + 32, 128 }, + { 96, 256, 128, 82, + "\x0a\x2c\xf5\x23\x71\xcf\x9d\x9f\x95\xb1\x01\x08\xfc\x82\xb4\xfd\x61" + "\x10\xa8\xba\x9a\x88\xa2\x60\x83\x68\x5a\xd2\x98\x26\x89\x1a", + "\x25\x38\xfc\x67\xaf\xb9\xea\xb3\x33\xf8\x32\x90", + "\x9e\xec\x54\x0b\xb0", + "\x0d\x8c\x69\x1d\x04\x4a\x39\x78\xd7\x90\x43\x2d\xc7\x1d\x69\xf8", + "\xa9\x88\xc0\x3c\x71\xb9\x56\xff\x08\x6d\x04\x70\xd7\x06\xbd\x34", + "\xb3\x5d\x7c\xbf\x2b\xeb\x89\x4b\x0c\x74\x6e\x07\x30\x42\x9e\x15", 1, + 40, 128 }, + { 96, 256, 128, 83, + "\x30\x7e\x88\x6b\x38\xbb\x18\xb4\x45\xf8\xa2\xc6\xd6\xf8\x93\x24\x92" + "\xa9\xce\xa8\xd0\x41\xba\x72\xeb\x5e\xfd\xfa\x70\xd0\xb8\xd2", + "\xa0\x71\xbe\x99\x91\x51\xe2\xa1\xc4\x1c\x81\xe9", + "\x56\xe0\x14\xd9\x7c\x74", + "\x9a\xba\x22\xb4\x95\xcb\x7e\xc8\x87\xdd\xaa\x62\x01\x9a\xa1\x4d", + "\x32\xbf\x95\xd4\xc1\x95\xdb\xaf\x58\xd9\xaf\x40\x01\xc6\xe5\x7d", + "\x43\x93\x80\x87\x03\xd6\x7a\x90\x87\x05\x78\x04\x6c\xd8\xb5\x25", 1, + 48, 128 }, + { 96, 256, 128, 84, + "\xda\xcd\x51\xa8\xa8\xe4\xd5\x90\x5b\x4c\xbb\x94\x7e\xf4\x01\x3e\xb2" + "\x96\x88\x93\x53\xf3\xc9\xee\x35\xf5\x57\x7b\x26\x73\x7a\x51", + "\x3f\xa3\x78\xa1\xbe\xfd\xdd\xd6\x1a\xe6\x8c\xf4", + "\xbb\x5a\x38\x12\xf0\xae\xfd", + "\xe1\x48\x31\x38\x83\xa7\x7d\xa1\x21\x12\x4d\x06\xb1\xc7\x7d\xca", + "\x2a\x20\x7c\xa7\xe9\xda\x6b\x13\xa2\x29\x60\x43\x04\xd8\x7e\xb1", + "\x8a\x6b\x6a\xfe\xc8\x7d\x93\xec\x6e\x8d\xbe\x13\xd8\x4c\x0f\x8c", 1, + 56, 128 }, + { 96, 256, 128, 85, + "\x7b\x5f\xbb\xb2\x02\xc1\x61\x08\xfd\x13\x06\x64\x46\x85\x3a\x85\x0d" + "\x8b\x34\xe9\xda\x40\x51\x95\x80\xda\x44\x6a\x92\x2f\x91\x62", + "\xaa\x07\x7a\x5c\xe9\x16\x1b\xde\x8d\x8e\xdc\x40", + "\xf9\x4b\xb9\x2c\x1c\x66\x8a\x69\x5b", + "\xda\x47\x1c\xd6\x93\x5a\x0c\xa8\x30\x7d\xde\xdc\x6b\x95\x99\x62", + "\x54\x8a\x5c\xa0\xae\x49\x21\x1c\xdf\x30\xbb\xdc\xb1\x35\x2d\x31", + "\x20\x4d\xac\xb9\x8f\x8c\x89\x08\xcc\x5e\xa2\x2b\xb2\x3f\x90\x1f", 1, + 72, 128 }, + { 96, 256, 128, 86, + "\x1f\xfd\x10\x1e\xb9\x75\x31\xf6\xfa\xa8\x21\xec\x4d\x5c\x57\x02\x72" + "\x5d\xd0\x33\xd3\xb8\x30\xbb\x76\x0c\x4e\xf2\x7b\xa9\x83\xdf", + "\x59\x81\x14\xe8\xcf\x7f\xbd\xea\x8a\xd2\x96\x83", + "\x21\x55\x62\x7e\xc1\x5a\x97\x8f\xbc\xb2", + "\x28\x66\x8c\xa8\xdb\x53\x5c\x7e\x8e\xb2\x74\x91\xad\x0f\xb7\xcb", + "\x28\xce\xda\xc2\x4f\x14\xca\xa3\x26\xc7\xfe\x40\x1f\x68\xa8\x7c", + "\x2b\xf1\xb2\xc4\x3d\x30\x39\xf8\xf5\xce\x35\x9c\x11\x02\xf8\x79", 1, + 80, 128 }, + { 96, 256, 128, 87, + "\xd2\xd0\xa9\x73\xd5\x95\x1a\xf3\x52\xcb\xee\x57\xac\x9d\xab\x1c\x28" + "\x4c\x99\xaf\x3b\x99\x2c\xe0\x15\xf2\x19\x50\x6f\x64\x88\x8d", + "\x9a\xcd\x21\x35\x70\xce\x9b\xb9\xd8\x86\xc6\xef", + "\x37\xad\x66\x8d\x4d\x4f\xe8\x89\x94\x97\x63", + "\x3f\x3f\x00\x76\x25\x03\x52\xe1\xb6\xb5\xc1\x2c\xfa\x12\x62\x5e", + "\x72\x56\xe8\x56\x87\x2a\xd3\xa5\x4b\x34\xa2\xa6\xbd\xca\x88\x38", + "\x3b\x12\xe4\x58\x6e\x45\x22\x3f\x78\xa6\xee\xa8\x11\xef\xb8\x63", 1, + 88, 128 }, + { 96, 256, 128, 88, + "\xad\xcc\x52\x0b\x38\x13\x82\x23\x7d\x05\xa6\x40\x0a\x7d\xfb\xcd\x07" + "\x71\xb6\xaa\x9e\xdb\x79\x66\x13\x1d\xde\xf6\xaf\x21\xf1\xbe", + "\x91\x83\xcd\xf3\xa8\xba\x73\x97\xb6\xb2\xd5\xd5", + "\xb3\x34\x37\x54\x15\xf6\x21\x5c\x0b\xf8\x9a\x9a", + "\x95\x82\x95\x61\x9c\xf1\xb3\x6f\x0b\x47\x46\x63\xc0\xbc\x79\xeb", + "\x85\x2c\x14\x1b\x42\x39\xa3\x1f\xee\xda\x03\x55\x0d\x70\xa2\xbe", + "\x5f\xc5\x92\x87\xb9\x2d\x3f\xcf\x7d\x66\xf1\x3d\xef\xb1\x1b\x0d", 1, + 96, 128 }, + { 96, 256, 128, 89, + "\xbd\x53\x4f\x7a\xde\xca\x46\x68\x44\xfb\x3b\xa3\x46\x58\xbe\x80\x7f" + "\x15\xc5\x29\x1e\xd6\x02\x68\x60\xa2\x4f\x17\x9b\x71\x2c\x89", + "\x41\x2c\x3e\x13\xee\x1f\x78\x64\xbd\x15\xce\x39", + "\x28\x66\xaf\xff\x0b\xcc\x61\x35\xdc\x63\xaf\x88\xc8", + "\xd9\x2f\x8c\xe5\xd8\xd0\xad\x2e\xb5\xf1\x1a\xf0\x2e\xf6\x39\x49", + "\x89\xd6\xd0\x89\xc4\xa2\x55\x95\x2a\xca\x11\xb2\x4a\x01\xff\x95", + "\xf8\x8f\xa4\x53\x12\x04\xda\x31\x5e\x73\x17\x97\x02\x40\xce\x9e", 1, + 104, 128 }, + { 96, 256, 128, 90, + "\x91\x0a\xde\x7d\x32\x4d\x2c\x96\x88\x43\x9e\x1f\x14\x2e\x0e\x5f\x9d" + "\x13\x0f\xf8\x32\xe5\x07\xfe\x19\x85\xe5\xa2\x64\x52\xa6\xd0", + "\x9b\xe0\x90\xdb\xa9\x3d\xef\xf2\x7a\xdf\x99\xee", + "\xea\x25\x75\xf1\x23\x26\x8e\x93\x6c\x8e\x4c\x8c\x1b\xb8", + "\x6e\x35\x60\x94\xed\x9d\x9a\x70\x53\xc7\x90\x6c\x48\xba\x3d\x9f", + "\x01\xff\xb3\x43\xc7\x57\xb2\x78\x43\xd8\xa9\x00\xa3\x6c\xe3\x9d", + "\xa3\x15\x54\x1b\x7d\x63\x13\xc6\xfd\xdf\x64\xb3\x03\xd7\x1d\x60", 1, + 112, 128 }, + { 96, 256, 128, 91, + "\x8e\x34\xcf\x73\xd2\x45\xa1\x08\x2a\x92\x0b\x86\x36\x4e\xb8\x96\xc4" + "\x94\x64\x67\xbc\xb3\xd5\x89\x29\xfc\xb3\x66\x90\xe6\x39\x4f", + "\x6f\x57\x3a\xa8\x6b\xaa\x49\x2b\xa4\x65\x96\xdf", + "\xbd\x4c\xd0\x2f\xc7\x50\x2b\xbd\xbd\xf6\xc9\xa3\xcb\xe8\xf0", + "\x16\xdd\xd2\x3f\xf5\x3f\x3d\x23\xc0\x63\x34\x48\x70\x40\xeb\x47", + "\xc1\xb2\x95\x93\x6d\x56\xfa\xda\xc0\x3e\x5f\x74\x2b\xff\x73\xa1", + "\x39\xc4\x57\xdb\xab\x66\x38\x2b\xab\xb3\xb5\x58\x00\xcd\xa5\xb8", 1, + 120, 128 }, + { 96, 256, 128, 92, + "\xcb\x55\x75\xf5\xc7\xc4\x5c\x91\xcf\x32\x0b\x13\x9f\xb5\x94\x23\x75" + "\x60\xd0\xa3\xe6\xf8\x65\xa6\x7d\x4f\x63\x3f\x2c\x08\xf0\x16", + "\x1a\x65\x18\xf0\x2e\xde\x1d\xa6\x80\x92\x66\xd9", + "\x89\xcc\xe9\xfb\x47\x44\x1d\x07\xe0\x24\x5a\x66\xfe\x8b\x77\x8b", + "\x62\x3b\x78\x50\xc3\x21\xe2\xcf\x0c\x6f\xbc\xc8\xdf\xd1\xaf\xf2", + "\xc8\x4c\x9b\xb7\xc6\x1c\x1b\xcb\x17\x77\x2a\x1c\x50\x0c\x50\x95", + "\xdb\xad\xf7\xa5\x13\x8c\xa0\x34\x59\xa2\xcd\x65\x83\x1e\x09\x2f", 1, + 128, 128 }, + { 96, 256, 128, 93, + "\xa5\x56\x9e\x72\x9a\x69\xb2\x4b\xa6\xe0\xff\x15\xc4\x62\x78\x97\x43" + "\x68\x24\xc9\x41\xe9\xd0\x0b\x2e\x93\xfd\xdc\x4b\xa7\x76\x57", + "\x56\x4d\xee\x49\xab\x00\xd2\x40\xfc\x10\x68\xc3", + "\xd1\x9f\x2d\x98\x90\x95\xf7\xab\x03\xa5\xfd\xe8\x44\x16\xe0\x0c" + "\x0e", + "\x87\xb3\xa4\xd7\xb2\x6d\x8d\x32\x03\xa0\xde\x1d\x64\xef\x82\xe3", + "\x94\xbc\x80\x62\x1e\xd1\xe7\x1b\x1f\xd2\xb5\xc3\xa1\x5e\x35\x68", + "\x33\x35\x11\x86\x17\x96\x97\x84\x01\x59\x8b\x96\x37\x22\xf5\xb3", 1, + 136, 128 }, + { 96, 256, 128, 94, + "\x56\x20\x74\x65\xb4\xe4\x8e\x6d\x04\x63\x0f\x4a\x42\xf3\x5c\xfc\x16" + "\x3a\xb2\x89\xc2\x2a\x2b\x47\x84\xf6\xf9\x29\x03\x30\xbe\xe0", + "\xdf\x87\x13\xe8\x7e\xc3\xdb\xcf\xad\x14\xd5\x3e", + "\x5e\x64\x70\xfa\xcd\x99\xc1\xd8\x1e\x37\xcd\x44\x01\x5f\xe1\x94\x80" + "\xa2\xa4\xd3\x35\x2a\x4f\xf5\x60\xc0\x64\x0f\xdb\xda", + "\xe6\x01\xb3\x85\x57\x79\x7d\xa2\xf8\xa4\x10\x6a\x08\x9d\x1d\xa6", + "\x29\x9b\x5d\x3f\x3d\x03\xc0\x87\x20\x9a\x16\xe2\x85\x14\x31\x11", + "\x4b\x45\x4e\xd1\x98\xde\x11\x7e\x83\xec\x49\xfa\x8d\x85\x08\xd6", 1, + 240, 128 }, + { 96, 256, 128, 95, + "\x07\x74\x33\x02\x2a\xb3\x4d\x38\x0f\xc1\x92\xfc\x24\xc2\xed\xc6\x30" + "\x1f\xec\x6f\x24\x44\x2f\x57\x2a\x10\x87\xff\x2e\x05\xb3\x9a", + "\x28\xad\xcb\xc7\x43\x64\xf2\x6d\xd4\xb3\x10\x8b", + "\xe0\x10\x0e\xb1\x16\xcd\xc5\xe2\x2a\x3b\x9f\x9b\x41\x26\xc1\x49\x59" + "\x5e\x75\x10\x7f\x6e\x23\x7c\x69\xe8\x29\x60\x05\x22\x70", + "\x03\xc8\x74\xee\xaa\xa6\xfa\x9f\x0d\xa6\x2c\x75\x8f\xb0\xad\x04", + "\x1e\x96\x87\xb3\x5f\xbc\x8e\xaa\x18\x25\xed\x38\x47\x79\x8f\x76", + "\x07\x88\xbf\x70\xfd\x04\x03\x0e\xcd\x1c\x96\xd0\xbc\x1f\xcd\x5d", 1, + 248, 128 }, + { 96, 256, 128, 96, + "\x39\x37\x98\x6a\xf8\x6d\xaf\xc1\xba\x0c\x46\x72\xd8\xab\xc4\x6c\x20" + "\x70\x62\x68\x2d\x9c\x26\x4a\xb0\x6d\x6c\x58\x07\x20\x51\x30", + "\x8d\xf4\xb1\x5a\x88\x8c\x33\x28\x6a\x7b\x76\x51", + "\xba\x44\x6f\x6f\x9a\x0c\xed\x22\x45\x0f\xeb\x10\x73\x7d\x90\x07\xfd" + "\x69\xab\xc1\x9b\x1d\x4d\x90\x49\xa5\x55\x1e\x86\xec\x2b\x37", + "\xdc\x9e\x9e\xaf\x11\xe3\x14\x18\x2d\xf6\xa4\xeb\xa1\x7a\xec\x9c", + "\x60\x5b\xbf\x90\xae\xb9\x74\xf6\x60\x2b\xc7\x78\x05\x6f\x0d\xca", + "\x38\xea\x23\xd9\x90\x54\xb4\x6b\x42\xff\xe0\x04\x12\x9d\x22\x04", 1, + 256, 128 }, + { 96, 256, 128, 97, + "\x36\x37\x2a\xbc\xdb\x78\xe0\x27\x96\x46\xac\x3d\x17\x6b\x96\x74\xe9" + "\x15\x4e\xec\xf0\xd5\x46\x9c\x65\x1e\xc7\xe1\x6b\x4c\x11\x99", + "\xbe\x40\xe5\xf1\xa1\x18\x17\xa0\xa8\xfa\x89\x49", + "\xd4\x1a\x82\x8d\x5e\x71\x82\x92\x47\x02\x19\x05\x40\x2e\xa2\x57\xdc" + "\xcb\xc3\xb8\x0f\xcd\x56\x75\x05\x6b\x68\xbb\x59\xe6\x2e\x88\x73", + "\x81\xce\x84\xed\xe9\xb3\x58\x59\xcc\x8c\x49\xa8\xf6\xbe\x7d\xc6", + "\x7b\x7c\xe0\xd8\x24\x80\x9a\x70\xde\x32\x56\x2c\xcf\x2c\x2b\xbd", + "\x15\xd4\x4a\x00\xce\x0d\x19\xb4\x23\x1f\x92\x1e\x22\xbc\x0a\x43", 1, + 264, 128 }, + { 96, 256, 128, 98, + "\x9f\x14\x79\xed\x09\x7d\x7f\xe5\x29\xc1\x1f\x2f\x5a\xdd\x9a\xaf\xf4" + "\xa1\xca\x0b\x68\x99\x7a\x2c\xb7\xf7\x97\x49\xbd\x90\xaa\xf4", + "\x84\xc8\x7d\xae\x4e\xee\x27\x73\x0e\xc3\x5d\x12", + "\x3f\x2d\xd4\x9b\xbf\x09\xd6\x9a\x78\xa3\xd8\x0e\xa2\x56\x66\x14\xfc" + "\x37\x94\x74\x19\x6c\x1a\xae\x84\x58\x3d\xa7\x3d\x7f\xf8\x5c\x6f\x42" + "\xca\x42\x05\x6a\x97\x92\xcc\x1b\x9f\xb3\xc7\xd2\x61", + "\xa6\x67\x47\xc8\x9e\x85\x7a\xf3\xa1\x8e\x2c\x79\x50\x00\x87\xed", + "\xca\x82\xbf\xf3\xe2\xf3\x10\xcc\xc9\x76\x67\x2c\x44\x15\xe6\x9b", + "\x57\x63\x8c\x62\xa5\xd8\x5d\xed\x77\x4f\x91\x3c\x81\x3e\xa0\x32", 1, + 376, 128 }, + { 96, 256, 128, 99, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x65\xb6\x3b\xf0\x74\xb7\x28\x39\x92\xe2\x4b\x1a\xc0\xdf\x0d\x22\xb5" + "\x55\xdb\xe2\x25\x4d\x94\xa4\x3f\x1d\xe7\x48\xd3\xcc\x6f\x0d", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x39\xf4\xfc\xe3\x02\x6d\x83\x78\x9f\xfd\x1e\xe6\xf2\xcd\x7c\x4f", 1, + 128, 256 }, + { 96, 256, 128, 100, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x65\xb6\x3b\xf0\x74\xb7\x28\x39\x92\xe2\x4b\x1a\xc0\xdf\x0d\x22\xb5" + "\x55\xdb\xe2\x25\x4d\x94\xa4\x3f\x1d\xe7\x48\xd3\xcc\x6f\x0d\x20\xc1" + "\x42\xfe\x89\x8f\xbb\xe6\x68\xd4\x32\x43\x94\x43\x4c\x1b\x18\xb5\x8e" + "\xad\x71\x0a\xed\x9c\x31\xdb\x1f\x2a\x8a\x1f\x1b\xb2", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\xf5\xea\xa8\x04\x60\x5c\x3a\x47\x85\xf9\xd7\xf1\x3b\x6f\x67\xd6", 1, + 128, 512 }, + { 96, 256, 128, 101, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x65\xb6\x3b\xf0\x74\xb7\x28\x39\x92\xe2\x4b\x1a\xc0\xdf\x0d\x22\xb5" + "\x55\xdb\xe2\x25\x4d\x94\xa4\x3f\x1d\xe7\x48\xd3\xcc\x6f\x0d\x20\xc1" + "\x42\xfe\x89\x8f\xbb\xe6\x68\xd4\x32\x43\x94\x43\x4c\x1b\x18\xb5\x8e" + "\xad\x71\x0a\xed\x9c\x31\xdb\x1f\x2a\x8a\x1f\x1b\xb2\x44\x05\xc1\x83" + "\xaf\x94\xee\x1a\xd6\x30\xcd\x93\x11\x58\xa6\x21\x3d\x48\xc8\xff\xf1" + "\x0d\x0a\x1f\x9e\xf7\x60\x18\x8e\x65\x88\x02\xaa\xd5\x5e\x41\xa1\xd9" + "\x90\x69\xa1\x8d\xb5\x5c\x56\xaf\x7c\x10\xa6\xf2\x1e\xcc\x8a\xf9\xb7" + "\xce\x0a\x7e\xa0\xb6\x74\x26\xe9\x25", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00", + "\x9b\x5c\x43\xa7\x8d\x95\x4e\x8a\x3c\x65\x9e\xeb\xc1\x3d\x5d\x55", 1, + 128, 1024 }, + { 96, 256, 128, 102, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x9a\x49\xc4\x0f\x8b\x48\xd7\xc6\x6d\x1d\xb4\xe5\x3f\x20\xf2\xdd\x4a" + "\xaa\x24\x1d\xda\xb2\x6b\x5b\xc0\xe2\x18\xb7\x2c\x33\x90\xf2", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x37\xe3\x39\x9d\x9c\xa6\x96\x79\x9f\x08\xf4\xf7\x2b\xc0\xcd\xd8", 1, + 128, 256 }, + { 96, 256, 128, 103, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x9a\x49\xc4\x0f\x8b\x48\xd7\xc6\x6d\x1d\xb4\xe5\x3f\x20\xf2\xdd\x4a" + "\xaa\x24\x1d\xda\xb2\x6b\x5b\xc0\xe2\x18\xb7\x2c\x33\x90\xf2\xdf\x3e" + "\xbd\x01\x76\x70\x44\x19\x97\x2b\xcd\xbc\x6b\xbc\xb3\xe4\xe7\x4a\x71" + "\x52\x8e\xf5\x12\x63\xce\x24\xe0\xd5\x75\xe0\xe4\x4d", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x3d\x52\x71\x0b\xec\x86\xd4\xea\x9f\xea\x2f\xf2\x69\x54\x91\x91", 1, + 128, 512 }, + { 96, 256, 128, 104, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x9a\x49\xc4\x0f\x8b\x48\xd7\xc6\x6d\x1d\xb4\xe5\x3f\x20\xf2\xdd\x4a" + "\xaa\x24\x1d\xda\xb2\x6b\x5b\xc0\xe2\x18\xb7\x2c\x33\x90\xf2\xdf\x3e" + "\xbd\x01\x76\x70\x44\x19\x97\x2b\xcd\xbc\x6b\xbc\xb3\xe4\xe7\x4a\x71" + "\x52\x8e\xf5\x12\x63\xce\x24\xe0\xd5\x75\xe0\xe4\x4d\xbb\xfa\x3e\x7c" + "\x50\x6b\x11\xe5\x29\xcf\x32\x6c\xee\xa7\x59\xde\xc2\xb7\x37\x00\x0e" + "\xf2\xf5\xe0\x61\x08\x9f\xe7\x71\x9a\x77\xfd\x55\x2a\xa1\xbe\x5e\x26" + "\x6f\x96\x5e\x72\x4a\xa3\xa9\x50\x83\xef\x59\x0d\xe1\x33\x75\x06\x48" + "\x31\xf5\x81\x5f\x49\x8b\xd9\x16\xda", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x51\x35\x63\x29\xe2\x80\xb1\x2d\x55\xd3\xd9\x8f\x0a\x58\x0c\xbe", 1, + 128, 1024 }, + { 96, 256, 128, 105, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\x65\xb6\x3b\x70\x74\xb7\x28\xb9\x92\xe2\x4b\x9a\xc0\xdf\x0d\xa2\xb5" + "\x55\xdb\x62\x25\x4d\x94\x24\x3f\x1d\xe7\xc8\xd3\xcc\x6f\x8d", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\xc1\x52\xa4\xb9\x0c\x54\x8c\x71\xdc\x47\x9e\xde\xaf\x92\x11\xbf", 1, + 128, 256 }, + { 96, 256, 128, 106, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\x65\xb6\x3b\x70\x74\xb7\x28\xb9\x92\xe2\x4b\x9a\xc0\xdf\x0d\xa2\xb5" + "\x55\xdb\x62\x25\x4d\x94\x24\x3f\x1d\xe7\xc8\xd3\xcc\x6f\x8d\x20\xc1" + "\x42\x7e\x89\x8f\xbb\x66\x68\xd4\x32\xc3\x94\x43\x4c\x9b\x18\xb5\x8e" + "\x2d\x71\x0a\xed\x1c\x31\xdb\x1f\xaa\x8a\x1f\x1b\x32", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00" + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\x40\xef\x63\x83\x05\x2d\x91\xc2\xe4\xb4\x61\x1b\x0e\x32\xc5\xff", 1, + 128, 512 }, + { 96, 256, 128, 107, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\x65\xb6\x3b\x70\x74\xb7\x28\xb9\x92\xe2\x4b\x9a\xc0\xdf\x0d\xa2\xb5" + "\x55\xdb\x62\x25\x4d\x94\x24\x3f\x1d\xe7\xc8\xd3\xcc\x6f\x8d\x20\xc1" + "\x42\x7e\x89\x8f\xbb\x66\x68\xd4\x32\xc3\x94\x43\x4c\x9b\x18\xb5\x8e" + "\x2d\x71\x0a\xed\x1c\x31\xdb\x1f\xaa\x8a\x1f\x1b\x32\x44\x05\xc1\x03" + "\xaf\x94\xee\x9a\xd6\x30\xcd\x13\x11\x58\xa6\xa1\x3d\x48\xc8\x7f\xf1" + "\x0d\x0a\x9f\x9e\xf7\x60\x98\x8e\x65\x88\x82\xaa\xd5\x5e\xc1\xa1\xd9" + "\x90\xe9\xa1\x8d\xb5\xdc\x56\xaf\x7c\x90\xa6\xf2\x1e\x4c\x8a\xf9\xb7" + "\x4e\x0a\x7e\xa0\x36\x74\x26\xe9\xa5", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00" + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80" + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00" + "\x80\x00\x00\x00\x80\x00\x00\x00\x80", + "\xae\x9b\x54\x25\x41\xe8\x4f\xc7\x45\x42\xee\xd6\xbe\x63\x8f\xee", 1, + 128, 1024 }, + { 96, 256, 128, 108, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\xe5\xb6\x3b\xf0\xf4\xb7\x28\x39\x12\xe2\x4b\x1a\x40\xdf\x0d\x22\x35" + "\x55\xdb\xe2\xa5\x4d\x94\xa4\xbf\x1d\xe7\x48\x53\xcc\x6f\x0d", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80" + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\x10\xfe\xe3\xec\xfb\xa9\xcd\xf7\x97\xba\xe3\x7a\x62\x6e\xc8\x3b", 1, + 128, 256 }, + { 96, 256, 128, 109, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\xe5\xb6\x3b\xf0\xf4\xb7\x28\x39\x12\xe2\x4b\x1a\x40\xdf\x0d\x22\x35" + "\x55\xdb\xe2\xa5\x4d\x94\xa4\xbf\x1d\xe7\x48\x53\xcc\x6f\x0d\xa0\xc1" + "\x42\xfe\x09\x8f\xbb\xe6\xe8\xd4\x32\x43\x14\x43\x4c\x1b\x98\xb5\x8e" + "\xad\xf1\x0a\xed\x9c\xb1\xdb\x1f\x2a\x0a\x1f\x1b\xb2", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80" + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\x74\x90\x79\x5b\xdb\xbb\xf5\xd0\xae\xcb\x9a\x4f\x65\xaa\x37\x9f", 1, + 128, 512 }, + { 96, 256, 128, 110, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\xe5\xb6\x3b\xf0\xf4\xb7\x28\x39\x12\xe2\x4b\x1a\x40\xdf\x0d\x22\x35" + "\x55\xdb\xe2\xa5\x4d\x94\xa4\xbf\x1d\xe7\x48\x53\xcc\x6f\x0d\xa0\xc1" + "\x42\xfe\x09\x8f\xbb\xe6\xe8\xd4\x32\x43\x14\x43\x4c\x1b\x98\xb5\x8e" + "\xad\xf1\x0a\xed\x9c\xb1\xdb\x1f\x2a\x0a\x1f\x1b\xb2\xc4\x05\xc1\x83" + "\x2f\x94\xee\x1a\x56\x30\xcd\x93\x91\x58\xa6\x21\xbd\x48\xc8\xff\x71" + "\x0d\x0a\x1f\x1e\xf7\x60\x18\x0e\x65\x88\x02\x2a\xd5\x5e\x41\x21\xd9" + "\x90\x69\x21\x8d\xb5\x5c\xd6\xaf\x7c\x10\x26\xf2\x1e\xcc\x0a\xf9\xb7" + "\xce\x8a\x7e\xa0\xb6\xf4\x26\xe9\x25", + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80" + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00" + "\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80" + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00" + "\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00" + "\x00\x80\x00\x00\x00\x80\x00\x00\x00", + "\x1d\x10\x96\xa8\xca\x9e\x2b\xda\x27\x62\xc4\x1d\x5b\x16\xf6\x2f", 1, + 128, 1024 }, + { 96, 256, 128, 111, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\x9a\x49\xc4\x8f\x8b\x48\xd7\x46\x6d\x1d\xb4\x65\x3f\x20\xf2\x5d\x4a" + "\xaa\x24\x9d\xda\xb2\x6b\xdb\xc0\xe2\x18\x37\x2c\x33\x90\x72", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\xaf\x84\x92\xc7\x92\xbf\x8d\x80\x62\xbe\x74\xff\x6e\xfb\x38\x69", 1, + 128, 256 }, + { 96, 256, 128, 112, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\x9a\x49\xc4\x8f\x8b\x48\xd7\x46\x6d\x1d\xb4\x65\x3f\x20\xf2\x5d\x4a" + "\xaa\x24\x9d\xda\xb2\x6b\xdb\xc0\xe2\x18\x37\x2c\x33\x90\x72\xdf\x3e" + "\xbd\x81\x76\x70\x44\x99\x97\x2b\xcd\x3c\x6b\xbc\xb3\x64\xe7\x4a\x71" + "\xd2\x8e\xf5\x12\xe3\xce\x24\xe0\x55\x75\xe0\xe4\xcd", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff" + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\xf2\x4d\xb6\x8c\x46\xb6\x7d\x6f\x40\x2f\xa6\xc8\x97\x91\x33\x68", 1, + 128, 512 }, + { 96, 256, 128, 113, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\x9a\x49\xc4\x8f\x8b\x48\xd7\x46\x6d\x1d\xb4\x65\x3f\x20\xf2\x5d\x4a" + "\xaa\x24\x9d\xda\xb2\x6b\xdb\xc0\xe2\x18\x37\x2c\x33\x90\x72\xdf\x3e" + "\xbd\x81\x76\x70\x44\x99\x97\x2b\xcd\x3c\x6b\xbc\xb3\x64\xe7\x4a\x71" + "\xd2\x8e\xf5\x12\xe3\xce\x24\xe0\x55\x75\xe0\xe4\xcd\xbb\xfa\x3e\xfc" + "\x50\x6b\x11\x65\x29\xcf\x32\xec\xee\xa7\x59\x5e\xc2\xb7\x37\x80\x0e" + "\xf2\xf5\x60\x61\x08\x9f\x67\x71\x9a\x77\x7d\x55\x2a\xa1\x3e\x5e\x26" + "\x6f\x16\x5e\x72\x4a\x23\xa9\x50\x83\x6f\x59\x0d\xe1\xb3\x75\x06\x48" + "\xb1\xf5\x81\x5f\xc9\x8b\xd9\x16\x5a", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff" + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f" + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff" + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", + "\x43\xf6\x51\xab\x2e\x2e\xb0\xf0\x4b\xf6\x89\xa4\x0d\x32\xda\x24", 1, + 128, 1024 }, + { 96, 256, 128, 114, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\x1a\x49\xc4\x0f\x0b\x48\xd7\xc6\xed\x1d\xb4\xe5\xbf\x20\xf2\xdd\xca" + "\xaa\x24\x1d\x5a\xb2\x6b\x5b\x40\xe2\x18\xb7\xac\x33\x90\xf2", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f" + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\x60\xd9\x52\x94\xa3\x69\x4c\xfa\xa6\x4b\x2f\x63\xbc\x1f\x82\xec", 1, + 128, 256 }, + { 96, 256, 128, 115, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\x1a\x49\xc4\x0f\x0b\x48\xd7\xc6\xed\x1d\xb4\xe5\xbf\x20\xf2\xdd\xca" + "\xaa\x24\x1d\x5a\xb2\x6b\x5b\x40\xe2\x18\xb7\xac\x33\x90\xf2\x5f\x3e" + "\xbd\x01\xf6\x70\x44\x19\x17\x2b\xcd\xbc\xeb\xbc\xb3\xe4\x67\x4a\x71" + "\x52\x0e\xf5\x12\x63\x4e\x24\xe0\xd5\xf5\xe0\xe4\x4d", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f" + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\xbe\xac\xa0\xb4\x70\x27\x19\x61\x76\x18\x6d\x94\x40\x19\xc1\xc8", 1, + 128, 512 }, + { 96, 256, 128, 116, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\x1a\x49\xc4\x0f\x0b\x48\xd7\xc6\xed\x1d\xb4\xe5\xbf\x20\xf2\xdd\xca" + "\xaa\x24\x1d\x5a\xb2\x6b\x5b\x40\xe2\x18\xb7\xac\x33\x90\xf2\x5f\x3e" + "\xbd\x01\xf6\x70\x44\x19\x17\x2b\xcd\xbc\xeb\xbc\xb3\xe4\x67\x4a\x71" + "\x52\x0e\xf5\x12\x63\x4e\x24\xe0\xd5\xf5\xe0\xe4\x4d\x3b\xfa\x3e\x7c" + "\xd0\x6b\x11\xe5\xa9\xcf\x32\x6c\x6e\xa7\x59\xde\x42\xb7\x37\x00\x8e" + "\xf2\xf5\xe0\xe1\x08\x9f\xe7\xf1\x9a\x77\xfd\xd5\x2a\xa1\xbe\xde\x26" + "\x6f\x96\xde\x72\x4a\xa3\x29\x50\x83\xef\xd9\x0d\xe1\x33\xf5\x06\x48" + "\x31\x75\x81\x5f\x49\x0b\xd9\x16\xda", + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f" + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff" + "\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f" + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff" + "\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff" + "\xff\x7f\xff\xff\xff\x7f\xff\xff\xff", + "\xd4\x81\x10\x28\xa5\x77\xd4\xdd\x69\xd6\xb3\x5d\x71\x7f\x73\xe3", 1, + 128, 1024 }, + { 96, 256, 128, 117, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x65\xb6\x3b\xf0\x8b\x48\xd7\xc6\x92\xe2\x4b\x1a\x3f\x20\xf2\xdd\xb5" + "\x55\xdb\xe2\xda\xb2\x6b\x5b\x3f\x1d\xe7\x48\x2c\x33\x90\xf2", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00" + "\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x10\xfb\x61\x27\x2b\x55\x5b\xee\x10\x4f\x5a\x71\x81\x87\x16\xd6", 1, + 128, 256 }, + { 96, 256, 128, 118, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x65\xb6\x3b\xf0\x8b\x48\xd7\xc6\x92\xe2\x4b\x1a\x3f\x20\xf2\xdd\xb5" + "\x55\xdb\xe2\xda\xb2\x6b\x5b\x3f\x1d\xe7\x48\x2c\x33\x90\xf2\x20\xc1" + "\x42\xfe\x76\x70\x44\x19\x68\xd4\x32\x43\x6b\xbc\xb3\xe4\x18\xb5\x8e" + "\xad\x8e\xf5\x12\x63\x31\xdb\x1f\x2a\x75\xe0\xe4\x4d", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00" + "\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00" + "\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00" + "\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x47\x56\x76\x4e\x59\x58\x35\x04\x18\x28\x77\xd8\xc3\x31\x20\xf0", 1, + 128, 512 }, + { 96, 256, 128, 119, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x65\xb6\x3b\xf0\x8b\x48\xd7\xc6\x92\xe2\x4b\x1a\x3f\x20\xf2\xdd\xb5" + "\x55\xdb\xe2\xda\xb2\x6b\x5b\x3f\x1d\xe7\x48\x2c\x33\x90\xf2\x20\xc1" + "\x42\xfe\x76\x70\x44\x19\x68\xd4\x32\x43\x6b\xbc\xb3\xe4\x18\xb5\x8e" + "\xad\x8e\xf5\x12\x63\x31\xdb\x1f\x2a\x75\xe0\xe4\x4d\x44\x05\xc1\x83" + "\x50\x6b\x11\xe5\xd6\x30\xcd\x93\xee\xa7\x59\xde\x3d\x48\xc8\xff\x0e" + "\xf2\xf5\xe0\x9e\xf7\x60\x18\x71\x9a\x77\xfd\xaa\xd5\x5e\x41\x5e\x26" + "\x6f\x96\xa1\x8d\xb5\x5c\xa9\x50\x83\xef\xa6\xf2\x1e\xcc\x75\x06\x48" + "\x31\x0a\x7e\xa0\xb6\x8b\xd9\x16\xda", + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00" + "\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00" + "\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00" + "\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00" + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff" + "\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff" + "\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff" + "\xff\x00\x00\x00\x00\xff\xff\xff\xff", + "\x95\xa2\xb1\x2a\x4a\x28\x00\x89\xd4\xbd\x4f\x90\x42\x53\xe7\x54", 1, + 128, 1024 }, + { 96, 256, 128, 120, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\x9a\x49\xc4\x0f\x74\xb7\x28\x39\x6d\x1d\xb4\xe5\xc0\xdf\x0d\x22\x4a" + "\xaa\x24\x1d\x25\x4d\x94\xa4\xc0\xe2\x18\xb7\xd3\xcc\x6f\x0d", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff" + "\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\x60\xdc\xd4\x59\x74\xbe\xbe\x03\x2e\xb7\xb8\x6c\x9d\x06\x34\x52", 1, + 128, 256 }, + { 96, 256, 128, 121, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\x9a\x49\xc4\x0f\x74\xb7\x28\x39\x6d\x1d\xb4\xe5\xc0\xdf\x0d\x22\x4a" + "\xaa\x24\x1d\x25\x4d\x94\xa4\xc0\xe2\x18\xb7\xd3\xcc\x6f\x0d\xdf\x3e" + "\xbd\x01\x89\x8f\xbb\xe6\x97\x2b\xcd\xbc\x94\x43\x4c\x1b\xe7\x4a\x71" + "\x52\x71\x0a\xed\x9c\xce\x24\xe0\xd5\x8a\x1f\x1b\xb2", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff" + "\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff" + "\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff" + "\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\xf0\xe6\xa3\xc1\xf2\x8a\xd9\x2d\x0d\xbc\x90\x0b\xe2\x91\xd8\x77", 1, + 128, 512 }, + { 96, 256, 128, 122, + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90" + "\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\x9a\x49\xc4\x0f\x74\xb7\x28\x39\x6d\x1d\xb4\xe5\xc0\xdf\x0d\x22\x4a" + "\xaa\x24\x1d\x25\x4d\x94\xa4\xc0\xe2\x18\xb7\xd3\xcc\x6f\x0d\xdf\x3e" + "\xbd\x01\x89\x8f\xbb\xe6\x97\x2b\xcd\xbc\x94\x43\x4c\x1b\xe7\x4a\x71" + "\x52\x71\x0a\xed\x9c\xce\x24\xe0\xd5\x8a\x1f\x1b\xb2\xbb\xfa\x3e\x7c" + "\xaf\x94\xee\x1a\x29\xcf\x32\x6c\x11\x58\xa6\x21\xc2\xb7\x37\x00\xf1" + "\x0d\x0a\x1f\x61\x08\x9f\xe7\x8e\x65\x88\x02\x55\x2a\xa1\xbe\xa1\xd9" + "\x90\x69\x5e\x72\x4a\xa3\x56\xaf\x7c\x10\x59\x0d\xe1\x33\x8a\xf9\xb7" + "\xce\xf5\x81\x5f\x49\x74\x26\xe9\x25", + "\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff" + "\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff" + "\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff" + "\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff" + "\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00" + "\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00" + "\x00\x00\xff\xff\xff\xff\x00\x00\x00\x00\xff\xff\xff\xff\x00\x00\x00" + "\x00\xff\xff\xff\xff\x00\x00\x00\x00", + "\x57\xef\xf4\xa5\x25\xee\xff\x2e\xbd\x7a\x28\xeb\x89\x42\x82\xbe", 1, + 128, 1024 }, + { 96, 256, 128, 123, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf5\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 124, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf6\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 125, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\x74\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 126, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x41\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 127, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\x37\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 128, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x28\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 129, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x2b\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 130, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x88\x14\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 131, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x15\xac\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 132, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\x8c\x51\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 133, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x50\x40\x54\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 134, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x55\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 135, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x56\x32\x3f\x44", 0, + 24, 0 }, + { 96, 256, 128, 136, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x45", 0, + 24, 0 }, + { 96, 256, 128, 137, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x46", 0, + 24, 0 }, + { 96, 256, 128, 138, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\x04", 0, + 24, 0 }, + { 96, 256, 128, 139, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x08\x14\xac\x51\x40\x54\x32\x3f\xc4", 0, + 24, 0 }, + { 96, 256, 128, 140, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xf4\x40\x9b\xb7\x29\x03\x9d\x88\x14\xac\x51\x40\x54\x32\x3f\xc4", 0, + 24, 0 }, + { 96, 256, 128, 141, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 24, 0 }, + { 96, 256, 128, 142, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "", "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 24, 0 }, + { 96, 256, 128, 143, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x28\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 144, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x2b\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 145, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\xa9\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 146, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x90\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 147, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x87\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 148, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa7\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 149, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa4\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 150, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\x53\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 151, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf0\x09\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 152, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x29\xbb\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 153, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xba\xa2\x1c\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 154, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1d\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 155, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1e\xe9\xa7\xd6", 0, + 24, 128 }, + { 96, 256, 128, 156, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd7", 0, + 24, 128 }, + { 96, 256, 128, 157, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\xd4", 0, + 24, 128 }, + { 96, 256, 128, 158, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\x96", 0, + 24, 128 }, + { 96, 256, 128, 159, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\xd3\xf1\x09\xbb\xa2\x1c\xe9\xa7\x56", 0, + 24, 128 }, + { 96, 256, 128, 160, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x29\x91\x40\x07\xa6\x11\x9d\x53\xf1\x09\xbb\xa2\x1c\xe9\xa7\x56", 0, + 24, 128 }, + { 96, 256, 128, 161, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 24, 128 }, + { 96, 256, 128, 162, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 24, 128 }, + { 96, 256, 128, 163, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x67\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 164, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x64\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 165, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\xe6\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 166, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x41\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 167, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x96\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 168, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe9\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 169, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xea\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 170, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\x3a\x92\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 171, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x93\xaa\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 172, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\x8a\x47\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 173, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x46\xf5\xce\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 174, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xcf\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 175, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xcc\xa5\x2b\x7a", 0, + 24, 264 }, + { 96, 256, 128, 176, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x7b", 0, + 24, 264 }, + { 96, 256, 128, 177, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x78", 0, + 24, 264 }, + { 96, 256, 128, 178, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\x3a", 0, + 24, 264 }, + { 96, 256, 128, 179, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\xba\x92\xaa\x47\xf5\xce\xa5\x2b\xfa", 0, + 24, 264 }, + { 96, 256, 128, 180, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x66\x40\x5a\x16\xe8\xb4\x4e\x3a\x92\xaa\x47\xf5\xce\xa5\x2b\xfa", 0, + 24, 264 }, + { 96, 256, 128, 181, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 24, 264 }, + { 96, 256, 128, 182, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", "\x00\x01\x02", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20", + "\xd0\x3b\xcb\x3c\xa5\x2d\x48\xd1\xd2\x03\xb1\xe7\xb1\xa5\x99\x5a\xf1" + "\xa0\x46\x6a\x61\xbb\x38\x6a\x2e\x12\xd1\x89\xa2\xc4\xea\x15\xe9", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 24, 264 }, + { 96, 256, 128, 183, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x8c" + "\x38\x2c\xf0\x71\x74\x14\x2e\xa5\x64\x92\x06\x12\x99\x7b\x1c\x2e\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x0f\xf9\xbc" + "\x3b\x94\x58\x29\x7b\xa0\x96\x7d\x86\xed\x09\x0b\x43\x51\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x03\xf3\xb7\x35\xb7" + "\x34\x54\x28\xeb\x93\xb3\xdb\x1d\x9b\x51\x87\xce\xbb\x88\x9a\xa1\x77" + "\xd8\x3e\x4f\x63\xfc\x9a\x5c\x05\x96\xee\xd9\x39\x88\x3d\x06\xaa\xcd" + "\xfd\xea\x44\xfd\xec\xdf\x5c\xb7\xfc", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xc2\x96\x43\x62\x46\xc3\xa7\xc4\xb3\xba\x09\xab\x2a\x6a\x08\x89", 1, + 512, 1024 }, + { 96, 256, 128, 184, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x05\x1e\x93\x73", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x93\x12\x27\x27\x4a\x89\xd0\xb3\xaa\xde\x7f\xac\x62\xc9\x62\x62\xc1" + "\xe7\x7b\x8d\xaf\xd2\x48\xf1\x0a\xd3\x7c\x6c\xcb\x69\xcb\x71\x31\xb0" + "\x41\x59\x3c\x8b\xb8\xc3\xdb\x38\xf3\x9d\xd8\xa1\x24\xc4\x24\xfc\xe4" + "\x38\x9d\xed\xe1\xd3\xcb\x9d\x46\xcf\x95\x97\x0a\xea\x98\x56\xb6\xe3" + "\x13\xd7\x56\x19\x7b\xaf\x4f\xcb\x58\xdf\x27\x5b\xca\x8a\x21\x88\xf9" + "\xe8\xa1\xad\x04\x35\x4e\xde\x54\x2d\xdc\x30\xe8\xb7\x35\xb2\xf5\x90" + "\x5f\x58\x11\x79\x92\x82\xbe\x94\xae\x84\x2e\xc1\x26\xc5\x5d\x2e\x66" + "\x72\x35\xe9\xac\xf1\xd4\x87\x98\xf0", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x99\xa3\xb0\xff\xf6\xfd\xcb\xcc\xe9\xdc\x58\x20\xf2\xa6\x48\x61", 1, + 512, 1024 }, + { 96, 256, 128, 185, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x04\x8c\x3c\x5f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x0d\xf9\x1f\x31\x23\x0e\x89\x41\xe7\x00\xa7\x52\xfe\xf0\x8c\x89\x7c" + "\x51\x1e\xd6\x18\xfd\xf8\xa3\x78\xa1\xf4\x39\x01\x3b\x40\xa4\x8d\x46" + "\x34\xc2\x7d\x9a\xda\x7c\x0b\xb6\xf3\xfa\x92\xe3\x41\x42\x59\x03\xd7" + "\xec\xd0\xc4\x9b\xee\x4c\x77\xe8\x4b\x11\xf1\xc7\x21\x92\x23\x08\x64" + "\x28\x85\xb8\x13\xfa\xe3\x64\xda\x32\xea\xf1\x20\xd6\xa4\x3a\x74\xfb" + "\x16\x32\x44\x36\x67\xbf\xea\x6e\xef\x1b\xe7\x3e\xb1\xc3\xc0\xb5\xa5" + "\x7c\xee\x8d\xc4\xfe\xed\x4a\x1f\xb9\xae\x02\xf7\xb1\x69\x55\x88\xc3" + "\xc8\x78\x45\x1c\xb6\xee\x0c\xb3\xdc", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xea\xff\x8f\x47\xef\x92\x68\xfd\x0d\x94\xe8\xa9\xc4\xb7\x8d\x24", 1, + 512, 1024 }, + { 96, 256, 128, 186, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x42" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\xce\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x58\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\x21\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xe2\x59\x0a\xf1\x20" + "\x1c\x7c\xfe\xc2\x29\x0c\xfc\xe9\x8a\x82\x2e\xbb\x8d\x1e\xd9\xdc\x4e" + "\x20\xd2\x41\x75\x5a\xff\x91\xcd\xfd\x10\xfd\xb6\x9e\xfa\x0d\x5c\x80" + "\x82\x69\x26\x01\xcb\xfb\xb9\x55\xc7", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x86\xed\x21\xfd\xa0\x80\xa7\xd1\x39\x81\x07\x8d\x86\xb3\xe3\xcd", 1, + 512, 1024 }, + { 96, 256, 128, 187, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x66\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x66" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xe8\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x50\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x64\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x98\x00\xac\x4e\xf9" + "\xd4\xe4\x01\x45\x24\xbc\x7c\xd3\x38\x72\x42\xe7\x74\xf4\xd1\xa7\xa0" + "\x52\x1e\x42\xec\x44\x84\x4d\x0b\xd8\xb9\xd7\x3f\xec\x95\x92\x12\xfd" + "\x7e\x8e\xac\xf4\xd9\x84\x99\x6d\x9b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x34\xf9\xe0\xfa\xa5\x15\xee\xe0\xe7\x84\xe6\xef\x26\x78\xbe\xfa", 1, + 512, 1024 }, + { 96, 256, 128, 188, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x26\xc6\x96\x1b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xe9\x72\x44\x25\x9a\xf5\xa3\x79\x23\x8d\xa0\xca\xd2\xa5\xf4\x93\x65" + "\x5e\xc0\xe5\x02\x4f\xd5\x53\xbb\xb3\xde\xb6\x6a\x94\x03\x6d\x10\x6c" + "\x3d\x51\x34\x07\xb2\xdd\x1c\xc5\x93\x6c\x4c\x9c\x1e\x4f\x4b\x37\xb5" + "\x4d\xec\x26\x1c\x60\x1d\xc9\x9e\x90\x68\x0e\x23\xe2\xdc\x5c\x9a\x8d" + "\x50\x3d\x8b\xea\x49\xa8\xcd\xca\x37\x06\xbf\xd2\xa3\xda\xa0\xaf\xb1" + "\x9a\x70\xfd\x3d\x35\x5f\xc3\x7c\x13\xf3\xf9\xe5\xc8\xd0\x86\x4a\x5f" + "\x80\xa7\x80\xb3\x6d\x46\x98\xec\x2c\xe9\xcc\xc2\x7b\x97\xec\xbe\x67" + "\x2e\x41\x62\x8e\xbd\x77\x3a\xcb\x81", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x3c\x94\xb9\xfe\x60\xbd\xb3\x5c\x6b\x7b\x73\xb7\x65\x08\x34\x92", 1, + 512, 1024 }, + { 96, 256, 128, 189, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x01\x3d\xa0\x60", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x94\x53\xaa\x15\x9c\x3d\x87\xf1\x7e\x21\xe8\x8a\xda\xbc\x37\xe5\x53" + "\xb9\x04\xd0\x0e\xef\xc6\x6b\x8e\x09\x05\xe2\x35\x76\xfb\xdc\x9c\x7b" + "\xea\x97\x77\xf3\xb8\x36\x84\x81\x93\x25\x34\xb3\x34\x4d\x30\x9e\x63" + "\x07\xcd\xdf\xe7\xb3\x54\x93\x00\xdd\x9c\xda\x7e\xfe\x9d\x43\xc8\xa1" + "\x15\x91\x2a\x39\x29\x04\x07\x9e\xe9\x2b\xcd\x33\x09\x9f\x70\x22\xea" + "\x94\xc1\xe7\x35\x3b\x89\xbf\xc5\x4d\xe3\xce\xb5\x6f\x52\x9a\x1a\x60" + "\x8b\xb5\xa9\x70\xe1\x35\x96\x09\xd1\xf5\x68\x06\xb3\x7f\x86\x05\xf4" + "\xc2\x74\x51\xda\x60\x66\xfc\x55\x7a", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x2b\x11\xcf\x9f\x8d\xb8\x49\x0d\x40\x9f\xc6\x2a\xfd\x73\x79\xf3", 1, + 512, 1024 }, + { 96, 256, 128, 190, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x07\xdb\x33\xde", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x2e\x18\x36\x64\x0d\x81\x0c\x27\x09\xfb\x83\xcc\xf1\xae\xf3\xa9\x71" + "\x08\x5d\x1b\xbf\xb5\x8a\x42\x5a\xbf\x75\xcc\xec\x70\xb3\xab\xde\x0e" + "\x80\x53\x9e\x83\xa8\x25\x46\xe7\x37\x2a\x19\x48\x15\x47\x05\x33\x08" + "\xdd\x78\x42\x67\x5e\x9c\x4f\x61\x30\x24\x26\xda\x0d\x71\xc1\xda\x31" + "\x02\x03\x10\x30\xed\x92\x81\x52\xbe\x00\x9b\x15\xb5\x2f\x71\xb5\x91" + "\x19\x91\xd3\x9f\x68\xa8\x65\x8d\x99\x72\x9d\xf2\xbb\xef\x31\xc8\x98" + "\x9f\x96\x04\x55\x8d\xf9\xf2\xab\xa4\xb3\x76\x6c\x58\xaa\xef\x35\x48" + "\xde\x54\x5e\xc1\xf0\x80\x22\x5a\x88", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xc9\xc8\x36\x69\x20\xf8\x83\x81\x40\x77\x12\xce\xc6\x1e\x66\x07", 1, + 512, 1024 }, + { 96, 256, 128, 191, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x02\xa1\x19\x42", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x0e\xcb\x4d\x85\xc9\x56\xb5\x26\x8c\x9b\x35\xa8\xc6\x3b\x4e\x9d\x3e" + "\x5c\xb7\x2b\x64\xef\x98\x77\x38\x41\xb9\x47\xbd\x7d\x59\xef\x7d\x0e" + "\xb0\xe1\xc0\x50\xd4\x9a\x54\x24\xce\x7d\xeb\x52\x7d\x76\x08\x7e\x47" + "\x46\x67\x4c\x95\x89\x65\xdf\x32\xd9\xe5\xfb\x03\xb4\x65\x01\x70\x61" + "\x28\xd4\x81\x21\x7a\xae\xae\x2f\x78\xf9\x25\x92\x73\x35\x8a\x29\x54" + "\xca\xc0\xbc\x2f\xbf\xe7\x74\x47\xd1\xd3\x87\xb9\x31\x4c\x65\x41\xb6" + "\x9f\x12\x70\xb3\x43\x8b\x10\x42\xb2\xb4\x66\x3e\x62\xba\x4d\x49\xc0" + "\x7a\xc6\xf1\x63\x03\x4a\xfa\x80\xaf", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x23\x73\xcf\xa2\xab\x24\x44\x6a\xd5\xa2\x36\x16\x7b\x80\x27\xfe", 1, + 512, 1024 }, + { 96, 256, 128, 192, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x3c\x0d\xf6\x37", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x2e\x8e\x45\xe9\x03\xbf\xab\x32\xf2\xf0\xd4\x9d\x9a\x3e\x44\x9b\xef" + "\x6f\x40\x93\xe2\x72\x2c\xda\xb2\xcf\x93\x5c\x18\x22\xb8\x30\xfb\x5a" + "\x40\x56\x51\x6d\x56\x0d\xfc\x86\x38\xc9\xa5\x7d\x29\x27\x20\x0a\x56" + "\xf0\xb6\x71\x53\x27\x1d\x49\x8e\x8f\x08\xdc\x88\x8c\x61\xef\x63\x4f" + "\x7a\xe4\x0f\x46\x08\xf9\x6f\x92\xfe\xa5\xa1\xe5\xbd\x45\x13\x11\x20" + "\x09\x8d\xc5\xde\x03\x78\xe5\x8f\x2d\xdb\x46\xfa\x4a\xa5\xad\xb3\x8f" + "\xe0\x06\xbb\x19\xb6\x91\x46\x38\x2f\x77\xa7\x9e\x06\x21\x4d\xef\x54" + "\x7c\xfb\x5c\xe3\x7a\x70\x08\xb9\xb6", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x5f\x93\x94\x64\x78\xd8\x08\x1e\x72\x47\xf4\x14\xad\x39\xa5\x15", 1, + 512, 1024 }, + { 96, 256, 128, 193, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x0b\xc6\x72\xc3", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x36\x19\xcb\x47\x0a\xf8\x6d\xce\xce\xb6\x94\x0f\x2d\x9a\xbb\x34\xc9" + "\xa9\x13\x14\x76\x05\x33\x87\x44\x5f\xfe\xbb\xe2\x40\xd4\xf9\x81\x83" + "\x77\x85\x56\x52\xf4\x6a\x82\x19\xc7\xf7\x1c\x35\x54\xf8\xac\xef\x82" + "\x58\xde\x4b\x7d\x17\xc0\xf3\xd3\x53\xac\x98\x1c\xc6\xa1\x32\x87\xbe" + "\x1e\x6b\x41\xdc\x6d\x13\x3d\xf4\xab\xab\xeb\xdf\x43\xd6\x65\xce\x7a" + "\x4a\x5c\x98\x2a\x0b\x13\x9c\xb8\x20\x2e\xeb\xc7\x41\x73\xe3\x22\x4a" + "\x44\x0e\x4c\x37\xd2\xb5\x95\xf3\x84\x29\x0e\x93\x9b\xa0\x16\xdf\x0d" + "\x49\xb3\x6c\xdb\x4b\xd9\x1c\x39", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x13\x3f\xe6\x23\x91\x74\x4d\x11\xce\x44\x59\x4b\x96\xc5\x3b\xaf", 1, + 512, 1016 }, + { 96, 256, 128, 194, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x03\xe9\xb9\xa4", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xaf\x20\x5b\xda\x81\x9f\x74\x51\xbe\x0f\x28\x66\x7d\x4b\x01\xb5\x9f" + "\xf2\xda\xa8\x17\x3c\xab\x52\x04\x6c\x3c\x9e\x0d\x98\x98\x89\xc5\xe0" + "\x21\xef\x7a\xfd\x06\xe9\xce\x6c\xc3\x0e\x3a\x6e\xba\xb5\x09\x13\x4b" + "\xa1\x0d\x10\xe5\x70\xc5\x55\x87\xc1\x3e\xee\x53\xe7\x3b\xe5\x48\x04" + "\xc8\x53\x9f\xfb\xf2\x3b\x35\x92\x2b\x1c\xa3\x7b\x9e\x9b\xc2\x4e\xe2" + "\x04\x83\x7c\xa5\xa2\x94\xce\x05\xd1\x26\x00\xc7\xef\xf6\xae\xe3\x22" + "\x70\xdb\x2f\xef\xf4\x7d\xc5\xa0\x41\x76\x16\x9e\x15\x85\x06\x28\xe6" + "\x03\x5f\x78\x99\x4f\x9f\x56\x03", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\xe3\x45\x1a\xdb\x9d\x23\xa7\x71\x0a\x1a\xaf\xba\x26\xf5\x63\x87", 1, + 512, 1016 }, + { 96, 256, 128, 195, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x07\x00\xb9\x82", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x68\xc6\x72\x72\x03\x6f\xb6\x52\xa0\x18\x2e\xeb\x47\x81\x35\x8e\x47" + "\x04\xa4\xa7\x02\xfd\x73\x1b\xf3\xb3\xea\x99\x47\x17\x98\x9e\x7d\x91" + "\x04\xe0\xae\x81\x73\x2a\x8c\x7e\x9a\x82\xb3\xd3\x1d\x54\x17\x61\xa3" + "\x66\xb6\x7c\x33\x96\xf1\xa6\xc6\x7e\x29\x3d\xdb\x65\xa5\x9e\x42\x54" + "\x1d\xda\x14\x4d\xc6\xc7\x83\x88\xcf\xca\x98\x2e\x23\x35\x09\x58\xac" + "\x5b\x3d\x54\xa1\x72\x2f\xd6\x47\x33\x57\x78\x62\xe1\x87\x9c\x9e\x94" + "\x45\xeb\xde\xc5\x31\x5d\x17\x06\xdb\x7e\xbb\xed\xd4\xc7\x79\x93\x5e" + "\x72\x05\x7e\x5b\x0e\xcd\xe0\x81", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\xb0\xbb\x8a\x55\xff\x5f\x52\xa5\x04\x3c\x6e\x77\x95\x84\x75\x57", 1, + 512, 1016 }, + { 96, 256, 128, 196, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x01\x98\x36\xbb", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xc4\x83\xb7\x33\x4e\xbe\x2e\x87\x9b\x0c\x3f\x9d\xb4\xfc\xd9\xf5\x21" + "\x90\x62\x36\x0d\x6c\xe4\x4c\xda\xe0\xf9\x4e\x04\xc8\x34\x5e\xa7\xe3" + "\xae\x33\x85\x51\x18\x74\x1d\xca\xfe\x0d\xe4\xae\x98\xc4\xe4\x3a\xf7" + "\xb1\x2b\x04\xee\x8a\xb1\x75\x62\x58\x23\xac\x04\x0e\x5a\xba\xc4\x40" + "\x3f\x1d\x45\x23\x8a\xdc\xb8\xc0\xcf\x44\xbd\x56\x91\x7f\x9f\x5d\x93" + "\x97\x4c\x82\xb5\x69\x51\x98\x6a\x9c\x04\x50\xbd\x90\x47\xb5\xa6\x16" + "\xe8\x14\x52\x6a\xd0\x58\x0e\x3e\xcd\x81\x89\xc9\xfe\xf2\xcd\xb9\x79" + "\xa2\x2a\xd3\xa0\x19\x30\xfb\xd1", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\xf4\xfc\x25\xf4\xc5\x54\x3a\x9a\xfe\xe9\x81\x9e\x29\x04\xfb\x68", 1, + 512, 1016 }, + { 96, 256, 128, 197, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x1d\x59\xf2\x88", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xbc\x7f\x4f\x15\xfd\x1e\x4c\x13\x99\x74\x08\x36\x67\x0a\xbe\x39\xa0" + "\x57\x07\xbe\x19\x95\x6c\xe1\x69\xb3\x23\x21\x75\x9e\x0f\x21\x3a\xe1" + "\x9a\xd3\x4a\xa6\x12\xb3\xa2\x9f\x02\xc4\xbb\xac\x9f\x78\x5a\x55\xa3" + "\xad\xfe\x41\x9a\xb8\x91\xbb\xe0\xac\xee\x99\x21\x32\x2e\xa2\x10\x02" + "\xc9\xdd\x3d\xcd\xd1\x3a\x7f\x85\x54\xdd\xdc\x10\xf9\xb5\x29\xce\x94" + "\xbe\x70\x50\x93\x7d\xab\x76\x55\x7b\x7e\xb1\x7c\x68\x5a\xad\x8f\x07" + "\x97\xe3\x9d\x62\x55\x39\x88\x98\x9a\xab\x1d\x97\x64\xfe\x43\x1c\xc1" + "\xd4\xc5\x95\x06\x2c\xe9\x3c\xe9", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x5e\x67\xa7\xb8\x73\x3e\x0e\x4b\x01\xac\x21\x78\xa2\x05\xae\x7e", 1, + 512, 1016 }, + { 96, 256, 128, 198, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x05\x52\xa4\x11", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xea\xcc\xaa\x77\x89\x35\xef\x24\x9e\x09\x00\x14\x9d\xd8\x89\x46\x2d" + "\x2a\x06\x14\x86\xba\x10\x2b\x8c\xae\xbe\x46\x5f\x39\x59\xfb\x31\x19" + "\xeb\xb5\x68\x96\x76\xff\xdd\x6d\x85\x1a\x26\x73\x9e\x77\x2b\x54\xa2" + "\xf5\xf4\x73\xea\x9c\x7e\x58\xcc\xbc\x4c\xfc\x95\x3e\x8c\x42\x0b\x21" + "\x75\xd9\xdd\x51\x92\x65\x63\x0b\xb7\x9b\xd8\x7a\x60\x1b\x11\x32\x31" + "\xa8\xb1\x6c\xe5\x4c\x33\x13\x47\xec\x04\xc2\xb1\xc9\x16\x0f\x38\x20" + "\x7a\xa4\x6e\x96\xfe\xb0\x6d\xee\x88\x3e\xb4\x22\xfa\x14\x90\x8d\xf3" + "\x00\xbb\x1a\x1e\xf7\x58\xc4\x08", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x17\x7a\x77\xfc\xe1\x14\xa4\x34\x9c\x4f\x8d\x5e\xc8\x25\xd0\x6f", 1, + 512, 1016 }, + { 96, 256, 128, 199, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x0c\x80\x7a\x72", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xa7\x6c\x33\x0e\x01\x50\x60\xa1\x7e\x64\xcb\x7b\x6d\x75\x3f\x20\x1f" + "\x75\xbe\x87\x59\xfd\x75\x39\xfb\x92\xb2\x2a\xef\x54\xc9\xd3\x02\x9d" + "\xba\x0c\x15\xcb\xf7\xc9\x51\x35\x88\x83\x19\xc6\xb2\xe6\x27\x6d\xa2" + "\x1e\x0c\x35\x1f\xd5\x22\xb2\x9a\xab\xb5\x88\x3a\x32\x91\xd6\xf4\x27" + "\xde\x77\x3b\x12\x43\x90\xef\x6f\xd9\x66\x21\xff\xbc\x42\xdf\xbf\x7a" + "\x34\xda\x27\x2c\xbc\x9c\xcb\x1a\x49\x8d\x07\x80\x33\xd1\xac\x3b\xf7" + "\xe9\x27\x15\x94\x8b\x06\xd6\x9d\x5c\x50\x39\xe9\x16\x4b\xa9\xc3\xa0" + "\x22\x19\xec\x59\x08\x20\x6b\x3b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x62\x3c\x7d\x44\x24\xf5\x49\x7a\xed\xfd\x13\x39\xcf\x8c\xec\xce", 1, + 512, 1016 }, + { 96, 256, 128, 200, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x03\x97\xa1\x43", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x22\x8a\x7e\x15\xbc\xce\x13\x05\x1d\xe9\x14\x5f\x77\xf7\xf4\xff\x79" + "\x21\x82\x8b\x4f\x99\xef\xc4\xff\x55\xee\x0d\x93\x44\x95\x5b\x69\xec" + "\x2d\x47\x98\xb0\x51\x7f\x02\x73\xc4\x45\x6a\xe5\xff\xc5\x92\x9c\xbe" + "\x74\xdd\xb0\xda\x51\xd4\xf2\xb4\xdf\x75\x78\xa3\x12\x40\xc8\x8a\xe9" + "\x22\xc3\xc5\xec\xa7\xb9\x7d\x72\xd4\x97\x06\x20\x50\xa5\x87\x44\x7c" + "\x56\x2b\x34\x3d\x5c\x71\x92\x19\x44\x87\x2f\x9f\xd0\x6b\x8f\x34\xb3" + "\xeb\x5d\x43\x41\xf5\xff\x8a\x90\x7d\xd7\xc2\xe1\x67\x6b\x81\x25\x27" + "\x26\xba\x54\x81\x4d\xa5\x1e\xab", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x1c\x18\xb6\x93\x54\xb1\x89\x73\x1a\x1a\x83\xfe\x8f\x0d\x57\xc9", 1, + 512, 1016 }, + { 96, 256, 128, 201, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x08\xcb\x0f\x3f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xc7\xd8\x43\x18\x8a\xb1\x93\xdf\xef\x5c\x4d\xaf\x58\x3f\x95\x2c\xd4" + "\xb1\x95\xf2\x40\xfa\x2e\x70\x4d\x02\x17\x23\x02\x3c\x12\x33\x71\xa4" + "\x1e\x87\xdf\xc6\xe6\xc3\x87\x4a\x42\xf3\x31\xcf\x03\x59\x88\xa3\x8c" + "\x72\xba\x2d\xa8\x54\xb1\x20\x8f\x98\xbf\x8c\xc2\x99\x48\x16\x94\x81" + "\xab\x3a\x40\x2d\x5f\xcc\x7f\xf7\x8f\x9e\x31\x92\x55\x76\xdc\x39\x38" + "\x07\x4b\x8c\x5b\x27\x96\x0e\x3a\xfc\x75\x0a\xd6\x86\x56\x36\x88\xb7" + "\x44\x17\x87\x28\x8d\x52\x56\xc1\x30\x1d\x56\x3b\x77\x44\x84\x3b\xd1" + "\xab\x4e\xff\x5b\xe6\xf1\x65\x3d", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x20\x45\x81\x5b\x82\x11\xb9\xa2\x99\x5e\xff\xe0\xb8\xed\x98\x68", 1, + 512, 1016 }, + { 96, 256, 128, 202, + "\x9d\xe8\x36\xaa\x57\x95\x85\x08\x1f\x33\x0a\x7c\x40\x36\xe2\x0e\x38" + "\xef\x15\xef\xf3\x94\x51\x84\xd2\x31\x86\x7f\x50\x5f\xff\xdf", + "\x00\x00\x00\x00\x10\x11\x12\x13\x0d\x8f\xcf\x4e", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xcf\xc3\xdb\x86\x31\xc8\x1c\x69\x02\x3a\x3c\x8a\x9a\xd6\x6c\x35\x05" + "\x36\x85\x14\x4c\x4f\xa2\xa9\x51\x0a\xdd\x72\xe2\x11\xda\xd9\xca\x5b" + "\x98\x2e\x4c\x19\x45\x91\xfd\xb7\x41\x16\x28\x03\x11\xd1\x29\x9a\xd8" + "\x12\x27\x25\x8c\xb5\x2f\x07\x9b\xbc\xb1\x2a\xff\x16\x1d\x27\x8d\xec" + "\x33\xa3\x26\xd7\x12\x76\xb3\xde\x01\xa8\x32\x7e\xe7\xf4\x5f\x94\x17" + "\x9d\xff\x18\xa3\xfe\x64\x3e\x56\xc3\x0c\xfd\x03\x87\x1c\x81\x10\xab" + "\x00\xf6\x61\x2b\x9e\x17\xa4\x64\x73\x60\xd7\x84\x7b\xb6\x3a\x31\x22" + "\x61\x3c\x2e\x7c\xdd\xdd\x08\xae", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff", + "\x1a\xe2\xed\x84\xea\x97\x74\xd7\x8d\x78\x2b\xf8\xd9\x72\xa8\xb8", 1, + 512, 1016 }, + { 96, 256, 128, 203, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x41" + "\x57\x71\xfd\xa4\xfb\xcc\x55\xc3\x77\xf7\x32\x03\xe6\x02\x26", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 1, + 256, 512 }, + { 96, 256, 128, 204, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xf1\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x61" + "\x5a\xf3\x9e\xdd\xb5\xfc\xd2\x51\x91\x90\xd5\x50\x7d\x3b\x06", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 256, 512 }, + { 96, 256, 128, 205, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xb5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x76" + "\x4e\x5d\x82\xce\x7d\xa0\xd4\x41\x48\x48\x4f\xd9\x6a\x61\x07", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 256, 512 }, + { 96, 256, 128, 206, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xfd\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2b" + "\xdb\xf1\x6d\x8e\xa4\xd3\x9d\xab\x8d\xcb\x3d\x4b\xc4\xe1\x04", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80\x00\x00\x00\x80", 1, + 256, 512 }, + { 96, 256, 128, 207, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xa9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xac" + "\xcd\x5e\xb3\x1d\x8f\xc9\x09\xe8\x4b\x0d\xe7\xde\x23\xbb\x08", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f\xff\xff\xff\x7f", 1, + 256, 512 }, + { 96, 256, 128, 208, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xd2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xdd" + "\x4b\x93\x3e\x7b\x1a\x7e\xd9\x3c\xc7\xc0\x50\xdb\x71\xdc\x03", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\x01\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00\x01\x00\x00\x00", 1, + 256, 512 }, + { 96, 256, 128, 209, + "\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50" + "\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa0" + "\x81\x64\x42\x5d\x76\x42\xe9\xe9\x0f\xc8\xd5\xc3\x2d\x2c\xf6", + "\xe4\x8c\xaf\x8a\x76\x18\x33\x27\xc9\x56\x1a\x46\x51\xc0\x7c\x82\x2c" + "\xcd\x16\x42\xc0\x66\x07\xd0\xd4\xbc\x0a\xfb\x4d\xe1\x59\x15\xdb\xfa" + "\x3b\x0b\x42\x2e\x77\xe1\x5c\x64\xbf\x62\x47\x03\x1f\x15\xfd\xb6\x43" + "\x11\x78\x09\x82\x18\x70\x00\x0a\xdf\x83\x83\x4d\xa5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + "\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 256, 512 }, + { 96, 256, 128, 210, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc6\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x7b" + "\xb6\x6f\x80\x90\xc1\x49\xe4\x52\xec\x7f\x20\x32\x7e\xb2\xea\x2e\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x0f\xf9\xbc" + "\x23\xc8\x97\xdf\x6b\x00\xaf\x86\x93\x1d\x6c\x81\x55\x51\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x03\xf3\xb7\x2d\xeb" + "\xfb\xa2\x38\x4b\xaa\x48\xce\xed\xfe\xdb\x91", + "\xe5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x08" + "\x71\xbc\x8f\x1e\x4a\xa2\x35\x08\x77\x12\xd9\xdf\x18\x36\x09\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xe7\xa3\x30\x09\xef\x5f\xc6\x04\xea\x0f\x9a\x75\xe9\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe7\xa3" + "\x30\x09\xef\x5f\xc6\x04\xea\x0f\x9a\x75\xe9", + "\x35\x72\x16\x27\x77\x26\x2c\x51\x8e\xef\x57\x3b\x72\x0e\x8e\x64", 1, + 32, 768 }, + { 96, 256, 128, 211, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc7\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x76" + "\x20\x9e\xef\x14\x16\x91\xfb\xa5\xd1\x0e\xaf\x58\x1a\xff\xe6\x2e\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x0e\x73\xd2" + "\xdc\x3b\xbd\x95\x49\x89\xcb\x84\x33\xb7\xd6\x59\x7b\x51\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x02\x79\xd9\xd2\x18" + "\xd1\xe8\x1a\xc2\xce\x4a\x6e\x47\x44\x03\xbf", + "\xe4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x05" + "\xe7\x4d\xe0\x9a\x9d\x7a\x2a\xff\x4a\x63\x56\xb5\x7c\x7b\x05\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\x75\x91" + "\x18\x50\x1a\x43\xcd\xd6\xa2\x06\x4a\xa5\x20\xad\xc7\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\x75\x91\x18\x50" + "\x1a\x43\xcd\xd6\xa2\x06\x4a\xa5\x20\xad\xc7", + "\x34\x72\x16\x37\x5f\x5b\x7b\x5c\x4e\x6b\xff\x49\x12\xfd\x94\x73", 1, + 32, 768 }, + { 96, 256, 128, 212, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xfc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x8b" + "\x79\x40\x3d\xfa\xab\xc0\xd8\xc1\x8d\x23\xa3\x46\x9c\x13\xe6\x2e\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x0a\x4b\x94" + "\x1e\x6b\x66\xfc\xc2\xed\x7d\x8c\xb3\xe8\xcc\x7f\xfc\x51\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x06\x41\x9f\x10\x48" + "\x0a\x81\x91\xa6\x78\x42\xee\x18\x5e\x25\x38", + "\xdf\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf8" + "\xbe\x93\x32\x74\x20\x2b\x09\x9b\x16\x4e\x5a\xab\xfa\x97\x05\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfa\x4d\xd7" + "\xda\x00\xc1\x2a\x46\xb2\x14\x0e\xca\xfa\x3a\x8b\x40\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfa\x4d\xd7\xda\x00" + "\xc1\x2a\x46\xb2\x14\x0e\xca\xfa\x3a\x8b\x40", + "\x30\x72\x16\x77\xff\x2e\xb8\x89\x4e\x5a\x9d\x84\x92\xb7\xb0\xaf", 1, + 32, 768 }, + { 96, 256, 128, 213, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xfa\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x9b" + "\xcb\xb8\xda\x47\x7d\x58\x0d\x77\x2d\xe4\x22\x9b\xba\x7d\xe2\x29\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x1e\x9d\xed" + "\xf9\xdd\x64\xa0\x68\x1b\xac\x29\x69\x54\x94\x25\xbc\x56\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x12\x97\xe6\xf7\xfe" + "\x08\xdd\x3b\x50\xa9\xe7\x34\xa4\x06\x7f\x78", + "\xd9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe8" + "\x0c\x6b\xd5\xc9\xf6\xb3\xdc\x2d\xb6\x89\xdb\x76\xdc\xf9\x01\xf8\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xee\x9b\xae" + "\x3d\xb6\xc3\x76\xec\x44\xc5\xab\x10\x46\x62\xd1\x00\xf8\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xee\x9b\xae\x3d\xb6" + "\xc3\x76\xec\x44\xc5\xab\x10\x46\x62\xd1\x00", + "\x2b\x72\x16\xc7\x87\x37\x44\xc2\x0e\xc5\xe2\xcd\xb2\x60\xd3\xfa", 1, + 32, 768 }, + { 96, 256, 128, 214, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xee\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xb9" + "\xf5\x5b\xd5\x6e\x0f\xd7\x4b\x46\x06\x3a\x96\x35\x4c\xfb\xee\x32\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x20\xc7\x88" + "\x86\xa6\xf6\x29\x2d\x6c\xc5\xfb\xdd\xb5\x46\xa2\xb0\x4d\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x2c\xcd\x83\x88\x85" + "\x9a\x54\x7e\x27\xc0\x35\x80\x45\xd4\xf8\x74", + "\xcd\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xca" + "\x32\x88\xda\xe0\x84\x3c\x9a\x1c\x9d\x57\x6f\xd8\x2a\x7f\x0d\xe3\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd0\xc1\xcb" + "\x42\xcd\x51\xff\xa9\x33\xac\x79\xa4\xa7\xb0\x56\x0c\xe3\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd0\xc1\xcb\x42\xcd" + "\x51\xff\xa9\x33\xac\x79\xa4\xa7\xb0\x56\x0c", + "\x22\x72\x16\x57\xb0\x13\x0d\x28\xcf\x1e\xc6\x51\x53\xc4\x11\x82", 1, + 32, 768 }, + { 96, 256, 128, 215, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xef\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xb4" + "\x6f\xca\x24\xd3\x53\xff\x5e\x49\xea\xc5\x15\x40\xe8\x40\xea\x30\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x3d\x31\x1e" + "\x57\x22\x02\x01\x1a\x75\xe9\x48\x58\x6f\xe2\x68\xb4\x4f\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x31\x3b\x15\x59\x01" + "\x6e\x7c\x49\x3e\xec\x86\x05\x9f\x70\x32\x70", + "\xcc\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc7" + "\xa8\x19\x2b\x5d\xd8\x14\x8f\x13\x71\xa8\xec\xad\x8e\xc4\x09\xe1\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xcd\x37\x5d" + "\x93\x49\xa5\xd7\x9e\x2a\x80\xca\x21\x7d\x14\x9c\x08\xe1\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xcd\x37\x5d\x93\x49" + "\xa5\xd7\x9e\x2a\x80\xca\x21\x7d\x14\x9c\x08", + "\x21\x72\x16\x67\x98\x48\x5c\x33\x8f\x9a\x6d\x60\xf3\xb2\x18\x91", 1, + 32, 768 }, + { 96, 256, 128, 216, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf5\x9d\x56\x15\x1d\xe2\x8b\xef\x83\x50\x5f\x6d\x89\xc0\xb0\xf7\xf7" + "\x5b\x2f\xa8\xe6\xdc\xe3\x86\x07\x5d\xb2\x83\xec\x85\xee\x62\x55\x5b" + "\xaf\xfa\xd4\x23\xaf\x25\xf6\x60\x69\xbb\x69\xfb\x6f\x4d", + "\xd6\xee\x4e\xe2\x5d\x3b\xde\xa8\x1e\x76\xde\x89\x34\xcc\x51\xfb\x84" + "\x9c\xfc\xa7\x68\x57\x08\x57\x5d\xc6\xdf\x7a\x01\xe3\x6a\x81\x84\x9c" + "\xfc\xa7\x68\x57\x08\x57\x5d\xc6\xdf\x7a\x01\xe3\x6a\x81", + "\x83\x13\x12\xcb\xb0\xf1\x65\xdc\x3e\x8f\xf5\x21\x25\xf4\x86\x40", 1, + 32, 384 }, + { 96, 256, 128, 217, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf7\x17\xf8\xd5\xb2\x80\x32\xd5\xc8\xe8\x06\x1c\xd4\x4d\x71\xe4\xf2" + "\xd5\x5d\xe7\x72\xfe\x7a\x91\xce\x85\xe4\x10\xdb\x3e\x2d\x8d\x50\xd5" + "\xdd\xb5\x40\x01\x36\x32\x3f\xb8\x3f\x28\x5e\x40\xac\xa2", + "\xd4\x64\xe0\x22\xf2\x59\x67\x92\x55\xce\x87\xf8\x69\x41\x90\xe8\x81" + "\x12\x8e\xe8\xfc\x75\x91\x40\x94\x1e\x89\xe9\x36\x58\xa9\x6e\x81\x12" + "\x8e\xe8\xfc\x75\x91\x40\x94\x1e\x89\xe9\x36\x58\xa9\x6e", + "\x82\x13\x12\xdb\x98\x26\xb5\xe7\xfe\x0a\x9d\x30\xc5\xe2\x8d\x4f", 1, + 32, 384 }, + { 96, 256, 128, 218, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf2\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xe6" + "\x8a\x92\x2c\x92\x19\xd3\x0f\x07\x55\x4d\x7d\x99\xf2\xbd\xe9\x2c\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\xe2\x4c\x07" + "\xdd\x98\xf9\xb2\x53\xab\x0c\x31\x8d\x9b\x14\xf6\xb1\x53\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\xee\x46\x0c\xd3\xbb" + "\x95\xcf\x00\xe0\x09\xff\xd0\x6b\x86\xac\x75", + "\xd1\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x95" + "\x4d\x41\x23\x1c\x92\x38\xde\x5d\xce\x20\x84\x74\x94\x39\x0a\xfd\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x12\x4a\x44" + "\x19\xf3\x5e\x64\xd7\xf4\x65\xb3\xf4\x89\xe2\x02\x0d\xfd\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x12\x4a\x44\x19\xf3" + "\x5e\x64\xd7\xf4\x65\xb3\xf4\x89\xe2\x02\x0d", + "\xc1\x04\x57\x69\xd4\x87\xd5\x45\xce\xf3\xf0\xd3\x4b\x7a\x87\x33", 1, + 32, 768 }, + { 96, 256, 128, 219, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x2e" + "\x67\x84\xd8\x57\xdf\x07\x54\x3d\x0d\xc7\x2f\x17\x99\x35\xfb\xed\xe8" + "\xc8\xba\xf0\x1e\xe2\x04\x4b\x16\x2c\xbb\x34\x3b\x35\x5a\xcc\x29\xd8" + "\x23\x27\xcd\x93\xf2\xbf\xd9\x18\x03\x4e\xd5\xc4\x2a", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x5d" + "\xa0\x57\xd7\xd9\x54\xec\x85\x67\x96\xaa\xd6\xfa\xff\xb1\x18\x3c\x2f" + "\x9b\xe7\x4c\x6a\x45\x76\xe0\xb0\x9a\x7a\x5c\x23\x30\x96\x3c\x2f\x9b" + "\xe7\x4c\x6a\x45\x76\xe0\xb0\x9a\x7a\x5c\x23\x30\x96", + "\x64\xe7\xef\xd2\x45\x16\xa8\x3e\x2c\x87\xe0\x6a\x76\xe2\xde\xa3", 1, + 32, 512 }, + { 96, 256, 128, 220, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf7\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x4c" + "\x6e\xad\x26\xf8\x4a\x02\x25\xd5\x57\x74\x5d\x32\xfc\x72\xe7\x2c\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x64\xdb\x33" + "\x4b\x69\xbe\xe5\x79\x38\x3e\x61\xae\x74\x2c\x71\xbb\x53\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x68\xd1\x38\x45\x4a" + "\xd2\x98\x2a\x73\x3b\xaf\xf3\x84\xbe\x2b\x7f", + "\xd4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x3f" + "\xa9\x7e\x29\x76\xc1\xe9\xf4\x8f\xcc\x19\xa4\xdf\x9a\xf6\x04\xfd\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x94\xdd\x70" + "\x8f\x02\x19\x33\xfd\x67\x57\xe3\xd7\x66\xda\x85\x07\xfd\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x94\xdd\x70\x8f\x02" + "\x19\x33\xfd\x67\x57\xe3\xd7\x66\xda\x85\x07", + "\xe6\xcc\x67\x29\xd7\x9b\xa5\x58\xcd\x73\xb0\x3c\xba\x54\xd6\x60", 1, + 32, 768 }, + { 96, 256, 128, 221, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf0\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x4f" + "\xd8\xc3\x75\x7c\x9f\x29\x38\xdc\x3b\x07\xd8\x58\x98\xbf\xe2\x2a\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x61\x55\x41" + "\x24\x15\xcb\xdd\x76\x01\x42\xb6\x2c\x2e\xc8\x3f\xbf\x55\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x6d\x5f\x4a\x2a\x36" + "\xa7\xa0\x25\x4a\x47\x78\x71\xde\x5a\x65\x7b", + "\xd3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x3c" + "\x1f\x10\x7a\xf2\x14\xc2\xe9\x86\xa0\x6a\x21\xb5\xfe\x3b\x01\xfb\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x91\x53\x02" + "\xe0\x7e\x6c\x0b\xf2\x5e\x2b\x34\x55\x3c\x3e\xcb\x03\xfb\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x91\x53\x02\xe0\x7e" + "\x6c\x0b\xf2\x5e\x2b\x34\x55\x3c\x3e\xcb\x03", + "\xe5\xcc\x67\x39\xbf\xd0\xf4\x63\x8d\xef\x57\x4b\x5a\x43\xdd\x6f", 1, + 32, 768 }, + { 96, 256, 128, 222, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf2\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xdf" + "\x03\xca\x84\x08\x2f\x7f\x70\xad\x8e\x40\x04\xca\xbd\x2c\xe4\x2b\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x28\xfd\x41" + "\x3c\xaa\xb1\xd0\x2b\xf1\xc6\x57\x53\xaa\x2a\xd3\xb9\x54\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x24\xf7\x4a\x32\x89" + "\xdd\xad\x78\xba\xc3\x99\x0e\x5a\xb8\x89\x7d", + "\xd1\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xac" + "\xc4\x19\x8b\x86\xa4\x94\xa1\xf7\x15\x2d\xfd\x27\xdb\xa8\x07\xfa\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd8\xfb\x02" + "\xf8\xc1\x16\x06\xaf\xae\xaf\xd5\x2a\xb8\xdc\x27\x05\xfa\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd8\xfb\x02\xf8\xc1" + "\x16\x06\xaf\xae\xaf\xd5\x2a\xb8\xdc\x27\x05", + "\x0f\xca\x70\x22\x28\x81\x7d\x53\xee\x64\xd1\x42\xb1\x92\xe6\x65", 1, + 32, 768 }, + { 96, 256, 128, 223, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf3\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x1f" + "\xfc\x31\xae\x69\x39\x93\x94\xb8\xc3\x38\x67\x4c\x3d\xfd\xe9\x29\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x47\x7e\xc8" + "\xcf\x3e\xa3\xd4\xd5\xd7\x6d\x85\xad\x2b\x7f\x0b\xb8\x56\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x4b\x74\xc3\xc1\x1d" + "\xcf\xa9\x86\x9c\x68\x4b\xf0\xdb\xed\x51\x7c", + "\xd0\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x6c" + "\x3b\xe2\xa1\xe7\xb2\x78\x45\xe2\x58\x55\x9e\xa1\x5b\x79\x0a\xf8\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb7\x78\x8b" + "\x0b\x55\x04\x02\x51\x88\x04\x07\xd4\x39\x89\xff\x04\xf8\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb7\x78\x8b\x0b\x55" + "\x04\x02\x51\x88\x04\x07\xd4\x39\x89\xff\x04", + "\xef\xc3\xb0\x35\xde\xd6\xb4\x60\xbf\xce\x6f\x49\x49\x55\xe6\x77", 1, + 32, 768 }, + { 96, 256, 128, 224, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\x2b\xfd\x0d\x56\xec\xe9\x87\x71\x75\x6d\x60\xd9\xd9\x10\x6c\xd0\xc6" + "\xfc\x10\x69\x36\xc7\xef\x34\x7c\x07\x8f\xd7\x1c\x54\x22\x81\x64\xfc" + "\x90\x3b\x04\x38\xa3\x97\x8d\x3a\x54\xef\x99\x2a\xa3\xae", + "\x08\x8e\x15\xa1\xac\x30\xd2\x36\xe8\x4b\xe1\x3d\x64\x1c\x8d\xdc\xb5" + "\x3b\xc3\x66\xb8\x4c\x04\xe5\x26\x9c\xe2\x2e\xf1\x32\xa6\x62\xb5\x3b" + "\xc3\x66\xb8\x4c\x04\xe5\x26\x9c\xe2\x2e\xf1\x32\xa6\x62", + "\x34\x5f\xc9\xfe\x57\x3c\x13\x6c\x1b\xe8\x37\x30\x50\x0c\xe6\x62", 1, + 32, 384 }, + { 96, 256, 128, 225, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf6\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x7c" + "\xc2\x25\x5d\xec\xdf\x8e\x0f\xe1\x37\x35\x91\xda\x0e\x28\xe4\x28\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\xe2\x91\xfb" + "\x48\x38\x01\x9c\x51\xdf\xb7\x14\x15\x15\xbb\x53\xb1\x57\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\xee\x9b\xf0\x46\x1b" + "\x6d\xe1\x02\x94\xb2\xda\x48\xe5\x29\x09\x75", + "\xd5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x0f" + "\x05\xf6\x52\x62\x54\x65\xde\xbb\xac\x58\x68\x37\x68\xac\x07\xf9\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x12\x97\xb8" + "\x8c\x53\xa6\x4a\xd5\x80\xde\x96\x6c\x07\x4d\xa7\x0d\xf9\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x12\x97\xb8\x8c\x53" + "\xa6\x4a\xd5\x80\xde\x96\x6c\x07\x4d\xa7\x0d", + "\x33\x6f\x97\xa5\xfa\xa9\x95\xa2\xa0\x37\x81\xb5\x91\x58\x8d\xa8", 1, + 32, 768 }, + { 96, 256, 128, 226, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc6\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x7a" + "\xb6\x6f\x80\x90\xc1\x49\xe4\x52\xec\x7f\x20\x32\x7e\xb2\xea\x04\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x8d\x26\x13" + "\xea\x0e\xf8\xb6\x56\xb2\x47\x37\x3e\xce\xc0\x15\xbc\x7b\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x81\x2c\x18\xe4\x2d" + "\x94\xcb\x05\xf9\x42\xf9\x63\x3e\x52\x4f\x78", + "\xe5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x09" + "\x71\xbc\x8f\x1e\x4a\xa2\x35\x08\x77\x12\xd9\xdf\x18\x36\x09\xd5\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7d\x20\x50" + "\x2e\x65\x5f\x60\xd2\xed\x2e\xb5\x47\xdc\x36\xe1\x00\xd5\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7d\x20\x50\x2e\x65" + "\x5f\x60\xd2\xed\x2e\xb5\x47\xdc\x36\xe1\x00", + "\x93\x51\xc6\x80\xc8\xa5\xd3\x48\x82\xd4\x21\x45\xe8\x97\x45\xc4", 1, + 32, 768 }, + { 96, 256, 128, 227, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc6\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x74" + "\xb6\x6f\x80\x90\xc1\x49\xe4\x52\xec\x7f\x20\x32\x7e\xb2\xea\x2e\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\xac\xd9\xec" + "\x85\x9e\x08\x66\x62\x0c\xc2\x4c\x8a\x97\xd5\xd9\xf5\x51\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\xa0\xd3\xe7\x8b\xbd" + "\x64\x1b\x31\x47\xc7\x82\xd7\x67\x47\x83\x31", + "\xe5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x07" + "\x71\xbc\x8f\x1e\x4a\xa2\x35\x08\x77\x12\xd9\xdf\x18\x36\x09\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x5c\xdf\xaf" + "\x41\xf5\xaf\xb0\xe6\x53\xab\xce\xf3\x85\x23\x2d\x49\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x5c\xdf\xaf\x41\xf5" + "\xaf\xb0\xe6\x53\xab\xce\xf3\x85\x23\x2d\x49", + "\xd7\x92\x66\xcd\x25\xa7\x84\x59\x9a\x0a\x8e\x31\xfc\x84\xd6\x04", 1, + 32, 768 }, + { 96, 256, 128, 228, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf7\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x42" + "\x51\xcd\x29\xb0\xaa\xa9\x60\x55\x7c\x9e\xa2\x82\x83\x34\xe4\xe4\xe2" + "\x31\xdb\x0a\x27\xfa\xc9\xec\x9e\x74\x48\x86\xeb\x01\x33\xc5\x23\x21" + "\x42\xdd\xf4\x8b\x3f\x18\x51\x40\xf0\xfc\x05\xf0\x43", + "\xd4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x31" + "\x96\x1e\x26\x3e\x21\x42\xb1\x0f\xe7\xf3\x5b\x6f\xe5\xb0\x07\x35\x25" + "\x62\x86\xb6\x53\x5d\xbb\x47\x38\xc2\x89\xee\xf3\x04\xff\x35\x25\x62" + "\x86\xb6\x53\x5d\xbb\x47\x38\xc2\x89\xee\xf3\x04\xff", + "\x9d\x67\x1d\x40\x7d\x76\x60\x45\x9d\x5d\x58\x2d\x83\x91\x5e\xfe", 1, + 32, 512 }, + { 96, 256, 128, 229, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xf5\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x73" + "\xbd\x9f\x01\xbf\x33\x31\xb1\x2e\x31\xdd\x14\xcf\x11\xfe\xee\x1d\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x62\x5c\x69" + "\x65\xf6\x1a\x1c\x36\x11\x8c\x74\x70\x76\xd5\xb7\xb7\x62\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x6e\x56\x62\x6b\xd5" + "\x76\x61\x65\x5a\x89\xba\x2d\x86\x47\xed\x73", + "\xd6\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00" + "\x7a\x4c\x0e\x31\xb8\xda\x60\x74\xaa\xb0\xed\x22\x77\x7a\x0d\xcc\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x92\x5a\x2a" + "\xa1\x9d\xbd\xca\xb2\x4e\xe5\xf6\x09\x64\x23\x43\x0b\xcc\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x92\x5a\x2a\xa1\x9d" + "\xbd\xca\xb2\x4e\xe5\xf6\x09\x64\x23\x43\x0b", + "\x7b\x20\x7c\x2c\x32\x78\xc6\x4f\x0d\x6b\x91\x3f\xe3\x71\xfe\x63", 1, + 32, 768 }, + { 96, 256, 128, 230, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xec" + "\x09\x33\xf0\xbf\xb9\x12\x18\xce\xa0\xd7\x4e\x06\x1f\x55\x9e\x2d\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x8d\x5b\x67" + "\xe0\xac\xee\x53\x4c\xe2\xd9\x79\x14\x87\xb1\xec\xb2\x52\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x81\x51\x6c\xee\x8f" + "\x82\x2e\x1f\xa9\xdc\xb7\x49\x77\x23\xb6\x76", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9f" + "\xce\xe0\xff\x31\x32\xf9\xc9\x94\x3b\xba\xb7\xeb\x79\xd1\x7d\xfc\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7d\x5d\x24" + "\x24\xc7\x49\x85\xc8\xbd\xb0\xfb\x6d\x95\x47\x18\x0e\xfc\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7d\x5d\x24\x24\xc7" + "\x49\x85\xc8\xbd\xb0\xfb\x6d\x95\x47\x18\x0e", + "\x36\x72\x16\x2b\xb1\xf3\xff\x53\x7e\xce\x01\x3f\x1a\xca\x4f\x68", 1, + 32, 768 }, + { 96, 256, 128, 231, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xee" + "\x83\xa1\x4f\x48\xdb\x69\x62\x91\x08\x0e\xdf\xcc\x89\x8b\x88\x2b\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x8a\xd5\xf6" + "\xb0\x28\x3a\x8b\x39\xeb\xed\xce\x92\x78\x5d\xa9\xb6\x54\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x86\xdf\xfd\xbe\x0b" + "\x56\xf6\x6a\xa0\xe8\x00\xcf\x88\xcf\xf3\x72", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9d" + "\x44\x72\x40\xc6\x50\x82\xb3\xcb\x93\x63\x26\x21\xef\x0f\x6b\xfa\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7a\xd3\xb5" + "\x74\x43\x9d\x5d\xbd\xb4\x84\x4c\xeb\x6a\xab\x5d\x0a\xfa\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7a\xd3\xb5\x74\x43" + "\x9d\x5d\xbd\xb4\x84\x4c\xeb\x6a\xab\x5d\x0a", + "\x35\x72\x16\x3b\x99\x28\x4f\x5f\x3e\x4a\xa9\x4d\xba\xb8\x56\x77", 1, + 32, 768 }, + { 96, 256, 128, 232, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xe8" + "\x7d\xd0\x8e\xd4\xe4\xe0\x4c\x58\x77\x61\x6c\xbb\x02\xca\xbb\x29\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x87\x4f\x04" + "\x01\xd4\x57\xe3\x36\xf4\x31\x1f\x11\x52\xf9\x57\xba\x56\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x8b\x45\x0f\x0f\xf7" + "\x3b\x9e\x65\xbf\x34\xd1\x4c\xa2\x6b\x0d\x7e", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9b" + "\xba\x03\x81\x5a\x6f\x0b\x9d\x02\xec\x0c\x95\x56\x64\x4e\x58\xf8\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x77\x49\x47" + "\xc5\xbf\xf0\x35\xb2\xab\x58\x9d\x68\x40\x0f\xa3\x06\xf8\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x77\x49\x47\xc5\xbf" + "\xf0\x35\xb2\xab\x58\x9d\x68\x40\x0f\xa3\x06", + "\x34\x72\x16\x4b\x81\x5d\x9e\x6a\xfe\xc5\x50\x5c\x5a\xa7\x5d\x86", 1, + 32, 768 }, + { 96, 256, 128, 233, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc8\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x6b" + "\xe4\x36\xe3\x46\xf8\xf2\xb3\x2f\x4c\xbb\xae\xf9\x51\x50\xef\x04\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x2f\xb7\x6b" + "\x51\x32\xe9\x30\xf6\xd0\xac\xf7\x08\x75\xe9\x77\xb5\x7b\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x23\xbd\x60\x5f\x11" + "\x85\x4d\xa5\x9b\xa9\x39\x55\x85\x7b\x2d\x71", + "\xeb\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x18" + "\x23\xe5\xec\xc8\x73\x19\x62\x75\xd7\xd6\x57\x14\x37\xd4\x0c\xd5\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xdf\xb1\x28" + "\x95\x59\x4e\xe6\x72\x8f\xc5\x75\x71\x67\x1f\x83\x09\xd5\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xdf\xb1\x28\x95\x59" + "\x4e\xe6\x72\x8f\xc5\x75\x71\x67\x1f\x83\x09", + "\x3a\x72\x16\xd7\xee\x1d\xa0\x18\xce\x84\x12\xf2\x51\x65\x6b\x19", 1, + 32, 768 }, + { 96, 256, 128, 234, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xc5\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x78" + "\x3c\xf9\x30\x2c\x7d\x22\x91\x4b\x38\xac\xa2\xe7\xd3\x74\xef\x1d\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x22\x8f\x2d" + "\x23\x59\x76\x40\xd5\x74\xf8\xe2\x0c\x4f\x6b\x6b\xb5\x62\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x2e\x85\x26\x2d\x7a" + "\x1a\x3d\x86\x3f\xfd\x2c\x51\xbf\xf9\x31\x71", + "\xe6\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x0b" + "\xfb\x2a\x3f\xa2\xf6\xc9\x40\x11\xa3\xc1\x5b\x0a\xb5\xf0\x0c\xcc\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd2\x89\x6e" + "\xe7\x32\xd1\x96\x51\x2b\x91\x60\x75\x5d\x9d\x9f\x09\xcc\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd2\x89\x6e\xe7\x32" + "\xd1\x96\x51\x2b\x91\x60\x75\x5d\x9d\x9f\x09", + "\x36\x72\x16\x17\x8f\xf1\xdc\x45\xce\x73\xb0\x2c\xd2\x1f\x87\x55", 1, + 32, 768 }, + { 96, 256, 128, 235, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x5d" + "\xb7\x2f\x89\xd1\x40\x2b\x1a\x03\x73\xff\x0a\x9c\x5c\xd4\x4b\x6d\x67" + "\xaf\x40\x79\x8f\x54\x55\x50\x17\x92\x95\x32\x48\xec\x23\x4c\xa6\xbf" + "\xd9\xae\x5c\x25\xa3\xa4\xd8\xa6\x2d\x48\xa6\x1d\x53", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2e" + "\x70\xfc\x86\x5f\xcb\xc0\xcb\x59\xe8\x92\xf3\x71\x3a\x50\xa8\xbc\xa0" + "\xfc\x1d\xc5\xfb\xf3\x27\xfb\xb1\x24\x54\x5a\x50\xe9\xef\xbc\xa0\xfc" + "\x1d\xc5\xfb\xf3\x27\xfb\xb1\x24\x54\x5a\x50\xe9\xef", + "\x0b\x49\x61\xc9\x52\x5e\xa2\xf2\xcd\xad\x62\x73\xe1\xc7\x82\x4c", 1, + 32, 512 }, + { 96, 256, 128, 236, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\x5f" + "\x21\x5e\xc8\x7d\x62\xa2\x64\xca\xdb\x51\x9b\x4a\xc9\x0a\x76\x68\xd1" + "\xdd\x03\xe5\x6e\xda\x63\x99\xac\x78\x03\xe7\xdd\x22\x11\x49\x10\xcd" + "\x9a\x32\xbd\xab\x95\x6d\x63\x4c\xbb\x9d\x33\xd3\x61", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2c" + "\xe6\x8d\xc7\xf3\xe9\x49\xb5\x90\x40\x3c\x62\xa7\xaf\x8e\x95\xb9\x16" + "\x8e\x5e\x59\x1a\x7d\x11\x32\x0a\xce\xc2\x8f\xc5\x27\xdd\xb9\x16\x8e" + "\x5e\x59\x1a\x7d\x11\x32\x0a\xce\xc2\x8f\xc5\x27\xdd", + "\x0a\x49\x61\xd9\x3a\x93\xf1\xfd\x8d\x29\x0a\x82\x81\xb6\x89\x5b", 1, + 32, 512 }, + { 96, 256, 128, 237, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x10\xab\xb1\x65", + "\xff\xff\xff\xff", + "\xdc\x8c\xe7\x08\xbf\x26\xaa\xb8\x62\xd9\x7e\x1b\x42\xf3\x1e\xf3\xd1" + "\x5a\xd5\x90\xdd\x0f\x40\xba\x18\xac\xd1\x68\xf6\xac\x77\x7a\x0f\x38" + "\xac\xa2\x43\x8b\x58\x8d\x54\x59\x49\x3e\x97\xe7\xfa\x33\x93\x2a\x09" + "\x7f\x1d\x39\xa0\x4a\xd3\x0f\x1b\x6c\x65\x02\x60\xbf\x70\x03\x47\x8f" + "\x28\x69\xb9\x3e\xe2\x9c\x83\x7e\x95\xfb\x6b\x99\x9f\x20\x02\x71\x3e" + "\x55\xdd\x19\x98\x0a\xd5\x31\x95\x90\x3a\x7b", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa2" + "\x9d\x06\x9f\x53\x84\xab\x6b\x42\x37\xbc\x91\x1b\xca\xf3\x99\xde\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x63\x2c\x4a" + "\xbb\x76\x9e\x76\xce\x8c\x66\x99\x15\x77\xf4\x94\x03\xde\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x63\x2c\x4a\xbb\x76" + "\x9e\x76\xce\x8c\x66\x99\x15\x77\xf4\x94\x03", + "\x35\x72\x16\x13\x55\x24\x09\x43\xde\x94\x06\x29\x2a\x64\xc5\x51", 1, + 32, 768 }, + { 96, 256, 128, 238, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x40\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x39" + "\x0e\xf9\x3a\xeb\x61\xaa\x30\x7f\x14\x13\x23\xc3\x8e\x06\x85\xfa\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x25\x94\x45" + "\xf4\xff\xc3\x1b\xce\x54\x01\x90\xed\xd6\xad\x20\x78\x76\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xed\xa1\xa5\xb6\x13" + "\x97\x50\xf9\x73\xf0\xd4\x84\x1b\xaa\x2c\xb8", + "\xd9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa0" + "\x09\xd7\x3c\x65\x44\x42\x8c\xfa\xc0\xb2\xd8\xc7\xbb\xef\x0b\xed\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x8a\x5e\xf6" + "\x07\x15\xbc\x4b\x07\xc9\x2b\x97\x07\x37\x6d\xa1\x05\xed\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x8a\x5e\xf6\x07\x15" + "\xbc\x4b\x07\xc9\x2b\x97\x07\x37\x6d\xa1\x05", + "\x19\x53\x2d\x9f\xa0\xb5\xfb\xd5\x82\xaa\xed\xa8\x30\x60\x2f\x1d", 1, + 32, 768 }, + { 96, 256, 128, 239, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x49\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\xe0" + "\x2b\x87\xae\xae\x8c\x3d\xa8\x89\x5f\x8c\xb0\xf6\xb9\xcc\x80\xf4\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\xcc\x4b\x7b" + "\x80\x3a\x5f\x8f\x46\x47\xdf\x16\x90\x80\xfe\x56\x7a\x78\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x04\x7e\x9b\xc2\xd6" + "\x0b\xc4\x71\x60\x2e\x52\xf9\x4d\xf9\x5a\xba", + "\xd0\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x79" + "\x2c\xa9\xa8\x20\xa9\xd5\x14\x0c\x8b\x2d\x4b\xf2\x8c\x25\x0e\xe3\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x63\x81\xc8" + "\x73\xd0\x20\xdf\x8f\xda\xf5\x11\x7a\x61\x3e\xd7\x07\xe3\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x63\x81\xc8\x73\xd0" + "\x20\xdf\x8f\xda\xf5\x11\x7a\x61\x3e\xd7\x07", + "\xad\xbd\x2c\xaf\xc8\xc8\xf0\xe5\x12\x50\xe7\xb8\x1c\x9d\x0a\x2d", 1, + 32, 768 }, + { 96, 256, 128, 240, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x43\xea\xda\xe0\x36\xf7\x33\xea\x9b\x5b\x7e\xb2\x2a\xee\x39\x5d\xb6" + "\xf5\x1a\x4d\x10\xbc\x24\x60\x81\x0c\x22\x96\x51\x55\x6a\xcf\x38\x4a" + "\xd8\x2e\x3e\x28\x0c\xad\x69\xf0\xdf\x25\xb4\x2b\x83\xb0", + "\xda\x04\x7b\x78\x25\xdb\x18\x02\xe8\xe8\xe1\xaa\xc6\xba\x88\xfc\x2f" + "\xf2\x34\x4b\x9e\x99\xcc\xdc\x04\xd8\x83\x6d\x55\x60\x83\x41\x2f\xf2" + "\x34\x4b\x9e\x99\xcc\xdc\x04\xd8\x83\x6d\x55\x60\x83\x41", + "\x97\x3e\x27\x0a\x7a\xfc\xab\x75\x34\x8e\x14\xdb\xe1\x9c\x51\x56", 1, + 32, 384 }, + { 96, 256, 128, 241, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x66\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x89" + "\x1b\x79\x75\x21\xba\x92\x5b\x24\x09\x0a\xaf\x6c\x44\x82\xba\xe8\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x6d\x50\xc3" + "\x2d\x05\xa9\x46\xcb\x8c\xea\x57\xc9\xf1\x44\x2c\xb1\x64\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xa5\x65\x23\x6f\xe9" + "\xfd\x0d\xfc\xab\x1b\x13\xa0\x3c\x43\x20\x71", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x10" + "\x1c\x57\x73\xaf\x9f\x7a\xe7\xa1\xdd\xab\x54\x68\x71\x6b\x34\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc2\x9a\x70" + "\xde\xef\xd6\x16\x02\x11\xc0\x50\x23\x10\x84\xad\xcc\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc2\x9a\x70\xde\xef" + "\xd6\x16\x02\x11\xc0\x50\x23\x10\x84\xad\xcc", + "\xe1\x7c\x27\x3f\x31\x75\x8e\x75\x23\x22\xae\x48\x69\xc1\xbf\xbb", 1, + 32, 768 }, + { 96, 256, 128, 242, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x6a\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x51" + "\x9c\xcc\xeb\xf7\x25\x73\xdb\xee\x8c\x12\xf7\x42\x55\xd1\x8c\x0a\xdd" + "\x10\x35\x86\x1f\xfc\x0b\x7f\x40\x07\x9b\x96\x9f\x8c\x63\xb2\xaf\x4f" + "\xa3\xcc\xd1\x6c\xb3\x8f\x42\x5c\x39\x96\x14\x0d\xef", + "\xf3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc8" + "\x9b\xe2\xed\x79\x00\x9b\x67\x6b\x58\xb3\x0c\x46\x60\x38\x02\x1d\x65" + "\xfc\x50\x26\xae\x3c\x7a\x12\x68\x5b\xd3\x77\xd4\x8c\x92\x1d\x65\xfc" + "\x50\x26\xae\x3c\x7a\x12\x68\x5b\xd3\x77\xd4\x8c\x92", + "\xa2\x23\x90\x22\x4c\x5d\xb0\xf0\x16\x96\x74\x3d\x87\x07\x25\xc5", 1, + 32, 512 }, + { 96, 256, 128, 243, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\xe2\x35\xb8\xc2\x13\x84\x55\x70\x85\xc3\xf2\xeb\x2a\x8f\xa3\x60\x58" + "\xcf\xfd\x2a\xf7\x43\xda\xcf\x96\xb4\xae\x4d\x51\xb4\xe4\x88\xd6\x70" + "\x3f\x49\xd9\xd7\xf2\x02\x7e\x48\x53\xfe\xb4\xca\x0d\xf7", + "\x7b\xdb\x19\x5a\x00\xa8\x7e\x98\xf6\x70\x6d\xf3\xc6\xdb\x12\xc1\xc1" + "\xc8\xd3\x2c\x79\x66\x32\x73\x13\x60\x0f\xb6\x55\x81\x0d\x06\xc1\xc8" + "\xd3\x2c\x79\x66\x32\x73\x13\x60\x0f\xb6\x55\x81\x0d\x06", + "\x43\x7d\x1e\xfa\xd2\x1b\x08\x65\xa5\x41\xb5\xca\xb6\x2e\x2a\x44", 1, + 32, 384 }, + { 96, 256, 128, 244, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x66\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x8f" + "\xab\x58\x57\x4a\x32\x2b\xac\x6f\x39\x44\x74\xe4\xce\x7e\xae\xc3\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x71\x53\x2d" + "\xfb\x0e\x91\x41\xb0\x09\x83\x39\x47\x22\x82\x9e\x7c\x4f\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xb9\x66\xcd\xb9\xe2" + "\xc5\x0a\x87\x2e\x72\x7d\x2e\xef\x85\x92\xbc", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x16" + "\xac\x76\x51\xc4\x17\xc3\x10\xea\xed\xe5\x8f\xe0\xfb\x97\x20\xd4\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xde\x99\x9e" + "\x08\xe4\xee\x11\x79\x94\xa9\x3e\xad\xc3\x42\x1f\x01\xd4\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xde\x99\x9e\x08\xe4" + "\xee\x11\x79\x94\xa9\x3e\xad\xc3\x42\x1f\x01", + "\xac\xf4\xff\xa2\x0c\x0d\x06\xd6\x1a\x18\xe9\xa8\xd4\xc8\x4d\x1d", 1, + 32, 768 }, + { 96, 256, 128, 245, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x61\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x5e" + "\xfe\x67\x9b\xa1\x73\x84\xc5\x5e\xb8\xcc\x19\x36\x66\xfe\x8d\x04\x60" + "\x8c\x35\x03\xd2\x17\xaa\x3f\x90\xa9\xb0\xe1\xb3\xb3\x13\xbc\x12\xd3" + "\xa3\x49\x1c\x87\x12\xcf\x92\xf2\x12\xe1\x38\x32\x9f", + "\xf8\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc7" + "\xf9\x49\x9d\x2f\x56\x6c\x79\xdb\x6c\x6d\xe2\x32\x53\x17\x03\x13\xd8" + "\x60\x50\xa3\x63\xd7\xdb\x52\xb8\xf5\xf8\x00\xf8\xb3\xe2\x13\xd8\x60" + "\x50\xa3\x63\xd7\xdb\x52\xb8\xf5\xf8\x00\xf8\xb3\xe2", + "\xcd\x46\x6d\x06\xe7\x5b\x7f\xd1\x8d\x5f\xe2\x1d\x92\x27\xd9\xa7", 1, + 32, 512 }, + { 96, 256, 128, 246, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x90\x64\xb8\x8a\x28\x20\x52\xa1\xee\x44\xdf\x05\xad\x21\x3d\xa6\x79" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\x51\xb2" + "\x65\x0e\xc9\x45\xfe\xc7\x05\x88\xbc\x65\xa6\x16\xa5\xf2\x4f\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\xdd\x55\x7e\xc8" + "\xa4\xd6\x3d\xf7\x27\x45\x94\x36\x7b\xef\x09\xcd", + "\x09\x8a\x19\x12\x3b\x0c\x79\x49\x9d\xf7\x40\x1d\x41\x75\x8c\x07\xe0" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x46\x0a" + "\x89\x6b\x69\xf4\x3e\xb6\x68\xa0\xe0\x2d\x47\x5d\xa5\x03\xe0\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x46\x0a\x89\x6b" + "\x69\xf4\x3e\xb6\x68\xa0\xe0\x2d\x47\x5d\xa5\x03", + "\xce\x8a\x3d\x4d\x88\x7d\x95\x61\x3d\x82\x9b\x53\x8e\xd0\x11\x96", 1, + 32, 640 }, + { 96, 256, 128, 247, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x43\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\xee" + "\xf6\x7b\xd4\x79\x5b\x74\x01\x5a\x34\x93\x90\x5d\x54\x4a\x86\xe8\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x31\x97\xbe" + "\x28\xef\xf8\x43\x59\x2b\xd8\xfc\x8d\x57\x84\x21\xd6\x64\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xf9\xa2\x5e\x6a\x03" + "\xac\x08\x6e\x0c\x29\xb8\xe4\x9a\x83\x2d\x16", + "\xda\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x77" + "\xf1\x55\xd2\xf7\x7e\x9c\xbd\xdf\xe0\x32\x6b\x59\x61\xa3\x08\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9e\x5d\x0d" + "\xdb\x05\x87\x13\x90\xb6\xf2\xfb\x67\xb6\x44\xa0\xab\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9e\x5d\x0d\xdb\x05" + "\x87\x13\x90\xb6\xf2\xfb\x67\xb6\x44\xa0\xab", + "\x08\x28\x9f\x51\x99\xdf\x47\x6f\xe9\x04\x75\xcb\x95\x22\x55\x66", 1, + 32, 768 }, + { 96, 256, 128, 248, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x6b\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x1e" + "\x34\x41\x2a\xb0\xa0\x56\xe8\x09\xd5\xd4\xb9\x2b\xe1\x12\x8a\x4b\x2a" + "\x65\x1a\x62\xae\xab\x26\xcf\x43\x7f\xb1\x95\x40\x75\x74\xf3\x58\x3a" + "\x8c\x28\x60\x3b\x9e\x3f\x41\x24\x13\x95\xcb\xf4\xf8", + "\xf2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x87" + "\x33\x6f\x2c\x3e\x85\xbe\x54\x8c\x01\x75\x42\x2f\xd4\xfb\x04\x5c\x92" + "\x89\x7f\xc2\x1f\x6b\x57\xa2\x6b\x23\xf9\x74\x0b\x75\x85\x5c\x92\x89" + "\x7f\xc2\x1f\x6b\x57\xa2\x6b\x23\xf9\x74\x0b\x75\x85", + "\x06\xdf\x93\xf6\x51\xea\x5c\xc5\x69\x11\xf3\x0d\x3e\x58\xf9\x97", 1, + 32, 512 }, + { 96, 256, 128, 249, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x3f\xe6\x06\x10\x8f\x35\x86\x9d\xf4\xc7\xaa\x01\x28\x46\x4a\x12\x65" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xfd\xbe" + "\x84\x3a\x0a\xd9\xbe\x25\x05\x59\x92\xab\x6d\xcb\xc9\xf1\x53\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x71\x59\x9f\xfc" + "\x67\x4a\x7d\x15\x27\x94\xba\xf8\xb0\x32\x65\xce", + "\xa6\x08\xa7\x88\x9c\x19\xad\x75\x87\x74\x35\x19\xc4\x12\xfb\xb3\xfc" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xea\x06" + "\x68\x5f\xaa\x68\x7e\x54\x68\x71\xce\xe3\x8c\x80\xc9\x00\xfc\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xea\x06\x68\x5f" + "\xaa\x68\x7e\x54\x68\x71\xce\xe3\x8c\x80\xc9\x00", + "\x92\x64\xfc\x0f\x47\xfe\xbb\x30\x66\x12\x54\xda\xf9\xa0\x61\x89", 1, + 32, 640 }, + { 96, 256, 128, 250, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x6e\x8e\xb9\x8c\xf7\xff\xfe\x4c\xd6\x83\x56\x8c\xf8\x92\x99\x15\x64" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xc7\x0f" + "\x5d\x8b\x30\xc6\x4b\xf2\xe6\xd1\xd6\x13\xf4\x0e\x0b\xf0\x52\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x4b\xe8\x46\x4d" + "\x5d\x55\x88\xc2\xc4\x1c\xfe\x40\x29\xf7\xa7\xcf", + "\xf7\x60\x18\x14\xe4\xd3\xd5\xa4\xa5\x30\xc9\x94\x14\xc6\x28\xb4\xfd" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd0\xb7" + "\xb1\xee\x90\x77\x8b\x83\x8b\xf9\x8a\x5b\x15\x45\x0b\x01\xfd\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd0\xb7\xb1\xee" + "\x90\x77\x8b\x83\x8b\xf9\x8a\x5b\x15\x45\x0b\x01", + "\x69\xa1\x24\xfc\x7f\x96\xe2\x20\xd1\xa0\x31\xce\xd5\x52\x72\x79", 1, + 32, 640 }, + { 96, 256, 128, 251, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x4f\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x41" + "\x56\x26\x9f\xe3\xda\x10\x1e\xeb\x0a\xbf\x8d\xda\x20\xfe\x8f\xff\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x6a\xec\xe9" + "\x83\xe6\x4f\x97\xe4\x3f\xf5\x29\x5b\xc8\x84\xfa\x77\x73\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xa2\xd9\x09\xc1\x0a" + "\x1b\xdc\xd3\x18\x04\x6d\x32\x05\x83\xf6\xb7", + "\xd6\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd8" + "\x51\x08\x99\x6d\xff\xf8\xa2\x6e\xde\x1e\x76\xde\x15\x17\x01\xe8\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc5\x26\x5a" + "\x70\x0c\x30\xc7\x2d\xa2\xdf\x2e\xb1\x29\x44\x7b\x0a\xe8\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc5\x26\x5a\x70\x0c" + "\x30\xc7\x2d\xa2\xdf\x2e\xb1\x29\x44\x7b\x0a", + "\x3e\xa8\xf9\xb2\x01\x23\x21\xe6\x3d\x5f\xb5\xbc\x2c\x5d\x33\x2d", 1, + 32, 768 }, + { 96, 256, 128, 252, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x66\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x18" + "\xf1\x25\xef\x37\x4c\x14\x54\xb6\x80\xe2\x34\x27\xe7\xdc\x69\xe4\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x85\x8b\x08" + "\xeb\x1d\x58\x15\x70\xa7\xcd\x1e\x48\x59\x3b\x75\x75\x68\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x4d\xbe\xe8\xa9\xf1" + "\x0c\x5e\x47\x80\x3c\x5a\x21\x94\x3c\x79\xb5", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x81" + "\xf6\x0b\xe9\xb9\x69\xfc\xe8\x33\x54\x43\xcf\x23\xd2\x35\xe7\xf3\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2a\x41\xbb" + "\x18\xf7\x27\x45\xb9\x3a\xe7\x19\xa2\xb8\xfb\xf4\x08\xf3\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2a\x41\xbb\x18\xf7" + "\x27\x45\xb9\x3a\xe7\x19\xa2\xb8\xfb\xf4\x08", + "\xdf\xaf\x8a\x3a\x15\xd4\x5e\x7f\x4c\x34\x30\x04\x8d\x85\x89\xf0", 1, + 32, 768 }, + { 96, 256, 128, 253, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\xb0\x2a\xb7\x47\xa3\x10\xd6\xa3\xbb\xdb\x97\x01\x8a\x3b\xe8\xb3\x41" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xb7\xa3" + "\x38\xbc\x34\x23\x89\x5f\x0f\xd9\x6c\xdb\x27\xa7\x87\xf2\x77\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x3b\x44\x23\x7a" + "\x59\xb0\x4a\x6f\x2d\x14\x44\x88\xfa\x5e\x2b\xcd", + "\x29\xc4\x16\xdf\xb0\x3c\xfd\x4b\xc8\x68\x08\x19\x66\x6f\x59\x12\xd8" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa0\x1b" + "\xd4\xd9\x94\x92\x49\x2e\x62\xf1\x30\x93\xc6\xec\x87\x03\xd8\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa0\x1b\xd4\xd9" + "\x94\x92\x49\x2e\x62\xf1\x30\x93\xc6\xec\x87\x03", + "\x34\x08\xeb\x2b\x13\xa9\xb7\x6b\xef\xce\xdf\x69\x94\x22\xd6\x1f", 1, + 32, 640 }, + { 96, 256, 128, 254, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x40\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x38" + "\x0e\xf9\x3a\xeb\x61\xaa\x30\x7f\x14\x13\x23\xc3\x8e\x06\x85\xf6\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x3f\x76\x9a" + "\x30\xe8\x95\x1f\xf2\xfb\x36\x5f\xa7\x80\xfd\xde\x7e\x7a\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xf7\x43\x7a\x72\x04" + "\xc1\x54\xc5\xdc\xc7\x1b\xce\x4d\xfa\xd2\xbe", + "\xd9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa1" + "\x09\xd7\x3c\x65\x44\x42\x8c\xfa\xc0\xb2\xd8\xc7\xbb\xef\x0b\xe1\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x90\xbc\x29" + "\xc3\x02\xea\x4f\x3b\x66\x1c\x58\x4d\x61\x3d\x5f\x03\xe1\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x90\xbc\x29\xc3\x02" + "\xea\x4f\x3b\x66\x1c\x58\x4d\x61\x3d\x5f\x03", + "\x09\xf4\xf2\xa3\x93\x6d\x74\x61\xa6\x7c\xe0\x22\x17\x6b\xb8\xdd", 1, + 32, 768 }, + { 96, 256, 128, 255, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x40\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x06" + "\x0e\xf9\x3a\xeb\x61\xaa\x30\x7f\x14\x13\x23\xc3\x8e\x06\x85\xee\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x2b\xca\x70" + "\xbf\xcd\xf1\x17\x1a\xb6\x11\xd1\x2b\xed\x5d\x62\x7a\x62\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xe3\xff\x90\xfd\x21" + "\xa5\x5c\x2d\x91\xe0\x95\x42\x20\x5a\x6e\xba", + "\xd9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x9f" + "\x09\xd7\x3c\x65\x44\x42\x8c\xfa\xc0\xb2\xd8\xc7\xbb\xef\x0b\xf9\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x84\x00\xc3" + "\x4c\x27\x8e\x47\xd3\x2b\x3b\xd6\xc1\x0c\x9d\xe3\x07\xf9\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x84\x00\xc3\x4c\x27" + "\x8e\x47\xd3\x2b\x3b\xd6\xc1\x0c\x9d\xe3\x07", + "\x2e\xb2\x67\x9a\xad\xfd\x82\x4a\x5f\xd8\xfa\x2e\x4a\x55\xa6\x5c", 1, + 32, 768 }, + { 96, 256, 128, 256, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x56\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x6c" + "\x7e\x13\x12\xc6\x77\x4f\xae\x7d\x1e\x5d\x0c\xc6\x09\x02\x8f\xf5\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x81\xc9\xe6" + "\x1c\xbe\xee\xd5\x54\x6b\x1c\xe5\xd8\xfe\xf2\x1a\x7a\x79\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x49\xfc\x06\x5e\x52" + "\xba\x9e\x63\x4c\xed\xa1\xb1\x33\xf5\x16\xba", + "\xcf\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf5" + "\x79\x3d\x14\x48\x52\xa7\x12\xf8\xca\xfc\xf7\xc2\x3c\xeb\x01\xe2\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2e\x03\x55" + "\xef\x54\x91\x85\x9d\xf6\x36\xe2\x32\x1f\x32\x9b\x07\xe2\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2e\x03\x55\xef\x54" + "\x91\x85\x9d\xf6\x36\xe2\x32\x1f\x32\x9b\x07", + "\x5e\x89\x34\x9f\x6b\x01\x1c\xd6\xe2\x4e\xe6\xac\x2f\x59\x0c\x21", 1, + 32, 768 }, + { 96, 256, 128, 257, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x2e\xa8\x41\x0b\x4d\xca\x8c\x9d\x53\x69\xa0\x33\xd8\xdb\x61\xe4\x6c" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xf0\xf5" + "\x8e\x8b\xba\x6c\xf1\xa5\x21\x46\x27\x3d\x8f\xe0\xc4\xfc\x5a\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x7c\x12\x95\x4d" + "\xd7\xff\x32\x95\x03\x8b\x0f\x6e\x52\x19\x68\xc3", + "\xb7\x46\xe0\x93\x5e\xe6\xa7\x75\x20\xda\x3f\x2b\x34\x8f\xd0\x45\xf5" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe7\x4d" + "\x62\xee\x1a\xdd\x31\xd4\x4c\x6e\x7b\x75\x6e\xab\xc4\x0d\xf5\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe7\x4d\x62\xee" + "\x1a\xdd\x31\xd4\x4c\x6e\x7b\x75\x6e\xab\xc4\x0d", + "\xb2\x45\x37\xfc\xb0\xdc\xb6\x20\x0b\x02\x85\xca\xfc\x9c\x3a\x7d", 1, + 32, 640 }, + { 96, 256, 128, 258, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x17\x05\x9a\x7c\x88\x83\xa2\x8b\x90\xbd\x94\xae\x44\xd1\x54\x36\x62" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xa2\x30" + "\x18\xbf\x8e\x68\xe4\x13\xe9\x9a\xc2\xd4\xab\x3f\x8d\xf1\x54\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x2e\xd7\x03\x79" + "\xe3\xfb\x27\x23\xcb\x57\xea\x87\x76\xc6\x21\xce", + "\x8e\xeb\x3b\xe4\x9b\xaf\x89\x63\xe3\x0e\x0b\xb6\xa8\x85\xe5\x97\xfb" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb5\x88" + "\xf4\xda\x2e\xd9\x24\x62\x84\xb2\x9e\x9c\x4a\x74\x8d\x00\xfb\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb5\x88\xf4\xda" + "\x2e\xd9\x24\x62\x84\xb2\x9e\x9c\x4a\x74\x8d\x00", + "\x43\x30\x04\x00\xea\x36\xe7\x20\x36\x11\x53\xce\x0c\x5d\x63\x7d", 1, + 32, 640 }, + { 96, 256, 128, 259, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\xaa\xa1\xb2\x58\xfd\x4b\x54\xb4\x97\xb5\x20\x80\x6a\x66\xd7\xaa\x68" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\x99\x13" + "\x2a\x23\x4a\x8c\x78\x9b\xf8\x54\x45\x47\x94\x0e\xc3\xf3\x5e\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x15\xf4\x31\xe5" + "\x27\x1f\xbb\xab\xda\x99\x6d\x14\x49\xf7\x6f\xcc", + "\x33\x4f\x13\xc0\xee\x67\x7f\x5c\xe4\x06\xbf\x98\x86\x32\x66\x0b\xf1" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x8e\xab" + "\xc6\x46\xea\x3d\xb8\xea\x95\x7c\x19\x0f\x75\x45\xc3\x02\xf1\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x8e\xab\xc6\x46" + "\xea\x3d\xb8\xea\x95\x7c\x19\x0f\x75\x45\xc3\x02", + "\xd7\x9a\x03\x10\x12\x4a\xdc\x30\xc6\xb6\x4c\xde\xf8\x99\x3e\x8d", 1, + 32, 640 }, + { 96, 256, 128, 260, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x4c\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\xbb" + "\x53\x57\xed\x31\x4a\xd7\x40\xb9\x91\x0f\xad\x6f\x01\xd7\x81\xf0\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\xc8\x04\x2b" + "\x41\x4f\xdd\x1b\xba\x3a\x6c\x93\x6b\x7e\xd6\x78\x79\x7c\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x00\x31\xcb\x03\xa3" + "\x89\x50\x8d\x1d\x9d\xd7\x02\xb3\xd1\x74\xb9", + "\xd5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x22" + "\x54\x79\xeb\xbf\x6f\x3f\xfc\x3c\x45\xae\x56\x6b\x34\x3e\x0f\xe7\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x67\xce\x98" + "\xb2\xa5\xa2\x4b\x73\xa7\x46\x94\x81\x9f\x16\xf9\x04\xe7\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x67\xce\x98\xb2\xa5" + "\xa2\x4b\x73\xa7\x46\x94\x81\x9f\x16\xf9\x04", + "\xe6\x02\x2c\xc3\xba\x20\xe3\xf9\x06\x5f\xdf\xcc\x43\xa9\xdc\x40", 1, + 32, 768 }, + { 96, 256, 128, 261, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x66\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\xf6" + "\x42\x96\x97\x5a\xf7\xfc\xed\x16\x81\x81\xf7\x6c\x65\x08\xe1\xc9\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x49\x75\x06" + "\x0f\x7d\xde\xf4\xa0\x98\x69\x93\x33\xb3\x0f\xbf\x7c\x45\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\x81\x40\xe6\x4d\x91" + "\x8a\xbf\x97\xbf\x98\xd7\x5a\x7e\x08\xb3\xbc", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x6f" + "\x45\xb8\x91\xd4\xd2\x14\x51\x93\x55\x20\x0c\x68\x50\xe1\x6f\xde\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe6\xbf\xb5" + "\xfc\x97\xa1\xa4\x69\x05\x43\x94\xd9\x52\xcf\x3e\x01\xde\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe6\xbf\xb5\xfc\x97" + "\xa1\xa4\x69\x05\x43\x94\xd9\x52\xcf\x3e\x01", + "\x35\x3e\x30\x4f\xd8\x55\x32\x86\xb2\x6e\x0d\x59\x94\x2f\xe7\xcd", 1, + 32, 768 }, + { 96, 256, 128, 262, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x98\x41\xcf\xc9\x27\xa5\x7d\xc4\x91\xab\x35\x42\x7f\xf9\x35\xe6\x6e" + "\xf8\xd1\xf9\x71\xda\x17\x43\x7a\x2b\x5e\x04\xfb\xca\x16\x71\xa6\x83" + "\xc8\xf9\xf9\xe6\x78\x0f\xda\x49\x40\xdd\xed\xd7\x6b\xf2\x58\x35\x4c" + "\x0c\x15\x80\xaf\x36\x62\xd5\xf8\x15\x1e\x3f\x7e\x82\x2a\x64\xd3\x3f" + "\x94\x75\xbb\x3f\xf8\x84\x68\x8e\x30\x2e\xc7\xcd", + "\x01\xaf\x6e\x51\x34\x89\x56\x2c\xe2\x18\xaa\x5a\x93\xad\x84\x47\xf7" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb1\x3b" + "\x24\x9c\x59\x57\xb8\x7e\xb7\x61\x1c\x95\x0c\x9c\x6b\x03\xf7\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb1\x3b\x24\x9c" + "\x59\x57\xb8\x7e\xb7\x61\x1c\x95\x0c\x9c\x6b\x03", + "\x0a\xeb\x04\xec\xf7\xde\xf4\x0c\x42\x02\x5b\xba\xe5\x50\x91\x69", 1, + 32, 640 }, + { 96, 256, 128, 263, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x42\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x0b" + "\x61\xbf\x9b\x7c\xaf\x83\xcc\x34\xda\x62\x55\x93\x51\x42\x89\xe8\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x69\x6a\x5c" + "\x7f\xb9\xda\x9c\xd4\xa3\x9c\x85\x91\x08\x6d\xb4\x2d\x64\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xa1\x5f\xbc\x3d\x55" + "\x8e\xd7\xe3\x84\x6d\xc1\xf8\xc5\x6a\xb8\xed", + "\xdb\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x92" + "\x66\x91\x9d\xf2\x8a\x6b\x70\xb1\x0e\xc3\xae\x97\x64\xab\x07\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc6\xa0\xef" + "\x8c\x53\xa5\xcc\x1d\x3e\xb6\x82\x7b\xe9\xad\x35\x50\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc6\xa0\xef\x8c\x53" + "\xa5\xcc\x1d\x3e\xb6\x82\x7b\xe9\xad\x35\x50", + "\x8f\xc4\xf7\x7a\x6e\xe0\x52\xa4\xc3\x14\x78\x0b\x8d\xf9\xa2\xd0", 1, + 32, 768 }, + { 96, 256, 128, 264, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x4b\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\xf2" + "\x8e\x4d\x0f\x20\xca\x16\x44\x47\x0c\x9c\xda\xc6\x00\x08\x87\xed\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x14\x64\x77" + "\x5b\xac\xd5\xc6\x9f\xe2\x6e\x1a\x74\x96\x8e\xa2\x7e\x61\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xdc\x51\x97\x19\x40" + "\x81\x8d\xa8\xc5\x9f\x5e\x1d\x5b\x89\xae\xbe", + "\xd2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x6b" + "\x89\x63\x09\xae\xef\xfe\xf8\xc2\xd8\x3d\x21\xc2\x35\xe1\x09\xfa\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xbb\xae\xc4" + "\xa8\x46\xaa\x96\x56\x7f\x44\x1d\x9e\x77\x4e\x23\x03\xfa\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xbb\xae\xc4\xa8\x46" + "\xaa\x96\x56\x7f\x44\x1d\x9e\x77\x4e\x23\x03", + "\x23\x2f\xf7\x8a\x96\xf3\x47\xb4\x53\xba\x71\x1b\x79\x36\x7e\xe0", 1, + 32, 768 }, + { 96, 256, 128, 265, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x2d\xd4\xcd\x40", + "\xff\xff\xff\xff", + "\x4d\x11\x5e\x67\xec\xd3\xd4\x17\x8c\x4c\x60\xe7\x13\xab\x4e\x5e\x6e" + "\xe6\x28\xfc\x4b\x58\x30\x18\x4c\xd2\x93\x36\x4a\x21\x3e\x84\xfe\x47" + "\x13\x9a\x5f\x4e\x3f\x8e\x92\xd7\xa3\xb7\x1e\xb4\xff\x0e\x29\xdb\x95" + "\x3a\xd5\x45\x8f\xea\x61\xf0\x13\xea\x18\x54\xfe\x75\x72\xa0\x08\x5c" + "\x32\xdd\xfc\xbe\xb0\x1a\x8b\xe4\xc3\x4d\x53\x31\xe1\xee\x75\x78\x39" + "\x11\xc4\xdd\x46\x01\x57\x83\xd5\x53\xf2\xb5", + "\xd4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf7" + "\xe1\x06\xfa\xc5\x7d\xd8\xa4\xc9\x06\x32\xcd\x4e\x14\xd7\x0a\xe9\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x86\x11\x26" + "\xc9\x3f\x3a\xdf\x23\xfc\xda\x14\x00\xf9\x94\x7f\x08\xe9\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x86\x11\x26\xc9\x3f" + "\x3a\xdf\x23\xfc\xda\x14\x00\xf9\x94\x7f\x08", + "\xe0\x0d\x2e\x8b\xae\x5d\x09\xc2\x8e\x9b\xf5\x94\x09\x54\x5d\x09", 1, + 32, 768 }, + { 96, 256, 128, 266, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x19\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xbc" + "\x28\x6f\xd9\x79\x80\x79\x51\xb1\x83\xa1\x88\x93\x0a\xd1\x5e\xdc\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x89\x0e\x65" + "\x9f\xd3\x02\x8c\x90\x4e\x65\x01\x8f\xdf\xd6\x03\x83\x33\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x33\xb7\xfb\x50\xc3" + "\xe7\xeb\xca\x97\x0f\x6f\x89\xa8\x8a\x82\xd6", + "\xf9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + "\x5d\x15\x65\x92\x4f\x6c\x74\x18\xde\x9b\xab\xf8\xbe\x44\x07\xed\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2e\x11\x0e" + "\x5e\x1c\x04\x68\xcb\xaa\xd9\x9c\x8a\xbe\xff\xff\x07\xed\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x2e\x11\x0e\x5e\x1c" + "\x04\x68\xcb\xaa\xd9\x9c\x8a\xbe\xff\xff\x07", + "\x47\xe5\xd4\x29\x42\x39\xdb\x73\xb8\x36\xc0\x40\x70\xff\x5b\x2d", 1, + 32, 768 }, + { 96, 256, 128, 267, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x83" + "\x9f\x81\x1a\xd0\x31\x0c\x77\x05\x2f\x45\x32\x0b\x0d\x95\x60\xc4\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x70\xd6\xb1" + "\x4f\xd2\x09\xfe\xdf\x26\x1f\xd1\xd2\x50\xd3\x47\x8d\x2b\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xca\x6f\x2f\x80\xc2" + "\xec\x99\x85\xff\x75\xbf\xd4\x27\x8f\xc6\xd8", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x3e" + "\xea\xfb\xa6\x3b\xfe\x19\x52\xac\x72\x7f\x11\x60\xb9\x00\x39\xf5\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd7\xc9\xda" + "\x8e\x1d\x0f\x1a\x84\xc2\xa3\x4c\xd7\x31\xfa\xbb\x09\xf5\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd7\xc9\xda\x8e\x1d" + "\x0f\x1a\x84\xc2\xa3\x4c\xd7\x31\xfa\xbb\x09", + "\x23\x2c\x88\x2f\x7a\x1a\x2f\x80\x8c\xcf\x26\x49\x6c\xff\x5b\x3d", 1, + 32, 768 }, + { 96, 256, 128, 268, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x97\x31\x1c\xd6\xe2\xd2\x5a\x7b\x4e\xaa\x16\xf0\xa6\x1c\xa6\x24\x6b" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\x95\x13" + "\x63\x10\xb6\xb6\xb5\xc1\x7c\x9f\x8c\x02\xba\x7d\x0a\xeb\x71\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\x7a\x32\x9e\xe1" + "\xa0\xaf\x16\x0f\xc7\x6d\x3d\xe7\xe9\x91\x02\xc3", + "\x77\x10\x78\xb7\xd5\x9f\xe2\x50\x9a\xeb\x0b\x0e\x34\x84\x4c\x61\xd6" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa4\x1c" + "\x2c\xb9\xeb\xa7\x86\x6f\x50\x68\x4b\x1b\x05\xe3\xab\x00\xd6\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xa4\x1c\x2c\xb9" + "\xeb\xa7\x86\x6f\x50\x68\x4b\x1b\x05\xe3\xab\x00", + "\xd7\x1b\xc7\x0d\x5a\xdc\x74\xe7\xdf\xd8\x94\x06\xfc\x15\xf0\x44", 1, + 32, 640 }, + { 96, 256, 128, 269, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x34\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x74" + "\xcf\x7e\x9d\x82\xb7\xe8\xed\x9e\xc9\x65\xf6\xea\x31\x09\x51\xdc\x10" + "\x49\x40\xe0\x8a\x42\x22\x55\x68\x28\xeb\xa4\x59\xf6\x5a\x4a\x00\x6d" + "\x28\x72\x9d\x95\xd7\x9d\x23\x72\xf7\x7a\xee\xab\x35", + "\xd4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc9" + "\xba\x04\x21\x69\x78\xfd\xc8\x37\x94\x5f\xd5\x81\x85\x9c\x08\xed\x1f" + "\x06\xe9\xbd\x9b\x71\x8c\x79\x9f\xef\xf2\x1b\xc7\x57\xb1\xed\x1f\x06" + "\xe9\xbd\x9b\x71\x8c\x79\x9f\xef\xf2\x1b\xc7\x57\xb1", + "\x21\xe6\x39\x87\xd4\x94\x67\x3f\x30\x40\xae\x9d\xe2\xbc\x0d\xa0", 1, + 32, 512 }, + { 96, 256, 128, 270, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\xe7\x2b\x83\x51\x4e\x5e\x50\x50\x90\x70\x35\x9c\x1c\xac\x7e\x1c\x42" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\xda\xd3" + "\x59\x50\xd8\xa9\xb5\x5a\x47\x2f\x9b\xb8\x86\x0a\x52\x63\x58\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\x35\xf2\xa4\xa1" + "\xce\xb0\x16\x94\xfc\xdd\x2a\x5d\xd5\xe6\x5a\x4b", + "\x07\x0a\xe7\x30\x79\x13\xe8\x7b\x44\x31\x28\x62\x8e\x34\x94\x59\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xeb\xdc" + "\x16\xf9\x85\xb8\x86\xf4\x6b\xd8\x5c\xa1\x39\x94\xf3\x88\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xeb\xdc\x16\xf9" + "\x85\xb8\x86\xf4\x6b\xd8\x5c\xa1\x39\x94\xf3\x88", + "\xe4\xfb\x94\x5d\x6a\x2d\x0b\x94\x78\x34\x31\x7c\xc4\x15\xf0\x24", 1, + 32, 640 }, + { 96, 256, 128, 271, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x8c\x61\x65\xf4\x45\x44\x35\x88\x04\x1b\x6e\x04\x4f\xb6\xba\xae\x72" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\x88\x1a" + "\x54\xc0\x95\x16\xa1\xf1\xca\xe7\xb9\xdd\x71\x13\x0e\xe1\x68\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\x67\x3b\xa9\x31" + "\x83\x0f\x02\x3f\x71\x15\x08\x38\x22\xff\x06\xc9", + "\x6c\x40\x01\x95\x72\x09\x8d\xa3\xd0\x5a\x73\xfa\xdd\x2e\x50\xeb\xcf" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb9\x15" + "\x1b\x69\xc8\x07\x92\x5f\xe6\x10\x7e\xc4\xce\x8d\xaf\x0a\xcf\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb9\x15\x1b\x69" + "\xc8\x07\x92\x5f\xe6\x10\x7e\xc4\xce\x8d\xaf\x0a", + "\xc0\x42\x48\x63\xa2\x0e\x5f\xa0\x4c\xcd\x97\x84\xc0\x15\xf0\x34", 1, + 32, 640 }, + { 96, 256, 128, 272, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x18\xe3\x61\x74\x54\x5f\xa7\xec\x9e\xa9\xf0\x5d\x70\x57\xc5\xca\x63" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\x43\x4e" + "\x1c\x5e\x71\x00\x5b\x69\x0c\xa5\xcb\x8d\x58\x0b\x89\xed\x79\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\xac\x6f\xe1\xaf" + "\x67\x19\xf8\xa7\xb7\x57\x7a\x68\x0b\xe7\x81\xc5", + "\xf8\xc2\x05\x15\x63\x12\x1f\xc7\x4a\xe8\xed\xa3\xe2\xcf\x2f\x8f\xde" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x72\x41" + "\x53\xf7\x2c\x11\x68\xc7\x20\x52\x0c\x94\xe7\x95\x28\x06\xde\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x72\x41\x53\xf7" + "\x2c\x11\x68\xc7\x20\x52\x0c\x94\xe7\x95\x28\x06", + "\xaa\x72\x93\xff\xe5\xdb\x30\xa3\x1f\x25\x81\xe0\xe7\xae\x56\xed", 1, + 32, 640 }, + { 96, 256, 128, 273, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x12\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x54" + "\x30\x5d\xff\x6b\x61\xc4\x0b\x77\x5c\x35\x2d\x02\x5c\x1a\x56\xd7\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xbc\xe5\x74" + "\xe9\xe1\x1a\xfe\xdb\xdc\xa0\x21\xe5\x3b\xb9\x18\x83\x38\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x06\x5c\xea\x26\xf1" + "\xff\x99\x81\x05\xca\x4f\xe3\x4c\xe5\x99\xd6", + "\xf2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe9" + "\x45\x27\x43\x80\xae\xd1\x2e\xde\x01\x0f\x0e\x69\xe8\x8f\x0f\xe6\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x1b\xfa\x1f" + "\x28\x2e\x1c\x1a\x80\x38\x1c\xbc\xe0\x5a\x90\xe4\x07\xe6\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x1b\xfa\x1f\x28\x2e" + "\x1c\x1a\x80\x38\x1c\xbc\xe0\x5a\x90\xe4\x07", + "\x42\xe5\xd4\x3d\x1e\x80\x8e\x79\xf0\x17\x14\x4d\x44\x98\xc2\x35", 1, + 32, 768 }, + { 96, 256, 128, 274, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xdf" + "\x05\x99\x19\x4b\x0c\xe8\x90\xcc\x1d\x8e\xb3\x83\xb5\x7f\x38\xdc\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x35\xdf\x81" + "\x07\x7d\x06\x80\x77\xce\x80\x5e\xa5\x92\xf6\xf8\x88\x33\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x8f\x66\x1f\xc8\x6d" + "\xe3\xe7\x2d\x17\xea\x30\xa3\xe5\xaa\x79\xdd", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x62" + "\x70\xe3\xa5\xa0\xc3\xfd\xb5\x65\x40\xb4\x90\xe8\x01\xea\x61\xed\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x92\xc0\xea" + "\xc6\xb2\x00\x64\x2c\x2a\x3c\xc3\xa0\xf3\xdf\x04\x0c\xed\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x92\xc0\xea\xc6\xb2" + "\x00\x64\x2c\x2a\x3c\xc3\xa0\xf3\xdf\x04\x0c", + "\x6c\xf2\xf9\x23\x0a\xf8\x67\x9e\x7e\xcb\x19\x42\x13\x62\xfc\xe3", 1, + 32, 768 }, + { 96, 256, 128, 275, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x39\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x40" + "\x92\xe1\xf9\xa2\x2c\x8b\x18\x18\x4d\x80\x5c\x12\x8a\xde\x57\xc7\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x64\xfe\x8b" + "\x9b\xdd\x21\x5a\x62\x09\x73\xaf\xfe\xfe\x93\x39\x85\x28\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xde\x47\x15\x54\xcd" + "\xc4\x3d\x38\xd0\x19\xc1\xf8\x89\xcf\xb8\xd0", + "\xd9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfd" + "\xe7\x9b\x45\x49\xe3\x9e\x3d\xb1\x10\xba\x7f\x79\x3e\x4b\x0e\xf6\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xe1\xe0" + "\x5a\x12\x27\xbe\x39\xed\xcf\x32\xfb\x9f\xba\xc5\x01\xf6\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xe1\xe0\x5a\x12" + "\x27\xbe\x39\xed\xcf\x32\xfb\x9f\xba\xc5\x01", + "\x6d\x46\xd2\x23\x0a\x98\x48\xd5\x18\xf9\xd9\x4b\xb2\xc4\x9c\xaa", 1, + 32, 768 }, + { 96, 256, 128, 276, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x12\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x32" + "\x7f\x3a\x1b\xef\xb4\x28\x7c\x17\x45\x03\x91\xed\x0e\xb8\x54\xd6\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x14\x60\xd3" + "\x54\x5c\x29\xdd\xc7\x90\x71\x1b\x8e\x75\x33\x69\x85\x39\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xae\xd9\x4d\x9b\x4c" + "\xcc\xba\x9d\x49\x1b\x75\x88\x02\x6f\xe8\xd0", + "\xf2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x8f" + "\x0a\x40\xa7\x04\x7b\x3d\x59\xbe\x18\x39\xb2\x86\xba\x2d\x0d\xe7\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb3\x7f\xb8" + "\x95\x93\x2f\x39\x9c\x74\xcd\x86\x8b\x14\x1a\x95\x01\xe7\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xb3\x7f\xb8\x95\x93" + "\x2f\x39\x9c\x74\xcd\x86\x8b\x14\x1a\x95\x01", + "\x74\xdd\xa1\x2e\x05\x58\x87\x7b\xc0\xe4\x0c\x3e\xac\xe0\xaf\x29", 1, + 32, 768 }, + { 96, 256, 128, 277, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1b\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x85" + "\xb6\x76\x64\xee\x49\xfa\x34\x7f\xbf\xd2\xdd\x92\x00\x7c\x57\xde\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xfb\x27\xee" + "\x07\x5b\x3c\x0f\x0f\x68\x2b\xab\xdd\xe6\x3d\xad\x87\x31\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x41\x9e\x70\xc8\x4b" + "\xd9\x68\x55\xb1\x41\xc5\xdb\x91\x61\x2c\xd2", + "\xfb\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x38" + "\xc3\x0c\xd8\x05\x86\xef\x11\xd6\xe2\xe8\xfe\xf9\xb4\xe9\x0e\xef\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x5c\x38\x85" + "\xc6\x94\x3a\xeb\x54\x8c\x97\x36\xd8\x87\x14\x51\x03\xef\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x5c\x38\x85\xc6\x94" + "\x3a\xeb\x54\x8c\x97\x36\xd8\x87\x14\x51\x03", + "\x50\x24\x55\x34\x3d\x39\xdb\x87\x94\x7d\x73\x46\xa8\xe0\xaf\x39", 1, + 32, 768 }, + { 96, 256, 128, 278, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x36\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x11" + "\x32\x81\x1b\x2f\x18\x32\x1b\xa9\x9b\x12\x43\x2c\x7f\x86\x5a\xa3\x35" + "\x2c\xd2\xd7\xac\x70\xb4\xc6\xf5\x41\x97\x67\x92\x6e\x20\x35\x25\x08" + "\xba\x45\xbb\xa7\x41\x0e\xbe\x1b\x8b\xb9\x25\x33\x4f", + "\xd6\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xac" + "\x47\xfb\xa7\xc4\xd7\x27\x3e\x00\xc6\x28\x60\x47\xcb\x13\x03\x92\x3a" + "\x63\x7b\x8a\xbd\x43\x1a\xea\x02\x86\x8e\xd8\x0c\xcf\xcb\x92\x3a\x63" + "\x7b\x8a\xbd\x43\x1a\xea\x02\x86\x8e\xd8\x0c\xcf\xcb", + "\x14\xfb\xa1\x49\xd1\xc0\xed\xc8\xaa\x66\x58\x51\x12\x6b\x5a\xfd", 1, + 32, 512 }, + { 96, 256, 128, 279, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xf9" + "\x99\x46\x10\x58\xf6\xd7\x73\x3e\x5c\xd0\xd1\x63\x9d\x90\x25\xcb\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x52\x0a\x0d" + "\xa5\x04\x39\xdb\x00\xe2\x89\xe1\x79\x13\x42\x06\x8e\x24\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xe8\xb3\x93\x6a\x14" + "\xdc\xbc\x5a\x3b\xe3\x8f\x7f\x64\x1e\x87\xdb", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x44" + "\xec\x3c\xac\xb3\x39\xc2\x56\x97\x01\xea\xf2\x08\x29\x05\x7c\xfa\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf5\x15\x66" + "\x64\xcb\x3f\x3f\x5b\x06\x35\x7c\x7c\x72\x6b\xfa\x0a\xfa\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf5\x15\x66\x64\xcb" + "\x3f\x3f\x5b\x06\x35\x7c\x7c\x72\x6b\xfa\x0a", + "\xbf\x7f\xbd\x42\x2c\xbf\x0e\x70\x0f\xd1\x60\x5b\xe8\xfd\x21\x2f", 1, + 32, 768 }, + { 96, 256, 128, 280, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x15\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xcc" + "\x16\x29\xa4\x0c\xd1\x1e\xaf\xdf\x04\x13\x8b\x45\xaf\xe4\x58\xef\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x34\x0a\xc9" + "\xb4\x5a\x58\x96\xa4\x18\xa8\xce\xe8\x03\x2e\x07\x8f\x00\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x8e\xb3\x57\x7b\x4a" + "\xbd\xf1\xfe\xc1\xc2\xa0\xee\x74\x72\x86\xda", + "\xf5\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x71" + "\x63\x53\x18\xe7\x1e\x0b\x8a\x76\x59\x29\xa8\x2e\x1b\x71\x01\xde\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x93\x15\xa2" + "\x75\x95\x5e\x72\xff\xfc\x14\x53\xed\x62\x07\xfb\x0b\xde\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x93\x15\xa2\x75\x95" + "\x5e\x72\xff\xfc\x14\x53\xed\x62\x07\xfb\x0b", + "\xc6\xf2\x32\x04\x86\x5b\x0a\xdd\xe0\x07\x00\x37\xd6\x53\x8d\xd3", 1, + 32, 768 }, + { 96, 256, 128, 281, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x31\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xff" + "\x74\x6e\xf5\x3e\xc3\x35\x7c\xbc\x3c\x3c\xe4\xab\x1d\x2d\x51\xed\x9e" + "\xb4\x56\xdc\x9d\x9b\x59\xf6\x56\xa5\xd2\xd9\x74\xd2\x6a\x7b\x8e\x90" + "\x3e\x4e\x8a\x4c\xac\x3e\x1d\xff\xce\x07\xc3\x8f\x05", + "\xd1\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x42" + "\x01\x14\x49\xd5\x0c\x20\x59\x15\x61\x06\xc7\xc0\xa9\xb8\x08\xdc\x91" + "\xfb\xff\x81\x8c\xa8\xf7\xda\xa1\x62\xcb\x66\xea\x73\x81\xdc\x91\xfb" + "\xff\x81\x8c\xa8\xf7\xda\xa1\x62\xcb\x66\xea\x73\x81", + "\x8c\xff\x61\xb7\xb3\x91\x9e\xd6\xbd\xe7\x2b\x36\xe0\xd3\x13\x26", 1, + 32, 512 }, + { 96, 256, 128, 282, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x19\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xbf" + "\x28\x6f\xd9\x79\x80\x79\x51\xb1\x83\xa1\x88\x93\x0a\xd1\x5e\xce\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x64\x41\x3d" + "\x71\x93\x9b\x9c\xb0\xa4\xd3\x2e\xf1\x15\xda\x9e\x10\x21\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xde\xf8\xa3\xbe\x83" + "\x7e\xfb\xea\x7d\xb9\x40\xf7\x62\x86\x1f\x45", + "\xf9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x02" + "\x5d\x15\x65\x92\x4f\x6c\x74\x18\xde\x9b\xab\xf8\xbe\x44\x07\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\x5e\x56" + "\xb0\x5c\x9d\x78\xeb\x40\x6f\xb3\xf4\x74\xf3\x62\x94\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\x5e\x56\xb0\x5c" + "\x9d\x78\xeb\x40\x6f\xb3\xf4\x74\xf3\x62\x94", + "\x36\x9c\xf1\x70\x11\xca\xe4\x75\x39\xe2\x72\x3f\x01\x0c\xf9\x80", 1, + 32, 768 }, + { 96, 256, 128, 283, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x19\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xbd" + "\x28\x6f\xd9\x79\x80\x79\x51\xb1\x83\xa1\x88\x93\x0a\xd1\x5e\xe3\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xf2\x5e\x78" + "\xfe\x1b\x53\xae\x41\x6d\x1f\xbc\x69\x85\x22\x61\x8f\x0c\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x48\xe7\xe6\x31\x0b" + "\xb6\xc9\x1b\xb4\x75\xd2\x6f\xf2\x7e\xe0\xda", + "\xf9\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00" + "\x5d\x15\x65\x92\x4f\x6c\x74\x18\xde\x9b\xab\xf8\xbe\x44\x07\xd2\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x55\x41\x13" + "\x3f\xd4\x55\x4a\x1a\x89\xa3\x21\x6c\xe4\x0b\x9d\x0b\xd2\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x55\x41\x13\x3f\xd4" + "\x55\x4a\x1a\x89\xa3\x21\x6c\xe4\x0b\x9d\x0b", + "\x53\x2e\xb8\xe2\x72\xa8\xd1\x71\x37\x8b\x0d\x42\xdf\xf2\xbe\xd9", 1, + 32, 768 }, + { 96, 256, 128, 284, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x32\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x25" + "\x8d\x5d\x3e\x44\x16\x83\xf5\x46\xbe\xba\x2e\x23\x75\x5f\x5c\xce\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x9d\x13\xfd" + "\xf8\xfa\x89\x98\x36\xfa\x5c\x41\x0d\x4c\xcd\x25\xea\x21\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x27\xaa\x63\x37\xea" + "\x6c\xff\x6c\x23\x36\x2f\x0b\x3b\x91\xa4\xbf", + "\xd2\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x98" + "\xf8\x27\x82\xaf\xd9\x96\xd0\xef\xe3\x80\x0d\x48\xc1\xca\x05\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x3a\x0c\x96" + "\x39\x35\x8f\x7c\x6d\x1e\xe0\xdc\x08\x2d\xe4\xd9\x6e\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x3a\x0c\x96\x39\x35" + "\x8f\x7c\x6d\x1e\xe0\xdc\x08\x2d\xe4\xd9\x6e", + "\xd1\xbe\x74\x26\xcd\x12\x44\x6f\xe5\x2e\x8d\x45\x33\x1e\x08\x35", 1, + 32, 768 }, + { 96, 256, 128, 285, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xd6" + "\x4a\xdd\x2a\xa3\xc5\xa3\x0a\x31\xd9\xe6\x5e\x90\xf9\x3a\xd1\xcb\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xde\x9a\xea" + "\xb8\x61\x44\xd5\x46\x48\x11\xb2\x37\x3b\xa4\xcc\x83\x24\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x64\x23\x74\x77\x71" + "\xa1\xb2\x1c\x91\x7b\xdc\x31\x4c\xf8\x4d\xd6", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x6b" + "\x3f\xa7\x96\x48\x0a\xb6\x2f\x98\x84\xdc\x7d\xfb\x4d\xaf\x88\xfa\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x79\x85\x81" + "\x79\xae\x42\x31\x1d\xac\xad\x2f\x32\x5a\x8d\x30\x07\xfa\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x79\x85\x81\x79\xae" + "\x42\x31\x1d\xac\xad\x2f\x32\x5a\x8d\x30\x07", + "\x62\x63\x0c\x18\xde\x8c\x10\x87\x6a\xdb\x9f\x30\xf3\x00\x96\x3f", 1, + 32, 768 }, + { 96, 256, 128, 286, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xcc" + "\x34\x92\x27\x2b\x8a\x4b\x11\x2a\x4e\x7d\x7c\xcf\x09\x26\x92\xce\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x30\xce\x67" + "\x8e\x93\x75\xb2\xaf\x0b\x82\xc2\xd2\xfb\xd7\x92\x8c\x21\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x8a\x77\xf9\x41\x83" + "\x90\xd5\xf5\xd2\xe8\xac\xd4\x8c\x8b\x13\xd9", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x71" + "\x41\xe8\x9b\xc0\x45\x5e\x34\x83\x13\x47\x5f\xa4\xbd\xb3\xcb\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x97\xd1\x0c" + "\x4f\x5c\x73\x56\xf4\xef\x3e\x5f\xd7\x9a\xfe\x6e\x08\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x97\xd1\x0c\x4f\x5c" + "\x73\x56\xf4\xef\x3e\x5f\xd7\x9a\xfe\x6e\x08", + "\xfe\xb6\x41\x2b\x90\x31\xf0\x76\xed\xdc\xd9\x42\x6f\xff\x5b\x31", 1, + 32, 768 }, + { 96, 256, 128, 287, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x34\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x72" + "\x2b\x65\x49\xc9\xdf\x0f\x4b\x04\xb5\xf7\x43\x22\x03\xfa\x54\xce\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x87\xde\x18" + "\x6c\xd2\x8e\x43\x54\x4c\x73\xde\x62\x8f\xd1\xd6\x0e\x21\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x3d\x67\x86\xa3\xc2" + "\x6b\x24\x0e\x95\x19\xb0\x64\xf8\x8d\x57\x5b", + "\xd4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xcf" + "\x5e\x1f\xf5\x22\x10\x1a\x6e\xad\xe8\xcd\x60\x49\xb7\x6f\x0d\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x20\xc1\x73" + "\xad\x1d\x88\xa7\x0f\xa8\xcf\x43\x67\xee\xf8\x2a\x8a\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x20\xc1\x73\xad\x1d" + "\x88\xa7\x0f\xa8\xcf\x43\x67\xee\xf8\x2a\x8a", + "\xda\xfd\xf4\x30\xc8\x12\x44\x83\xc1\x75\x40\x4b\x6b\xff\x5b\x41", 1, + 32, 768 }, + { 96, 256, 128, 288, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x3d\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xc5" + "\x62\x96\x99\xcf\xd4\xd9\x03\x6c\xef\x47\x8e\xd7\x05\xbe\x56\x50\xf5" + "\x75\x88\x2c\x38\x00\xf7\x57\xea\x6e\x0f\x8c\x6d\x47\xac\xc6\xe5\x51" + "\xe0\xbe\x2f\xd7\x02\x9f\xa1\x34\x13\x52\xda\x1a\xc3", + "\xdd\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x78" + "\x17\xec\x25\x24\x1b\xcc\x26\xc5\xb2\x7d\xad\xbc\xb1\x2b\x0f\x61\xfa" + "\x3a\x21\x71\x29\x33\x59\x7b\x1d\xa9\x16\x33\xf3\xe6\x47\x61\xfa\x3a" + "\x21\x71\x29\x33\x59\x7b\x1d\xa9\x16\x33\xf3\xe6\x47", + "\xf8\x80\x0c\x5b\x62\x83\xdd\xdf\xc4\x1f\x93\x5c\x01\xbd\x0d\x24", 1, + 32, 512 }, + { 96, 256, 128, 289, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x66" + "\xd6\x24\xf2\x88\xf5\x29\x41\xca\x24\x86\x5c\xe9\x6f\x0d\x97\x36\xff" + "\x33\xa2\x7c\x23\xf4\x97\x6f\xc7\x4f\x1f\xcd\x82\xf5\xcc\xa0\xef\x17" + "\xca\xee\x34\x23\x62\xa7\x8c\x15\x03\x13\x35\xa8\xa3", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xdb" + "\xa3\x5e\x4e\x63\x3a\x3c\x64\x63\x79\xbc\x7f\x82\xdb\x98\xce\x07\xf0" + "\x7c\x0b\x21\x32\xc7\x39\x43\x30\x88\x06\x72\x1c\x54\x27\x07\xf0\x7c" + "\x0b\x21\x32\xc7\x39\x43\x30\x88\x06\x72\x1c\x54\x27", + "\x38\xbf\xb8\x31\x8c\x62\x7d\x86\xc3\x4b\xab\x1f\x1e\xbd\x0d\xb0", 1, + 32, 512 }, + { 96, 256, 128, 290, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\xf4\xeb\xbe\x3f\xca\x96\xbc\x48\x85\xb3\x55\x82\xc4\x3e\x0e\xb3\x58" + "\x8a\x85\x43\x14\x30\xea\xda\x56\xa2\xc5\xdc\x94\x4b\x6a\xa6\xb4\x57" + "\x0e\x84\x46\xe8\x86\xbc\xbf\xf8\x2a\x24\xf4\x9b\xe5\xed\x42\xe0\x94" + "\x3e\x30\xf9\x1b\xa4\x1b\x43\x62\xfa\x9e\xd6\x03\x7b\x5b\x76\xf3\x75" + "\x50\xf1\x25\x72\x04\x0a\x9b\xc1\xa7\x77\xed\xc5", + "\x14\xca\xda\x5e\xfd\xdb\x04\x63\x51\xf2\x48\x7c\x56\xa6\xe4\xf6\xe5" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x85\x58" + "\x41\x2d\x1b\xf9\xb5\x12\x93\x0f\xed\x3d\x4b\x05\x44\x06\xe5\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x85\x58\x41\x2d" + "\x1b\xf9\xb5\x12\x93\x0f\xed\x3d\x4b\x05\x44\x06", + "\xaf\x72\x93\xeb\x09\x95\x7d\x9d\xe7\x43\x2d\xd4\x13\x16\xf0\xe4", 1, + 32, 640 }, + { 96, 256, 128, 291, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1a\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x57" + "\x1a\x3f\xca\x3c\xda\x7d\xef\x4c\x93\xd4\xa3\x82\xca\x3a\x57\xea\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\x76\xcd\xdb" + "\xee\x2f\x18\x57\x76\x17\x4f\x6d\xf3\xbb\xe5\xb3\x81\x05\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\xcc\x74\x45\x21\x3f" + "\xfd\x30\x2c\xce\x25\x03\xf5\xcc\xb9\x32\xd4", + "\xfa\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xea" + "\x6f\x45\x76\xd7\x15\x68\xca\xe5\xce\xee\x80\xe9\x7e\xaf\x0e\xdb\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd1\xd2\xb0" + "\x2f\xe0\x1e\xb3\x2d\xf3\xf3\xf0\xf6\xda\xcc\x4f\x05\xdb\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xd1\xd2\xb0\x2f\xe0" + "\x1e\xb3\x2d\xf3\xf3\xf0\xf6\xda\xcc\x4f\x05", + "\xe1\x78\xb0\xd5\xeb\x9b\xc5\x51\xfa\x64\x5c\x49\xf9\xf1\x76\x67", 1, + 32, 768 }, + { 96, 256, 128, 292, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x1f\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\xbe" + "\x31\xa5\x01\x53\x6a\x7c\x91\xe4\xa1\x02\xcc\x27\xcd\xfe\x09\xd2\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xdd\x94\x16" + "\xa1\x2e\x2f\x81\xbd\xee\x02\x3d\x46\x2f\xee\xf7\x83\x3d\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x67\x2d\x88\x6e\x3e" + "\xca\xe6\xe7\x37\x68\x53\x40\x58\xb2\x76\xd6", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x03" + "\x44\xdf\xbd\xb8\xa5\x69\xb4\x4d\xfc\x38\xef\x4c\x79\x6b\x50\xe3\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7a\x8b\x7d" + "\x60\xe1\x29\x65\xe6\x0a\xbe\xa0\x43\x4e\xc7\x0b\x07\xe3\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x7a\x8b\x7d\x60\xe1" + "\x29\x65\xe6\x0a\xbe\xa0\x43\x4e\xc7\x0b\x07", + "\xbd\xbf\x63\xdb\x23\x7d\x19\x5e\xce\xfd\xc2\x51\xf5\xf1\x76\x77", 1, + 32, 768 }, + { 96, 256, 128, 293, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x03\xe7\x6f\x6f", + "\xff\xff\xff\xff", + "\x3e\xde\x9b\x9e\xc8\xb2\x47\xd4\x2b\xbe\xe2\x01\x6d\x67\x15\xba\x85" + "\x67\xa7\xfd\xe8\x12\xa3\xaa\x2f\x55\x2a\x33\xc1\x71\x8c\x58\xe2\xf0" + "\xb0\x56\xa2\xee\xcc\x51\xd3\x08\x38\xe6\x40\x61\x5e\x14\xbb\x87\x29" + "\xfd\x14\x8f\x23\xb2\xa9\x16\xb7\xf4\x0f\x2f\x29\x81\x0d\xd1\x4d\xa7" + "\xb4\xf7\x6f\x9f\x68\xfa\x89\x03\x13\x8d\x56\x3c\x01\x3e\xb7\x32\x04" + "\x6a\x44\xe8\x70\x7c\xd9\xf2\x78\x73\xa8\xd4", + "\xde\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x38" + "\x12\xdd\x41\x03\xdd\xb6\x8f\x86\x08\x10\x10\xaa\xc5\x19\x01\xd3\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x1c\x98\x42" + "\x3c\xdb\x89\xc7\xe9\x4d\xaa\x2a\xf1\x6e\x06\xd5\x05\xd3\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x1c\x98\x42\x3c\xdb" + "\x89\xc7\xe9\x4d\xaa\x2a\xf1\x6e\x06\xd5\x05", + "\xb4\xcc\xb4\x22\xbc\x5f\x72\x64\xaf\xf7\x3f\x36\x75\xff\x5b\x19", 1, + 32, 768 }, + { 0, 256, 128, 294, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", "", "", "", "", 0, 0, 0 }, + { 64, 256, 128, 295, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07", "", "", "", "", 0, 0, 0 }, + { 88, 256, 128, 296, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a", "", "", "", "", 0, 0, + 0 }, + { 104, 256, 128, 297, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c", "", "", "", + "", 0, 0, 0 }, + { 112, 256, 128, 298, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d", "", "", + "", "", 0, 0, 0 }, + { 128, 256, 128, 299, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "", "", "", "", 0, 0, 0 }, + { 160, 256, 128, 300, + "\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30" + "\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "", "", "", 0, 0, 0 }, + { 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 0, 0, 0 } +}; diff --git a/test/wycheproof/gmac_test.json.c b/test/wycheproof/gmac_test.json.c new file mode 100644 index 00000000..8730d33a --- /dev/null +++ b/test/wycheproof/gmac_test.json.c @@ -0,0 +1,3263 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* AES-GMAC, 0.8r12 */ +#include "mac_test.h" +const struct mac_test gmac_test_json[] = { + { 128, 128, 1, + "\x98\xb0\x8a\x72\xff\xde\x0d\xed\x4b\xec\x9d\x2a\x8d\xb5\x72\x35", + "", + "\x51\x18\xcc\x71\x50\x1c\x82\x73\xa4\x36\x62\xb9\x81\x19\x17\x50", 1, + 0, "\x15\x95\x24\x87\x35\x31\x0e\xb7\x10\x51\x9c\x2b", 96 }, + { 128, 128, 2, + "\xf0\xcf\xce\x28\x06\x56\xfa\xbd\x93\xf6\x8b\xa6\xb3\xa3\xad\x6e", + "\x4b", + "\x86\x77\xa0\x16\x0a\x92\x3c\xe7\x43\x7c\xa9\x4b\x8d\xe9\x7d\xa5", 1, + 8, "\x0a\x38\xca\x62\x6b\x43\x0e\xd8\x4a\x2a\x8d\xfe", 96 }, + { 128, 128, 3, + "\xfd\x3c\x53\x81\xf5\x88\xbf\xe3\x32\x70\xe3\x36\xa5\xb0\x28\x96", + "\x02\x6f", + "\xeb\xc6\x96\x93\x10\x51\x0a\x2e\xb8\xac\xb9\xec\x3d\x63\x1f\x29", 1, + 16, "\x02\xd9\x16\x63\x1f\xba\xcf\x27\xc2\x74\xb7\x4c", 96 }, + { 128, 128, 4, + "\x53\x10\x7d\x29\xdc\x58\x4d\x32\xd3\xef\x32\x1a\x85\xee\xc2\x57", + "\x03\x9e\x0f\x5b", + "\x8c\x99\xcb\xf2\x8c\x43\xf9\x0f\xc3\x60\x91\x26\xbd\xf3\x0f\x4b", 1, + 32, "\xdd\x91\xa3\x3d\xce\x80\x9e\x7b\xfe\x68\xd8\x43", 96 }, + { 128, 128, 5, + "\x8e\x2c\x0c\xc6\x24\x72\x8a\xf2\x19\x5d\x46\x77\x55\x02\xec\x13", + "\xf7\xef\xb0\x72\x10\x0c\xc6", + "\x2b\x4e\x06\xa5\x58\xa4\x05\x42\xc0\xfd\xc7\x14\x96\x78\xc8\xb8", 1, + 56, "\xe8\xd4\x3d\x56\x46\xbb\x7d\xde\x8a\x1e\x93\x74", 96 }, + { 128, 128, 6, + "\xf5\x43\xfb\x6a\x66\x73\xe9\xdf\x99\x8a\x30\x58\x08\x4d\xcc\xc1", + "\x80\x63\x71\x9e\x25\xc0\xbc\x9a", + "\xd3\x2c\x12\xb4\xb7\x84\x5a\xda\xca\xda\xbe\x23\x87\xe2\x43\xd4", 1, + 64, "\x0e\x78\xb2\xa2\x25\x0b\x5d\xd1\xda\xeb\xcb\x7b", 96 }, + { 128, 128, 7, + "\xfe\x3f\x26\x6f\x5f\x23\x66\x69\x58\x2b\xd8\x06\x18\x76\x05\x43", + "\xdc\x25\xf0\x4a\xcf\xbd\xbc\x98\x6a\x10\x70\x43\x28\x6e\xae", + "\x60\x39\x2a\xd4\x1d\xeb\xec\x0d\xb4\x3d\x97\xdf\xb3\x79\x8c\x91", 1, + 120, "\x01\x43\x13\x72\x3d\x18\x60\xbf\x8f\x4b\x11\xfd", 96 }, + { 128, 128, 8, + "\x1d\x4a\x92\x39\x4c\x73\x56\xa6\xf5\x28\xcf\xb3\xad\xb8\xf2\x53", + "\x19\xb6\x9d\x41\x78\xd4\x9c\x20\xb4\xee\x58\x46\xcb\x44\x0c\x99", + "\x37\x90\xb3\x2d\x5c\xda\x9f\x8e\xf7\x2b\x64\x3d\x70\x10\x77\xd3", 1, + 128, "\x02\x19\x4e\xce\x3b\xc5\x0c\x51\xc5\x2b\xdd\x83", 96 }, + { 128, 128, 9, + "\xc9\x85\x9c\x8b\x39\xd8\xd7\xe8\x11\xd8\xae\x45\xd9\xb8\x58\x82", + "\x36\x71\x70\x76\x74\x8d\x93\x51\xe5\xa3\xcc\x67\xb9\xe8\x83\x3e" + "\x07", + "\xf0\xb2\x6b\x17\xb2\x8e\xa9\x27\x08\xb1\x2b\x87\x1f\x3b\x30\x71", 1, + 136, "\x11\x0d\x55\x01\x19\xd7\x1a\x09\x45\xbf\xec\xbb", 96 }, + { 128, 128, 10, + "\x8b\x4f\xd3\xd3\x05\x57\x46\xcb\x2d\xcd\x9b\x08\xc2\x98\x38\x6f", + "\x83\xe5\xab\xbb\xfe\xd5\xee\xb3\x53\xb2\x5f\x36\xe4\xad\xc0\xf2\x90" + "\x60\xc5\x5c\x08\xae\x4f\x80", + "\x2c\x9c\x99\xe0\x71\xf1\xd2\x78\x3a\x47\xc7\xa8\x47\xa3\xb7\x6a", 1, + 192, "\x1a\xac\x5c\xca\x3d\x02\xd4\x0f\x57\xfa\xbe\xcf", 96 }, + { 128, 128, 11, + "\x46\xc3\x3f\xff\x88\x98\xc4\xa4\x98\x55\x99\xcc\xc0\x5d\x05\x71", + "\x03\x85\x37\xbf\x94\xe1\xd7\xe1\x4f\x68\xb8\xd5\x45\x82\x41\xe3\x4f" + "\x51\x58\xde\xa6\xf8\x05\x20\x49\xb9\xda\xd8\xfb\x66\xcb\x6e\x0b\xa6" + "\xcc\x22\x3f\x67\x56\x14\xe9\x5d\x15\x27\xc7\x46\xe6\x50\xe9\xfc\x6a" + "\xca\x69\xe6\x82\xd9\xe3\x0a\xc0\x6e\x0a\x48\xc0\xa0\x42\x8e\xc1\xae" + "\x23\x73\x9d\x82\xfc\x24\x6e\x4c\xd6\xbc\x27\xcd\x1d\x54\xc0\xe6\x30" + "\xc1\x62\x4f\xe3\xdb\xd0\xa8\xce\xa3\xb7\xc8\xf2\xd7\xc1\xcc\xa8\xb8" + "\xe0\x86\xca\xb0\x21\x53\xbe\x76\x2d\x59\xe4\x9c\x53\x3c\xb3\x9e\x65" + "\xab\x37\xf6\xca\xd2\x29\x0d\x0d\xfe\x2a", + "\xfe\xee\xc6\x27\x6f\x89\x39\x3b\x5a\x52\x22\xe0\xeb\xb1\x60\xfd", 1, + 1032, "\x01\xda\x1c\xa7\x2e\x26\xf6\xd6\xfa\x9c\xe2\x81", 96 }, + { 128, 128, 12, + "\xd0\x96\xbe\xd9\x70\xd1\x7a\x34\x00\xcd\x60\xce\xf5\x7b\x4e\x49", + "\x30\xf8\xa0\xee\xa9\x37\x56\x7b\xfe\xc3\xb1\x5d\x1c\x58\xa1\x70\xb4" + "\xf8\xaa\xe9\x49\x17\x8a\xd0\x4d\x8e\xa4\x6b\x6e\x62\x5b\x50\xdc\xe2" + "\x52\xdc\x38\x0e\x9e\xff\x53\xca\x20\x57\x22\x80\x39\xed\xd7\x8a\xe3" + "\xe2\x28\x19\xaf\x6e\x3b\xdc\x83\xb1\xf0\x73\x6a\xbb\x76\xeb\x40\x44" + "\xe4\x74\x34\x64\xfb\xc6\x1c\x62\xb0\x68\x39\xe3\x72\x28\x9a\xa8\x21" + "\x8e\xa0\xa6\xe4\x43\xa3\x7a\x65\xc9\x48\x85\x73\x80\x31\x8f\xe0\x7b" + "\xd4\x82\xdf\xba\xef\x86\x9c\xdf\x75\xfd\x2d\x95\x75\xf6\x04\x0f\x90" + "\xf2\x0b\xb4\x1b\x39\x0d\x4d\x1c\xda\xa3\xb7\xb6\xa5\xd9\xa9\x47\xbc" + "\x71\xf0\x6a\x74\xdb\x8f\x65\x13\x5f\x59\x82\xb7\x5b\x21\xa0\xdc\xc3" + "\x3e\xaf\x7c\x6b\x26\xda\x62\xbf\x0f\xe9\x07\x4f\x3f\x96\x1e\x73\x9f" + "\x22\x92\x19\x2e\x33\x09\x39\x7d\x19\xde\xd4\xd4\x4a\x6c\xe7\x48\xf3" + "\x53\x0c\x7b\xee\xc7\x6d\x65\xd4\x0c\x6b\xc8\xd2\x81\xf7\xc2\x3d\x56" + "\x37\x07\x10\xd8\x95\x2d\x7f\x68\xfc\xac\x5e\x8f\x53\xb5\x8d\x07\x14" + "\x23\x8a\x9c\x79\xc8\x36\xd9\xb7\x91\x54\x1e\xd5\x27\xa9\xd6\xef\x87" + "\xbc\xef\x11\x42\x65\x3e\x73\x4e\x66\x60\x1d\x2a\x8a\x37\xdd\x20\x07" + "\x16", + "\x01\x4c\x13\xda\x92\xa0\xe9\x93\x29\xa8\x8c\x9c\xc8\x4e\x08\xd4", 1, + 2048, "\x23\x18\x54\x32\x3b\xc9\x56\x13\xa1\x5d\xd7\xd5", 96 }, + { 128, 128, 13, + "\x6f\x8b\x47\x45\x2e\xf0\x25\x02\x85\x60\xf6\x16\x01\xb6\x4e\x46", + "\xf3\x6e\x40\x72\xc2\x04\x5b\x89\x95\xed\x8f\xb9\x90\x7b\xc9\x85\x60" + "\x6b\x01\xd1\x92\x0b\x09\xe3\x93\xe3\x79\x26\x37\x7f\x89\x47\x8e\x6e" + "\xd5\x53\x14\xc5\x4f\x29\xff\x0b\xd2\x5f\xf7\x5e\x99\xf1\xde\xa9\xeb" + "\x7a\xe0\x86\x05\x93\xe5\x96\x7c\x57\xa8\x35\x2f\x13\x5a\x33\xc0\x46" + "\x4b\x90\xdd\x6d\x5e\x8f\x9a\xef\xb9\x42\x83\x91\xce\xff\xf6\x19\xf2" + "\x93\x08\x4d\xc9\x59\x2b\x0b\x72\xad\x03\x4f\xad\x93\x09\x00\x18\x70" + "\xcc\x1f\x58\x97\xeb\x94\xb6\x8a\xd1\x85\x61\xc8\x3b\x7f\xb5\x55\x36" + "\xb4\xbe\x80\x4b\x4a\xe8\xe9\x80\x05\xc4\xf1\x18\x0b\xad\xd7\xe4\xd9" + "\x8a\x6e\x0f\x62\xa5\x90\xc1\xbf\xae\xbb\x2e\xf7\xca\x56\x96\x24\x5a" + "\x69\xb3\x48\x63\xf9\x95\x26\x96\xd3\x18\xb6\x9d\x14\xa1\xed\x7c\xe7" + "\x04\x64\x54\x2f\x3c\xfd\xa2\xe9\x3b\x22\x86\xab\x3d\x1a\x43\x54\x4a" + "\x24\xa4\x91\xde\x66\x04\x3c\x0e\x83\xc6\x99\x13\x8f\xa2\x94\xe6\xfd" + "\x29\x3d\x20\xb5\x96\xf9\x28\x4e\x75\x52\x85\xd3\x6e\x32\xd1\x1e\x3d" + "\x51\x83\x0f\x55\x40\x5a\x91\xcb\x13\xce\xc0\xef\x4e\xb3\x68\x3b\x0c" + "\x28\x8b\x72\xbc\xc0\x4e\x8e\x53\x59\xb1\xae\x44\xc3\x34\x0c\x64\x04" + "\x42\x6d\x9a\x3b\xa5\x05\x9c\xbb\xa0\x50\x6a\x3a\x69\x87\xa5\x9e\x9a" + "\x0a\x2e\xac\x96\xf9", + "\x53\x7a\x54\x05\xae\x5d\xfd\x3a\x83\xad\x6e\x9e\x4b\x4d\x0a\x1d", 1, + 2216, "\xc8\x99\x82\xa6\xa7\x87\xca\xa3\x8f\x16\x6e\xc7", 96 }, + { 128, 128, 14, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8c\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 15, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7c\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 16, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8f\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 17, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7f\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 18, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x0d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 19, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xfd\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 20, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf6\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 21, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x03\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 22, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\x6d\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 23, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x03\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 24, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb8\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 25, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb6\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 26, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xbb\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 27, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb5\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\x7a\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x12\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xac\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcd\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\x2d\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\x4c\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x3b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x41\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x02\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x53\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x52\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x49\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x51\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x4a\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\xd3\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\xc8\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xe9", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x05", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xea", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x06", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\xa8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x44", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\xfa\xad\x1b\x03\x8c\x53\xb3\x20\x68", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x92\xcc\x61\x52\x8e\x48\xcb\x02\x84", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8c\xf7\xd8\xed\xb9\x91\x65\xfa\xac\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7c\x02\x8e\x83\xb7\x27\xda\x92\xcd\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\x6d\xb9\x91\x65\x7a\xad\x1b\x03\x8c\x53\xb3\x20\xe8", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x03\xb7\x27\xda\x12\xcc\x61\x52\x8e\x48\xcb\x02\x04", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8d\xf7\xd8\xed\xb9\x91\x65\x7a\xad\x1b\x03\x8c\x53\xb3\x20\x68", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7d\x02\x8e\x83\xb7\x27\xda\x12\xcc\x61\x52\x8e\x48\xcb\x02\x84", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x72\x08\x27\x12\x46\x6e\x9a\x05\x52\xe4\xfc\x73\xac\x4c\xdf\x17", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x82\xfd\x71\x7c\x48\xd8\x25\x6d\x33\x9e\xad\x71\xb7\x34\xfd\xfb", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x0d\x77\x58\x6d\x39\x11\xe5\x7a\x2d\x9b\x83\x0c\xd3\x33\xa0\x68", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xfd\x82\x0e\x03\x37\xa7\x5a\x12\x4c\xe1\xd2\x0e\xc8\x4b\x82\x84", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x8c\xf6\xd9\xec\xb8\x90\x64\xfb\xac\x1a\x02\x8d\x52\xb2\x21\xe9", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x7c\x03\x8f\x82\xb6\x26\xdb\x93\xcd\x60\x53\x8f\x49\xca\x03\x05", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 128, 128, 68, + "\x82\x6b\xa4\x9b\x4a\xff\x2a\xdc\x6b\x22\xdd\x4a\x84\xf2\x99\x41", + "\x32\x72\x5b\xc6\x38\x59\x49\xb2\x98\x75\xf8\x39\xe7\x5c\x06\x71", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, "\xce\x88\xbe\x4f\x89\xc9\x46\x02\x7d\x7c\x54\x2a", 96 }, + { 128, 128, 69, + "\x82\x6b\xa4\x9b\x4a\xff\x2a\xdc\x6b\x22\xdd\x4a\x84\xf2\x99\x41", + "\x4a\x41\x61\x2e\xec\x3b\x21\x56\x4f\xf6\x23\x91\xeb\xae\x07\x6c", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, "\xce\x88\xbe\x4f\x89\xc9\x46\x02\x7d\x7c\x54\x2a", 96 }, + { 128, 128, 70, + "\x82\x00\x8d\xa5\xb6\x5a\x6e\x99\xa3\xe9\x78\xab\x5a\x98\xe9\xb0", + "", + "\x82\xf2\xab\xe9\xa8\x97\xcc\xda\x3d\x2c\xbd\x39\x57\x97\xc8\x3e", 1, + 0, "\x63\x8c\x5f\x29\xff\x29\x23\x33\xab\x0e\xb7\xcd\x66\x56\xde\x69", + 128 }, + { 128, 128, 71, + "\x7c\x3f\x22\x46\xf4\xa3\x26\xae\x60\x05\x4f\x41\x7c\x20\xe9\xc1", + "\x2d", + "\xa0\x79\xf6\x04\x8f\xed\x16\xf0\x04\x1f\xa0\x4d\x7d\x70\x92\x5e", 1, + 8, "\x16\xa0\xff\x55\x39\xe1\x0a\x86\xec\x54\x33\x76\x4d\xa7\x1b\x59", + 128 }, + { 128, 128, 72, + "\x0f\x62\x4e\x31\x8b\x51\xa9\xcc\x6c\x30\xcf\x4e\x45\xa2\x87\xdb", + "\x0e\x45", + "\xd0\x7a\x83\xbb\xd5\x44\xf0\xb0\x79\xee\xc6\x32\xf8\x09\x74\xc9", 1, + 16, + "\x0d\x4d\xf0\x21\x63\x3d\xce\xbb\x58\xf2\xa3\x89\xe1\x91\x1b\x66", + 128 }, + { 128, 128, 73, + "\x57\x43\x6a\xb1\x0e\xe8\x28\x55\x2e\x63\xe1\xbf\xbf\x83\x81\xdd", + "\x95\x8a\x56\x55", + "\x66\x5b\x87\xde\x5e\x3e\xec\x09\x8f\x9e\x6a\xd7\x40\x12\xf2\x36", 1, + 32, + "\x0c\x5d\x7c\x05\xd7\x21\xf4\x22\xda\x93\xd6\x01\xe9\xfe\xbe\xf0", + 128 }, + { 128, 128, 74, + "\x18\xd6\xdf\x42\x42\xd1\x00\xb5\x71\x21\x57\xc7\x72\xc8\xe1\x28", + "\xc9\xc7\xd4\x6f\xd7\x9e\x72", + "\x92\xe3\xc1\x24\xab\xdd\x0f\x9c\x10\xce\xbe\xa8\x4f\xd8\x5e\x0a", 1, + 56, + "\xee\xad\x99\xcd\xb2\x04\x42\x2d\xb1\xa0\x08\xfe\x6d\xea\x6a\x3a", + 128 }, + { 128, 128, 75, + "\xf9\x81\x9a\x0d\x58\x53\xfe\x80\x56\x45\x23\x1e\x43\x33\x62\xce", + "\x37\x4d\xd4\xb4\x66\xa5\x1b\xf5", + "\x56\xb0\x62\x73\x96\x08\xc6\x7a\xe6\xbd\x1a\xa6\x1e\x83\xbe\x2b", 1, + 64, + "\x23\x07\xa6\x4b\x29\xef\x75\xd4\x64\xc0\x51\x06\x1d\x06\x9b\xbe", + 128 }, + { 128, 128, 76, + "\x47\xff\x15\xa9\xa9\xf5\x65\xdf\x93\x25\x57\x74\xa1\x29\x6b\x11", + "\xf3\x5d\xf7\xb3\x38\x71\x7e\x4e\xc9\x97\x45\x48\xb9\x9f\x21", + "\x46\x08\xea\x9a\x7e\x40\xa7\x29\xd2\x62\x59\xa9\xc6\x2b\xc5\x86", 1, + 120, + "\xeb\xff\xd4\x1f\x0f\x5b\x3b\xdc\x64\x7d\xa8\x03\x6b\xe5\xe4\xdd", + 128 }, + { 128, 128, 77, + "\x75\x91\x85\xe9\x95\xfb\x6d\xee\xc8\x01\xba\x11\x9d\xb8\x7b\xcc", + "\xde\x69\xd4\x92\x6f\xbc\x7d\xaa\xc4\x1a\xdd\xdf\x69\x2f\xe1\x6b", + "\xd2\x12\xc3\xe2\x57\x83\x74\x81\xe9\x7b\x31\xe7\xa8\x99\xb1\x26", 1, + 128, + "\x80\x86\x24\x48\x2a\xd1\x9d\x2a\x8b\x76\xac\x96\xa3\x09\x05\x13", + 128 }, + { 128, 128, 78, + "\x77\x22\x74\xb7\x24\x0c\x3a\xda\x54\x68\xfa\x3f\x12\xbc\xe8\x1d", + "\xdf\x69\x01\x0d\xc9\x2f\xc4\xaf\x5e\xc1\x9d\xd1\x65\xb0\xd2\x6a" + "\x97", + "\xb6\x85\x46\x3b\xed\xda\xd5\x56\x68\xe4\x48\xb0\xc1\x3f\xb3\x2f", 1, + 136, + "\xd0\x7e\xfa\x93\xb8\x88\x8c\xd7\xf0\xd3\xfe\x53\xb3\x59\x2b\x62", + 128 }, + { 128, 128, 79, + "\xf2\xd0\xfd\x91\xee\xca\x7f\x55\xe5\xf7\xb5\xc5\x7f\x59\xc5\x77", + "\x7c\x51\x95\x7d\x97\x4a\x84\x78\x77\xe8\xb7\xf7\x16\x5d\xe4\x6e\xc0" + "\x50\xcb\x5c\xb0\x3c\xd5\x6f", + "\xa5\xbf\x64\x54\x74\x86\x54\xfb\xd9\x01\xca\x5c\x28\xc0\x09\xd6", 1, + 192, + "\x27\xbb\x91\x06\x32\x86\x93\x14\x97\x9d\xea\x4a\xa6\x80\x16\x13", + 128 }, + { 128, 128, 80, + "\x97\xca\xc4\x81\x2b\x0d\x1e\xbc\x7f\x26\xf2\xa7\xb8\x11\x81\x2b", + "\xa9\xd9\x97\xfa\x61\x44\x43\x93\xce\x45\xf9\xaa\xcb\x2c\x9d\xcc\x50" + "\x98\xae\xdc\xc4\xd5\x69\xcd\x92\xa6\x75\x6b\x4e\x53\x9b\x28\x37\xf7" + "\xec\x79\x8c\x05\x61\xf8\x93\xcf\xf8\x8b\x16\x0a\x68\xd2\xcd\xe2\xcb" + "\x09\x77\x89\x60\xf6\x68\xa9\x8e\xc3\xd2\x15\x20\xa3\x56\xb9\x12\x82" + "\x14\x28\x3f\xa7\xb1\x3d\xef\x37\xb9\x2c\xa3\x33\x60\xc6\xaf\x90\x6d" + "\x9f\x2d\x5b\x94\xd4\x00\x7b\xfa\xe0\xf9\x5f\xcb\xe6\x04\xe4\x22\x36" + "\x01\xf6\x4b\xae\xe1\x50\xfa\xaa\xfd\x7e\x9c\xbc\xdc\x67\xbb\x9d\xfd" + "\x8f\x70\x14\xd4\x91\xd9\xae\x5f\x67\x7a", + "\x03\x50\x77\x2a\x98\xaa\x49\xdf\x0a\x66\xc6\xd4\x6d\x4a\x60\x34", 1, + 1032, + "\xe1\x2b\x41\x5a\x79\x1b\x51\xdb\x9d\x35\xf7\x35\xac\x50\x78\xc6", + 128 }, + { 128, 128, 81, + "\xcb\xf3\x1f\xf2\x9c\x06\x80\x99\x32\x93\x50\x1c\x2e\x1f\xf3\x4c", + "\x7b\xfb\x18\x64\xf9\x2d\x4a\x92\x58\xf0\xac\x72\x1e\xb8\x74\xcf\xf7" + "\xd7\x34\x27\x23\x7c\xea\x9f\x4d\x0e\x28\x39\x84\x85\xc4\xed\xa7\x83" + "\x7c\x60\x44\xb1\x5d\x3d\x30\xe1\xea\x75\x07\x47\x0e\xdf\x0b\x46\x5a" + "\x4f\x48\xa8\x6d\xe4\x59\xc3\xbb\x32\x55\x3c\x51\x54\x3c\x7c\xc6\xbb" + "\x9a\x3c\x28\x40\x8f\x56\x5e\xcf\x9f\x16\xb3\x9a\xb0\xc7\x86\x73\xeb" + "\x2d\xda\x8c\x18\xc2\xfb\x96\x5c\x82\x5a\xc6\x11\x6e\xf3\xd7\x5f\x9e" + "\x7d\x8d\xd8\x7e\x89\xd3\x9e\x29\x4e\x63\x09\xe8\xe0\xbf\x33\x02\xd2" + "\x03\xf3\xe9\x40\x1e\x77\x18\x44\x45\x81\xff\x54\xfd\xa6\x66\x48\x91" + "\xbc\x3d\x7e\xc5\xa0\x97\x92\xf0\xc5\xeb\x7d\x2a\xb6\x0c\x68\x3b\xb1" + "\x8d\x19\x15\x9f\xe5\x6a\x40\x20\x7b\x8b\x1c\x1a\x0a\x47\xc6\x66\x4c" + "\x9d\xc6\x3a\x75\xc7\x14\xb6\xd3\x4b\x95\x75\x9d\x4c\x1c\xb6\x81\x13" + "\xfa\xab\x3f\x63\xcd\x0a\xf1\x00\xcf\x66\x02\x2c\xe3\x51\xf8\xc4\xf4" + "\x27\xee\xa9\x77\xca\x67\x34\x3c\x4d\xd2\xda\xf2\x36\x62\x30\x93\x3f" + "\x16\xa0\x76\xca\x65\x3a\xff\x91\x25\x7f\xc9\x48\x74\xbb\x56\x4b\xfd" + "\xf3\x9e\xa4\xce\x06\x1f\x56\x2b\xaf\xbf\x67\x40\xe2\x45\x3d\xef\xae" + "\x5f", + "\x40\x12\xf2\xc6\xf3\x12\x93\xc1\xd3\x90\x53\x5d\xfe\xbc\xa6\x18", 1, + 2048, + "\x85\x0a\xea\x39\xc6\x6d\x85\xc5\x9a\x4a\x65\xa0\x67\xe9\xd6\xfc", + 128 }, + { 128, 128, 82, + "\xde\x20\xc3\x9c\x12\x01\x1e\x61\xfb\x40\xc9\xd7\xdc\x1a\x26\xdb", + "\x2e\x11\xd7\x8f\x73\xc8\x30\xc1\xd3\xd3\xf7\x87\x47\x9b\xc3\x58\x48" + "\x91\xdc\xe8\xae\xa3\x7c\x72\xcf\x87\x6f\x9f\x37\xc9\x25\x5d\x28\xd3" + "\xad\x4e\xfb\xdf\xc9\x63\xf2\xc4\xec\x4d\xfa\xe4\xeb\x56\x8e\x1a\x3f" + "\xac\x31\x38\xcd\x2e\xfc\x28\x66\x91\x9e\x42\x52\xa5\xd5\xcf\x07\x0c" + "\xe4\x83\xd6\x09\x74\x2b\x65\x42\xa4\x49\xdc\xe4\x49\xa1\xf5\x18\x4b" + "\x92\x04\x2b\x65\x77\x91\x6c\x11\x50\xa8\xe2\x70\x56\xc1\xa3\x3a\x65" + "\xbd\xb2\x51\x69\x02\x74\x0a\xcd\xb0\x9f\x90\xce\xb6\x71\x3f\xb9\x92" + "\xfd\x81\x36\xee\xf0\xeb\xe9\xb5\xd7\xdf\x10\x2e\xef\x2b\x34\xa4\x4f" + "\x18\x4a\x9f\x8b\xdc\x92\x3c\x0c\x7d\x46\x3d\xbd\xd3\x23\xa5\x00\xdf" + "\xb5\xb7\xde\x8f\x43\xfc\xea\x5e\x7e\x73\x26\xd4\xee\x0d\x1c\xea\x46" + "\xc9\x39\x3e\x7a\x96\x0f\x84\xda\x95\xbb\x14\x10\xbc\xe1\x3c\xbb\x88" + "\x28\x0d\xd7\xd6\x46\xf9\x93\xf1\x7e\x89\xc1\x16\x74\x24\x27\x43\xfe" + "\x1a\x7a\xf4\x53\xde\x01\xbe\x2c\x64\x36\x06\xe4\x65\x1a\xc0\x59\x5a" + "\x65\x48\xba\x47\x4f\x25\x95\x1e\xfb\xda\x00\x95\x3e\x12\x22\x09\x3c" + "\x86\x3d\x0f\xb0\x08\x7e\xcc\xc0\x45\x7d\x51\xe6\x20\x30\x6b\xaf\xea" + "\x0a\xb3\xfb\x72\x82\x7e\xd5\x70\x51\xa8\xcf\x6f\x15\xe5\x4b\x2e\x84" + "\xb3\xec\x8a\x33\xbb", + "\x10\x6a\x8b\x1d\x9a\xed\x24\x63\x87\x59\x5c\xe9\xae\xcf\x16\x3f", 1, + 2216, + "\xce\xb4\x9c\x91\xd2\x09\x42\x78\x86\x23\xc8\x56\xa5\xf4\x44\xd4", + 128 }, + { 128, 128, 83, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xdf\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 84, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2f\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 85, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xdc\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 86, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2c\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 87, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x5e\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 88, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xae\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 89, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc6\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 90, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x33\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 91, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\xb8\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 92, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\xd6\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 93, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8b\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 94, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x85\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 95, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x88\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 96, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x86\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 97, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\xd3\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 98, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\xbb\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 99, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xfe\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 100, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9f\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 101, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\x7f\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 102, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x1e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 103, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x3f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 104, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x45\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 105, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x85\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 106, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd4\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 107, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x58\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 108, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x43\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x5b\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x40\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\xd9\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\xc2\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf6", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x1a", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xf5", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x19", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\xb7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x5b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\x53\xff\x1f\x84\xf1\x59\x99\x74\x77", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\x3b\x9e\x65\xd5\xf3\x42\xe1\x56\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xdf\xc7\xf8\x38\x8a\x5a\x84\x53\xfe\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2f\x32\xae\x56\x84\xec\x3b\x3b\x9f\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\xb8\x8a\x5a\x84\xd3\xff\x1f\x84\xf1\x59\x99\x74\xf7", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\xd6\x84\xec\x3b\xbb\x9e\x65\xd5\xf3\x42\xe1\x56\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xde\xc7\xf8\x38\x8a\x5a\x84\xd3\xff\x1f\x84\xf1\x59\x99\x74\x77", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2e\x32\xae\x56\x84\xec\x3b\xbb\x9e\x65\xd5\xf3\x42\xe1\x56\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x21\x38\x07\xc7\x75\xa5\x7b\xac\x00\xe0\x7b\x0e\xa6\x66\x8b\x08", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd1\xcd\x51\xa9\x7b\x13\xc4\xc4\x61\x9a\x2a\x0c\xbd\x1e\xa9\xe4", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x5e\x47\x78\xb8\x0a\xda\x04\xd3\x7f\x9f\x04\x71\xd9\x19\xf4\x77", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xae\xb2\x2e\xd6\x04\x6c\xbb\xbb\x1e\xe5\x55\x73\xc2\x61\xd6\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xdf\xc6\xf9\x39\x8b\x5b\x85\x52\xfe\x1e\x85\xf0\x58\x98\x75\xf6", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x2f\x33\xaf\x57\x85\xed\x3a\x3a\x9f\x64\xd4\xf2\x43\xe0\x57\x1a", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 128, 128, 137, + "\xb0\x91\x32\xf1\xb7\x5e\xef\x72\x9e\x4f\x61\x66\xc2\x64\x90\xdb", + "\x20\x08\xb1\x1d\x8a\x75\x6a\xfc\x3d\x63\x40\x14\x1d\xf6\x77\x5b", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, + "\x2a\xda\x15\x3c\x42\x3a\x82\xf8\x45\xdb\x0c\xaa\xa9\x4b\x96\x6d", + 128 }, + { 128, 128, 138, + "\xb0\x91\x32\xf1\xb7\x5e\xef\x72\x9e\x4f\x61\x66\xc2\x64\x90\xdb", + "\x17\xc0\x16\x62\x31\x2d\x93\xbc\xe1\x5b\x1e\xcb\xa9\x68\x2b\x2d", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, + "\x2a\xda\x15\x3c\x42\x3a\x82\xf8\x45\xdb\x0c\xaa\xa9\x4b\x96\x6d", + 128 }, + { 192, 128, 139, + "\xde\xd6\xff\x22\x55\x65\x87\x32\xf0\xfe\xf0\x84\x07\x13\x68\x93\xf1" + "\x2e\xdf\xeb\xdf\x86\xa2\x14", + "", + "\x59\xa1\x46\x0f\x62\x83\x16\x8b\x32\xc7\xf5\x27\xb7\xa7\x6f\x12", 1, + 0, "\x05\x36\xe9\xff\x30\x45\x94\xf2\xad\xfe\x5e\x02", 96 }, + { 192, 128, 140, + "\x6a\xda\x91\xc9\x74\x2d\xc0\x1e\x95\x3e\xf1\x20\xb2\x2a\x26\x49\xc0" + "\xbe\x93\x04\x38\x65\x15\xa2", + "\x0d", + "\xb1\xd2\xa3\xdb\x72\x4f\xf1\xef\x11\xaa\x76\x95\x4e\x39\x49\x8e", 1, + 8, "\xe1\x9f\xaa\x23\xf8\x71\xfc\x66\x58\xf5\xb5\x96", 96 }, + { 192, 128, 141, + "\x9b\xee\x48\xdc\x40\xa8\xb5\x22\xbd\x7a\xbd\xe7\x99\x9a\xde\xdd\x70" + "\xba\xb2\xef\xf8\x00\xa0\x50", + "\x94\xc8", + "\x36\xb0\x21\xf1\x9c\x12\x4c\x46\x50\x0e\x40\x7f\xc8\xe0\xcd\x5e", 1, + 16, "\x6f\xaa\xe4\x20\x1f\x79\xc9\xd5\x32\xf6\x3d\x85", 96 }, + { 192, 128, 142, + "\xf7\x9e\x59\x0f\x07\x38\x81\x24\xc8\x00\xdd\x4a\xda\xda\xc2\x54\x6c" + "\x62\x67\x96\xc4\x6c\x3d\x2d", + "\xea\x09\x34\x8f", + "\x5f\xa2\x25\x2f\x2b\xbe\x35\x08\xc8\xa9\x19\xc2\x68\xb9\x51\x4a", 1, + 32, "\xc2\x36\x00\x65\x82\x07\xbf\x2d\xc3\x08\x08\x7c", 96 }, + { 192, 128, 143, + "\x35\x6a\xe8\x0d\x9f\xf5\x1f\x72\x92\x0a\x0b\xcc\x80\x78\x37\xcd\x79" + "\x77\x68\xad\xba\x48\x75\x1c", + "\xa8\xab\x4d\x4d\x07\xc7\x1e", + "\x04\x31\xff\x77\x84\xb4\x55\x6e\xc6\x31\x24\xce\xa0\x3e\xb0\x39", 1, + 56, "\xae\x0a\x4c\x45\x2d\x43\xa2\xc8\x41\xd6\xe0\xfd", 96 }, + { 192, 128, 144, + "\x26\x8b\x5c\xb7\x84\x37\xe8\x7a\xf7\x5f\x27\xab\xc7\x53\x90\x9b\xda" + "\xa2\x34\x1f\x88\x91\xd7\x7e", + "\xc4\x2a\xb5\x35\xa8\x76\x94\x2b", + "\x93\xb3\xff\x81\x65\x4e\xe9\x57\x91\xc8\x66\x82\xdb\x37\x89\x4a", 1, + 64, "\xee\x1a\x14\xe5\x50\x24\x2a\x65\x3f\xd4\x50\x9f", 96 }, + { 192, 128, 145, + "\x9c\x42\x6c\x09\xc0\xa0\xc5\x19\xa5\x54\x84\x1e\xb7\x69\x88\x6a\xa3" + "\xd8\x27\x25\xe0\x1b\x25\x05", + "\xe6\x56\x80\xb9\xae\xeb\x43\xbb\x41\x44\x90\x08\x00\xf3\xbd", + "\x0f\x39\x2a\x50\xfe\x0f\x18\x3a\xd9\x41\x1a\x61\x65\xd5\x4b\xda", 1, + 120, "\x8f\x64\x41\xff\xb9\x77\xde\x17\x6c\x4f\x53\x36", 96 }, + { 192, 128, 146, + "\x60\x31\xb3\xa1\x77\xb4\x30\x45\xae\x58\x7f\xfb\x0d\x22\x8b\x04\x14" + "\xff\x26\xb7\x20\x6c\x30\xf5", + "\xdc\x89\xf1\xdd\x15\xf3\x25\x8b\x7e\x92\x11\x73\x65\x0e\xb9\xba", + "\x68\xa1\x24\x9d\x64\x87\xf4\x6c\x10\x7b\xdb\x84\x17\x27\x1e\x26", 1, + 128, "\x8a\xf8\xef\x74\xbd\x25\x21\xf6\xe7\x11\x95\x0b", 96 }, + { 192, 128, 147, + "\x36\x42\x89\xec\xcf\xf5\x3e\xfb\x98\xda\x8e\xe4\x37\xd6\xd3\x89\x77" + "\x5a\x3c\xf2\xb4\xc9\x70\xcd", + "\x66\x47\xb5\xda\x47\xd1\x69\x83\x20\x5c\xe2\x44\x88\x0c\xa6\x05" + "\x19", + "\x73\xb9\x8c\x93\x98\xc7\x0a\x83\x3e\x3d\xbe\xf4\x55\x6d\x8f\xa7", 1, + 136, "\x72\x45\xa5\x15\x65\x6a\x22\x36\xc3\x8c\x85\xaa", 96 }, + { 192, 128, 148, + "\x4c\x41\xb2\xfe\x3e\x60\xbc\xe3\xa9\x45\xe2\xfa\xcd\x97\x54\xc4\xf6" + "\x0b\x19\x2d\xa1\xfe\x11\x0e", + "\x11\xc1\x07\xe4\xb3\x02\xfd\x91\xea\x92\x19\xd7\xfc\x00\x79\xe8\xac" + "\x4b\xdc\xfc\x71\xff\x02\x7a", + "\xfa\xcd\x4d\x12\xdf\xaf\xd6\xbd\x1f\x42\x1a\x5c\x4c\x62\x17\xac", 1, + 192, "\x4a\xb5\x1f\xc4\x59\xe4\xf4\xa5\xb4\x4d\xcf\xa3", 96 }, + { 192, 128, 149, + "\x22\x61\x96\x23\x65\xe2\xeb\x22\xe1\x20\x02\xe0\x53\x95\x0b\x14\x29" + "\x53\xa6\xce\x60\x2d\x19\x59", + "\x44\x21\x60\xc2\x8d\x0c\x08\x32\x87\x32\x19\xdf\x81\xf1\x93\x23\x34" + "\x2b\xd5\x1b\x56\x68\x45\xaa\x3b\xff\x39\xd3\x02\xdb\x3f\x09\x85\xc9" + "\x16\xf7\xc7\x7c\x6d\x63\x87\xd9\x3c\xbc\xdc\x40\xbd\x6e\x45\x8d\x67" + "\xea\x5e\x79\x3a\xc6\xda\x12\x2d\x32\x31\x49\xf1\xb8\x4b\x8c\xd9\x94" + "\xb6\xd8\xbf\x01\x5d\xfa\xcf\x47\x5f\xc9\xc5\x09\xec\x20\xad\xae\x68" + "\x3e\x57\xc2\xb9\x96\x8a\x64\xcf\x56\xde\x90\xcb\xac\x6b\x85\xb9\x67" + "\x92\x96\xa3\x09\x7d\xfc\x76\xfb\xfd\x67\xcb\xc3\x37\x69\x7f\x93\x15" + "\xc2\x3f\xe0\xa4\xc0\xd4\x4d\x79\xc4\x73", + "\x54\xb1\x0f\x53\xa3\x04\xfd\x1b\xee\x6f\x9f\xb2\x46\xca\x40\xef", 1, + 1032, "\x35\xcb\xde\x01\xe0\x7a\x10\xda\x05\xba\x2a\x34", 96 }, + { 192, 128, 150, + "\x5a\x94\x6c\x7e\x78\xad\x42\x7e\x00\xbc\x52\x91\x7f\xd2\x7f\x00\x9c" + "\x98\x7a\x87\x1b\x22\xf6\xf8", + "\x8a\xd8\xea\xf0\xec\xc7\x29\x3f\x9a\x50\x42\xc9\xc7\xb7\xc0\x40\xa2" + "\x39\xa8\xd8\xc6\xee\xd4\xcf\x1c\x2c\x5c\x97\xb5\x68\x62\x67\x11\x10" + "\xdd\xc0\x27\xaf\x7b\x9c\x6f\x87\x49\x66\x3e\xbc\xe7\x34\xa4\x42\xd6" + "\x6d\x1e\xc2\x13\x3f\xc1\xe9\x23\xd9\xf6\xb5\xd0\x9b\x06\x9a\xde\x86" + "\x61\x05\x10\x05\xe5\x4f\x24\xe2\x04\x5a\x40\xd6\x5a\xd4\xa1\x2e\xd0" + "\x57\x9d\xd1\x92\xaf\x76\x11\x8b\xa3\x95\x33\x5f\xb4\xa9\x46\x64\x81" + "\x2b\xe3\xe5\xde\x91\xf4\xa4\xa6\x19\x6e\xae\x48\xc6\x39\xa8\xd2\xba" + "\xf5\x4b\xc9\xe4\x7b\xf1\x44\x86\x2f\x03\x77\xd5\x1b\x87\xab\x3e\xf6" + "\x49\xbe\xc7\xe7\x8e\xf2\x78\xf6\x2a\xea\xe1\xb5\xdf\x61\xd2\x1b\xec" + "\x30\x4a\xb1\x79\x3b\x2b\x95\x8d\xc9\xd9\xc5\xaa\xe3\x8d\x35\x43\xbf" + "\x6d\x29\xef\xf2\x5f\xae\xcb\x7c\xf8\xca\x1e\x75\x84\x70\xb4\xb3\x0b" + "\x4f\xc0\xba\xe2\x56\xff\x83\x01\xf6\x31\x30\xc0\x28\x74\xe6\x7c\xe7" + "\x25\x87\x06\xea\x3a\x45\xf2\xac\x11\x66\x4a\xcb\x43\xed\x90\xff\xc8" + "\xf7\xbd\x73\x6b\x50\x27\x56\x5c\x1e\xb9\x8e\x54\xd9\xf0\x0b\x34\xc8" + "\xf3\x89\x62\x6f\x83\xda\x9f\xfb\x9f\xd0\xfd\x48\x45\xab\x58\x41\x56" + "\x37", + "\x5a\x79\xd3\x22\x8c\x67\x30\x5d\xfc\xf1\x5d\x68\xae\x03\x28\x60", 1, + 2048, "\xbf\x96\x93\x96\x00\x66\x0e\x86\x25\x42\x03\x62", 96 }, + { 192, 128, 151, + "\x57\xc0\x7c\x71\xe6\x13\x61\xa2\xe7\x5a\xf1\x84\x82\xd9\xbd\xe5\x83" + "\xff\x2b\x09\xb5\xc1\xda\x77", + "\x94\x6c\xcc\xb8\xff\x9b\x67\xb3\x56\x50\x66\x12\x5a\x59\x52\x98\x57" + "\xac\x62\xc3\xe3\x93\xd4\x8d\xe1\x74\x1e\xfe\x35\x1c\xa1\x94\x85\xfa" + "\x4e\x45\xb4\xcf\x08\x31\xbf\x30\xcc\xef\x3d\x38\x9c\x0b\xb1\xcd\x16" + "\x25\x7a\xc6\x47\xa7\x7b\x96\x5a\xd9\x03\x62\xcf\x27\xbd\xa5\xe4\x2f" + "\x16\x00\x5c\x69\x7e\x9e\x46\x5c\xc1\x28\x73\x88\xab\xd4\x5e\xd7\x24" + "\xc6\xf0\x1a\x87\x01\xd8\x6e\xb2\x24\x39\xd0\xb4\xe6\x09\xf7\xf8\x10" + "\x37\xed\x59\xa1\x36\xe2\x0e\x8b\xb8\x70\xfe\x57\x74\x44\xf6\xde\xa2" + "\x15\x3b\x4e\x2c\x0c\x0c\x30\x0c\x14\x04\xe8\x31\x51\x0b\x93\x0b\x68" + "\x19\x82\xb9\xcd\xd5\x13\xb3\x94\x7f\xcf\x9e\x07\x9d\xac\xd2\x0f\xa7" + "\x8a\x08\x19\x9c\x77\x72\x75\x56\xd7\x24\x80\x93\x2f\x15\xe2\x01\x2f" + "\xf0\x3e\x59\x3b\x98\x7c\x89\xa1\xdd\x1a\xb5\xae\x0b\xd2\x3d\xe0\xd9" + "\xfc\x3c\x1c\x37\xe0\xfd\x6c\x46\xc0\xec\xb1\x49\x5d\xc3\xe6\x2d\xc6" + "\x7b\xd7\xc5\x15\x0d\xbf\x5f\xcb\xc6\xf6\xfe\xcc\xd5\xd4\x7b\x8d\x6b" + "\x59\x3a\x14\x3d\xf5\x43\x91\xed\x1f\xcc\x8e\xaf\xe5\x45\x67\xef\xc6" + "\x30\x87\x49\x4e\xad\xaa\xdb\x0b\x61\x98\x19\x4e\x1f\x5a\x5d\x9b\xcf" + "\x1d\xe6\x08\xa5\x3f\x42\x2b\xae\x1a\x40\xc9\xa6\x6d\xfb\x47\xc2\x6e" + "\xfa\x85\xb8\x46\xa2", + "\x8f\x13\x64\xd1\xd9\xe5\x82\x6a\x79\x25\x4c\xa6\x02\xbb\x1e\x4c", 1, + 2216, "\x7c\xc9\x7a\x3e\x4f\x65\x71\xae\xdd\x69\xee\x95", 96 }, + { 192, 128, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x35\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x36\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8e\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xb4\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xec\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x76\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x98\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\x45\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x01\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 163, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x57\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 164, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x02\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 165, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x54\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 166, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x8f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 167, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\x3e\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 168, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x10\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 169, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2a\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 170, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x91\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 171, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\xab\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 172, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x34\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 173, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xfc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 174, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbd\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 175, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5b\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 176, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe2\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 177, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xce\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 178, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe1\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 179, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcd\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 180, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\x63\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 181, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\x4f\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 182, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x76", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 183, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7e", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 184, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x75", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 185, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7d", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 186, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\x37", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 187, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\x3f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 188, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x0f\x11\x14\xbc\xd4\xe3\x4b\xab\xf7", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 189, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\xbe\x2b\xdc\x5a\x8f\xcf\xc9\x38\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 190, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x35\xed\x91\x18\x00\xfd\xfa\x0f\x10\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 191, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\x77\x1d\xc5\x56\x3b\x63\xbe\x2a\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 192, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x98\x00\xfd\xfa\x8f\x11\x14\xbc\xd4\xe3\x4b\xab\x77", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 193, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\x45\x56\x3b\x63\x3e\x2b\xdc\x5a\x8f\xcf\xc9\x38\x7f", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 194, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x34\xed\x91\x18\x00\xfd\xfa\x8f\x11\x14\xbc\xd4\xe3\x4b\xab\xf7", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 195, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8c\x77\x1d\xc5\x56\x3b\x63\x3e\x2b\xdc\x5a\x8f\xcf\xc9\x38\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 196, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xcb\x12\x6e\xe7\xff\x02\x05\xf0\xee\xeb\x43\x2b\x1c\xb4\x54\x88", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 197, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x73\x88\xe2\x3a\xa9\xc4\x9c\x41\xd4\x23\xa5\x70\x30\x36\xc7\x80", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 198, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 199, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 200, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 201, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 202, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xb4\x6d\x11\x98\x80\x7d\x7a\x8f\x91\x94\x3c\x54\x63\xcb\x2b\xf7", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 203, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\xf7\x9d\x45\xd6\xbb\xe3\x3e\xab\x5c\xda\x0f\x4f\x49\xb8\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 204, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x35\xec\x90\x19\x01\xfc\xfb\x0e\x10\x15\xbd\xd5\xe2\x4a\xaa\x76", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 205, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\x76\x1c\xc4\x57\x3a\x62\xbf\x2a\xdd\x5b\x8e\xce\xc8\x39\x7e", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 192, 128, 206, + "\x17\x5b\x5a\x75\xe4\xfd\x7f\x9c\x1f\x99\x77\x81\xb9\x76\xa7\x2a\x01" + "\x0d\x69\x55\x9a\xbb\x0d\x1e", + "\xfc\xcc\x74\x87\xa8\xae\x2a\x85\xf6\x05\x5f\x02\x11\xa7\x3d\x0b", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, "\x55\x0e\x0a\x2d\x5c\xb9\x87\xba\xf4\xb9\xc3\x24", 96 }, + { 192, 128, 207, + "\x17\x5b\x5a\x75\xe4\xfd\x7f\x9c\x1f\x99\x77\x81\xb9\x76\xa7\x2a\x01" + "\x0d\x69\x55\x9a\xbb\x0d\x1e", + "\xc2\xeb\xff\x3f\x88\xc5\xd3\x7f\x41\x80\x02\x11\xd2\xbc\x60\xdd", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, "\x55\x0e\x0a\x2d\x5c\xb9\x87\xba\xf4\xb9\xc3\x24", 96 }, + { 192, 128, 208, + "\x9f\x93\x4f\xa4\xd1\xf3\xcc\xa7\x44\xc8\x1c\x87\x93\x72\x45\xd7\x20" + "\x76\x25\xb6\xb2\xac\x3d\x84", + "", + "\x9c\x04\x53\x1b\xce\xaf\x43\x8d\xc1\x34\x2f\x3c\x5f\x7f\x62\xb8", 1, + 0, "\x19\x12\x4f\xf0\x81\x47\x17\xab\xea\xa2\xe1\xfc\x9f\xc8\x86\xed", + 128 }, + { 192, 128, 209, + "\x0a\xe0\xba\x46\x74\xc5\xc6\x85\x35\x4e\x24\x1b\xc8\x60\x34\xcb\xef" + "\x80\xf8\x8a\xcf\x1f\x74\x88", + "\xc7", + "\xf2\xf5\xcd\xa5\x95\xfb\x7d\x7b\x0a\x7d\x69\x9b\x6a\xbc\x7f\x52", 1, + 8, "\x08\x32\x66\xeb\xda\x2b\x3d\xb8\xb1\x4c\x23\x5e\xb2\x97\x59\x88", + 128 }, + { 192, 128, 210, + "\x9f\x47\x9e\x91\x18\xe5\x60\x3b\xf1\x22\xfc\x5e\x2b\x8b\xf3\xe4\x48" + "\xac\xb9\xfb\xde\x5e\x16\xdc", + "\x96\xaf", + "\x25\xfc\x87\x7b\xc2\xfd\x80\x04\x92\x84\x33\x16\xa0\x9b\x55\x82", 1, + 16, + "\xd2\x64\x27\xfc\xa9\xc8\xe1\x71\x7a\xcd\x34\xad\x3c\xc5\xd5\x42", + 128 }, + { 192, 128, 211, + "\x41\x9a\x56\xe8\xf4\x1f\x33\x7d\xee\x4a\x63\xed\x38\x6a\x8e\xd7\xe8" + "\x5b\x25\x96\x44\x94\xb0\x75", + "\x8a\x3d\xc8\xad", + "\x3d\x13\x1a\xa1\x25\x38\xc6\x33\x22\x83\x2e\xfb\xc8\x35\x13\xdf", 1, + 32, + "\xab\x25\x03\x31\x65\xc6\xcf\x97\x60\xd1\xa5\xa4\x43\x91\xe7\x7a", + 128 }, + { 192, 128, 212, + "\xee\xc6\x72\xd9\xe7\xb9\xbd\x93\x3e\x5e\xe3\x6a\xe3\x81\xe6\x5b\x87" + "\x2a\xc8\x37\x3c\x3e\x56\x78", + "\x80\x9d\xe8\xdb\xe2\x81\x47", + "\xb6\xb7\x69\xa1\x1d\x31\x9c\xb2\x2b\xe9\x2f\x1d\x33\xad\x8c\x4c", 1, + 56, + "\x51\xe0\x64\x2b\x8e\x49\x45\x81\xdd\x04\xc5\xb1\x6b\xd8\x8b\x89", + 128 }, + { 192, 128, 213, + "\x41\x5c\x12\xb7\x84\x05\xdc\x19\x16\xb9\xf6\x5c\xc1\xd9\x60\xb6\x41" + "\x06\x2a\xb5\x89\x26\x09\x9a", + "\xb5\xde\x1b\x30\xf7\x22\x9f\x1d", + "\x2d\x71\x22\xb7\x4f\xeb\xd2\xe5\x7b\xd7\xe1\xc0\xcc\xf7\x22\x9f", 1, + 64, + "\x80\xd3\x37\x0f\x38\xc8\xd9\xc6\x74\x16\x05\x00\xa3\x36\x27\xf3", + 128 }, + { 192, 128, 214, + "\x8c\xf2\x66\x62\x23\x06\x0e\x2a\xde\xce\x28\x96\x9a\x8d\x88\x9e\xce" + "\x7f\xe3\x2c\x0a\xe3\xb9\xc6", + "\xfb\xfd\x57\x3c\x9f\x12\x14\xf7\xc6\x50\xbc\xf5\xe7\x2b\xac", + "\xbd\x53\xa4\xc7\xda\xa8\xd0\x8e\xe1\x01\xca\xc5\x86\x1c\x7f\xf9", 1, + 120, + "\xcf\x0b\xd9\xd8\x76\x7a\x3e\xae\x62\x15\xaf\x33\xb7\x21\x0f\x99", + 128 }, + { 192, 128, 215, + "\x90\xd1\x19\x6f\xf0\x04\xc2\xc1\x84\xe6\x9f\xbe\x4c\x51\x2a\xfe\x5d" + "\xd5\x30\x60\xd0\xaa\x29\xee", + "\xe0\x3b\xa4\xce\x91\xba\xc7\x71\x50\xda\x73\x11\x25\xb4\x63\x4e", + "\xae\xc2\x77\x21\xe0\xe5\x51\x30\x75\xa6\x89\x53\xcc\x97\xca\xb1", 1, + 128, + "\xb1\x65\x71\x49\x8f\x84\x36\x8f\x1c\x24\xb6\x31\x78\xb8\x8a\x5b", + 128 }, + { 192, 128, 216, + "\x2f\x08\x4b\x35\x67\xbe\xca\xf9\x1d\x1a\x9a\x39\x1a\xac\x5e\x45\x76" + "\xf9\x03\x4e\x4a\x11\x5f\x8c", + "\x8f\x86\x08\x47\xf1\x8d\x38\x7b\x64\x66\x1c\xec\x46\x20\x8c\x70" + "\x50", + "\x28\x3b\x96\x34\x4c\x2d\xe9\x37\xf4\xa7\x27\x9a\x65\x98\xe6\xbd", 1, + 136, + "\x82\x5e\x71\x3c\x0b\xcb\xe7\x89\x2d\x61\x7d\xe7\x98\xab\x9b\xd9", + 128 }, + { 192, 128, 217, + "\xe4\xdb\x14\xa3\xfc\xdb\x48\xfa\x98\x49\xf0\x3b\xf5\x2a\x4b\x44\xd3" + "\x9e\x9e\x6b\x33\x97\xdb\x59", + "\xd7\xd6\x63\x79\x83\x17\x60\x3e\x56\xbc\x8b\x22\xfc\xec\x7a\xf3\xa6" + "\xc1\x5a\x87\x92\x68\xcb\xc7", + "\x73\xc9\x5c\x43\x0d\x20\x7c\x1b\x53\x32\x68\x65\xb7\x98\x0c\x04", 1, + 192, + "\x5d\x53\x8b\x3a\xc4\x1f\xcf\xc5\x60\x65\xe3\xfa\x75\xc5\x33\x85", + 128 }, + { 192, 128, 218, + "\xa1\x4e\x5d\x0b\x28\x23\xed\x9e\x3a\x19\x1a\x3f\x89\x60\x05\xd9\xbf" + "\x18\x65\x19\xdb\xb2\x61\xb5", + "\x6e\x9f\xab\x68\xc0\xb2\x44\x37\x8d\x50\x3b\xf5\x82\xf4\x9a\x43\xbe" + "\x6c\xb2\x06\xed\xf6\xee\x02\xbb\x55\xa3\x6b\x9c\x35\x33\x52\xc1\x9a" + "\xd0\xd2\xa2\x04\x88\x09\xe4\xdd\xc1\x60\x48\x5a\x2f\xa9\xf5\x43\x88" + "\x51\xeb\x9a\xa5\x0b\x15\x3e\x4a\x2d\x9b\xe1\xb1\xa1\x77\x2e\xae\x7b" + "\x4d\xce\x9d\xd6\x49\xad\x43\x57\xa2\xe2\xfb\x4e\x90\x24\x53\x9b\x01" + "\x4c\x94\xd2\x00\x5b\x5f\xa3\x2e\x47\xc5\x0e\x9f\x91\x4f\x08\x4e\xec" + "\x58\xf6\xe1\x08\xa9\xde\x3c\x29\xcc\xe5\xd9\x29\x80\xf4\xf4\x91\x1f" + "\xc0\x0e\x96\x13\xb2\x63\xcc\xa8\xa5\x97", + "\x8a\xdb\x4c\x0c\x1d\xc1\x5e\xcb\xf6\x6b\x43\x1a\x03\x0e\xc5\x14", 1, + 1032, + "\x6d\xac\xfb\x15\xf7\xe6\xfb\x26\xe4\x23\xc0\xee\x05\xf6\x86\xba", + 128 }, + { 192, 128, 219, + "\x0c\xf9\x1c\xfd\xf3\xb0\x64\x58\x1d\xd0\x15\x34\xa3\x04\x29\x67\x74" + "\xc1\x7b\xa3\x0c\x5e\xa4\xee", + "\xb4\x57\x70\xb4\x39\xc4\x87\x75\x76\x4d\x12\xc9\xad\xbf\xcd\x2b\xe5" + "\x9a\xab\xc9\x8e\x74\x06\xdd\xd0\xc0\x5c\xb2\xde\xbb\xda\x43\x2a\x95" + "\x80\x3c\x35\x87\xe0\x1d\x67\x99\x0d\xd4\x93\xf9\x6f\xbd\x3f\x4a\x33" + "\xae\xca\x61\x71\xde\xc4\x22\x0b\xb8\x26\x47\xb3\x44\xd0\x22\x58\x56" + "\x73\xd3\xfc\x40\x78\x7f\xb3\xcc\xbf\xf5\x1c\x16\x62\xb9\xf9\xda\x1b" + "\x79\xbe\xff\x7b\x6f\x05\x87\xc6\xcd\xa6\x64\xb7\x3b\x48\xce\xfd\xcf" + "\x7a\x41\x57\x2f\xdd\x51\x49\x09\x7c\x31\xa6\x5c\x74\x82\xf3\xc7\x69" + "\x49\x3d\x6f\xb9\xbf\xca\xf7\xee\x20\x80\x59\x32\xb9\xba\x2b\xff\xee" + "\x44\xc1\xd1\x8b\x9d\x59\xad\x80\x8b\x71\x46\x69\xca\xe5\x36\x03\xa4" + "\x51\x88\x13\xb3\x1d\x7d\x4f\x03\x79\xcc\xcb\x1e\x0d\xdd\x09\xf0\x0d" + "\x9c\x06\xff\x2c\x69\xfd\xb4\x2a\xe4\x61\x67\xd6\xa5\x1e\x96\x78\x1e" + "\x6b\x91\xbe\xd4\x22\x90\xf3\x60\xb5\x01\x97\x09\x16\xa3\x66\x13\xba" + "\x10\x54\xf6\x38\xba\x59\x2e\xc8\x69\xc0\x85\x75\x7b\x94\x2d\x40\xd4" + "\xdc\xff\x3e\xeb\x3b\x99\xbd\x81\x81\xb4\x0e\x7c\xec\x75\x28\xbd\x48" + "\x73\xe4\x27\xbb\x67\xc8\xfa\x8b\xc5\xf4\x7e\xba\x7e\x55\xc9\x05\xe2" + "\x24", + "\xd5\xe0\x9f\x52\x98\x6b\x33\xaf\x35\x77\xe9\x75\xdd\x9a\x90\x16", 1, + 2048, + "\xa9\x8d\x89\x33\x2c\x05\xb1\x3d\xb1\x87\x48\x48\x11\x19\x8a\xb9", + 128 }, + { 192, 128, 220, + "\x93\xce\xcc\xe9\x6d\x37\x40\x62\x86\xa8\x81\xdb\x0d\x0d\x53\xce\x8f" + "\x7b\xf5\x3f\x4b\xdc\x74\x6a", + "\xcc\xca\x5c\x74\x61\xb8\x10\xd4\x14\xe2\x4f\xec\xf4\xd3\xee\xac\x3a" + "\x38\xc2\x96\xf7\xb8\x88\xf4\x36\x87\xd8\x8c\x8b\xa8\x19\x17\x98\xfc" + "\x43\x1f\x8b\x6c\x05\x76\xad\x1a\xa8\x41\xd3\xc9\x54\x50\x60\xdf\xfb" + "\x76\x75\xc8\xcf\xa2\xfa\x41\x98\x89\x57\x2b\x39\xe5\xed\xf1\xa4\x0b" + "\x54\xa6\x44\x30\xa6\x06\xac\x7f\x19\xf4\xeb\x28\x7a\xd2\x60\x9e\x23" + "\x80\xd7\xfe\x48\x34\xf5\xae\xac\x98\x32\xb5\xed\x0c\x66\x99\x4a\xc1" + "\x7e\x0d\x11\xbb\x57\x64\x3a\xa5\xf6\xad\x44\xcf\x57\xc5\x4d\xcd\x94" + "\x93\x1e\x52\x85\xf9\x3e\x03\x10\x79\x13\xae\xa5\x04\x8a\xd0\x4a\x4e" + "\xe8\xe3\x51\x0a\xc0\x30\x5a\x13\xa5\x74\xd7\xc9\x55\xf2\x91\x2e\xf2" + "\x39\xb4\x0d\xcf\x5a\xd7\x43\x8a\x19\xad\x7b\xec\xe2\xe7\x9e\xbd\x13" + "\x5e\x16\xed\x2b\xa5\x9f\x7e\x78\x57\xdc\x64\x80\xf7\x80\x30\x31\x57" + "\x89\x57\x84\xa7\x9f\x04\xf7\x84\x15\x07\x07\x88\x05\x2d\xcd\x74\x58" + "\xd9\xff\x86\xa7\x05\x68\x4b\xce\x83\xd2\x9f\xc8\xc0\x96\xfa\xc2\x98" + "\x53\x60\x35\x00\xb4\x9a\xce\xc5\xd4\x38\xe7\x60\x72\xde\x71\x84\x11" + "\x93\xf8\x44\x09\x81\x69\x3b\x62\x5b\x30\x82\xe2\xba\x89\x8d\xcc\x79" + "\x23\xa6\x0c\x8d\x95\x3c\xb3\x9f\xa5\xcb\xdd\x42\xea\xb1\x37\x33\x26" + "\x12\x33\x8c\xe2\x0d", + "\x66\x0b\x84\x72\x7a\x22\x9e\x09\x22\xb5\x61\xe8\xd0\xdd\xbd\x5e", 1, + 2216, + "\xbf\x13\x68\xbb\x09\x37\x3b\x1e\x0e\xed\x89\xee\xea\x93\x21\xc1", + 128 }, + { 192, 128, 221, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa0\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 222, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x18\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 223, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa3\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 224, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x1b\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 225, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x21\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 226, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x99\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 227, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1c\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 228, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x86\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 229, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x1a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 230, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\xc7\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 231, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x87\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 232, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd1\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 233, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x84\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 234, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd2\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 235, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\xfe\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 236, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\x4f\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 237, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbe\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 238, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x84\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 239, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\x3f\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 240, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x05\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 241, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x3c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 242, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xf4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 243, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb2\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 244, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x54\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 245, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf5\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 246, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd9\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 247, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf6\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 248, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xda\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 249, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\x74\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 250, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\x58\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 251, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x92", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 252, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x9a", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 253, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x91", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 254, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x99", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 255, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\xd3", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 256, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\xdb", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 257, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\x7e\xbf\x1c\xb3\x87\xf4\xcf\xdc\x13", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 258, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\xcf\x85\xd4\x55\xdc\xd8\x4d\x4f\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 259, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa0\x1d\xa3\x9a\x86\xf6\x46\x7e\xbe\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 260, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x18\x87\x2f\x47\xd0\x30\xdf\xcf\x84\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 261, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x1a\x86\xf6\x46\xfe\xbf\x1c\xb3\x87\xf4\xcf\xdc\x93", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 262, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\xc7\xd0\x30\xdf\x4f\x85\xd4\x55\xdc\xd8\x4d\x4f\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 263, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa1\x1d\xa3\x9a\x86\xf6\x46\xfe\xbf\x1c\xb3\x87\xf4\xcf\xdc\x13", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 264, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x19\x87\x2f\x47\xd0\x30\xdf\x4f\x85\xd4\x55\xdc\xd8\x4d\x4f\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 265, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x5e\xe2\x5c\x65\x79\x09\xb9\x81\x40\xe3\x4c\x78\x0b\x30\x23\x6c", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 266, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe6\x78\xd0\xb8\x2f\xcf\x20\x30\x7a\x2b\xaa\x23\x27\xb2\xb0\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 267, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 268, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 269, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 270, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 271, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x21\x9d\x23\x1a\x06\x76\xc6\xfe\x3f\x9c\x33\x07\x74\x4f\x5c\x13", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 272, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x99\x07\xaf\xc7\x50\xb0\x5f\x4f\x05\x54\xd5\x5c\x58\xcd\xcf\x1b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 273, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa0\x1c\xa2\x9b\x87\xf7\x47\x7f\xbe\x1d\xb2\x86\xf5\xce\xdd\x92", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 274, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x18\x86\x2e\x46\xd1\x31\xde\xce\x84\xd5\x54\xdd\xd9\x4c\x4e\x9a", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 192, 128, 275, + "\xdf\x23\xcd\x79\x69\x38\x87\x21\x43\x7b\xa1\x3d\x56\x2a\xc4\x59\x39" + "\x2a\x8f\xb8\x8f\x51\x92\x5e", + "\x3c\x11\x58\x1c\x4b\x96\x64\x92\x6d\x77\xe1\xa7\xd1\x87\xb0\x0a", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, + "\x73\x5c\xdb\x81\xfb\xbd\x61\x72\xcb\x7f\xc0\xca\xe1\x3b\x7a\xc9", + 128 }, + { 192, 128, 276, + "\xdf\x23\xcd\x79\x69\x38\x87\x21\x43\x7b\xa1\x3d\x56\x2a\xc4\x59\x39" + "\x2a\x8f\xb8\x8f\x51\x92\x5e", + "\x69\x58\xfb\xba\x11\x73\xef\xad\x3c\x90\x03\xba\xae\x27\x8d\x3d", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, + "\x73\x5c\xdb\x81\xfb\xbd\x61\x72\xcb\x7f\xc0\xca\xe1\x3b\x7a\xc9", + 128 }, + { 256, 128, 277, + "\xd7\x96\xa0\x5c\xef\x10\x9d\x0e\xf3\xa8\x11\x07\xa1\x1b\xd3\x69\x15" + "\x93\xc1\x1a\x0c\x98\xeb\x4d\x31\xad\x91\x3b\x1e\x4e\x93\xbe", + "", + "\x75\xb0\x98\xb6\x02\x51\x55\xfd\x69\x9e\x66\x90\x8c\x40\x01\x56", 1, + 0, "\x36\xdc\x70\x0a\xbd\x99\xd9\x9d\x8f\x5b\xc0\xce", 96 }, + { 256, 128, 278, + "\x31\xe3\x9f\x00\x79\xa1\x82\xc9\xb2\xba\x38\xb8\xbc\xc4\x85\x18\xa2" + "\xdf\xf1\xdf\x88\x52\x85\xfc\x05\x42\xf8\x3b\x25\x5a\x52\x0c", + "\x45", + "\x1e\xe0\x76\xf7\x08\xe6\x4c\xc5\x5c\x71\xd8\x63\x3e\xc2\x92\x31", 1, + 8, "\x2c\xc7\x45\x9f\xa6\xa5\x3f\x8c\x9a\xf1\x41\x54", 96 }, + { 256, 128, 279, + "\xcc\xa7\x77\xcf\x57\x59\x24\x60\x27\xd6\xe4\xc7\x3b\xaf\x3d\xb9\xb9" + "\xd8\x34\xc3\x70\x32\x03\x57\xd1\x69\xe5\x31\x93\x60\x61\xf4", + "\xc0\x0b", + "\xeb\xe4\xe5\x69\x51\xe1\xef\x7c\x72\x1f\x42\x9e\xec\x78\xc1\x5e", 1, + 16, "\x5b\x48\x47\xb7\x7b\xab\x2a\x6e\x54\x60\xda\x65", 96 }, + { 256, 128, 280, + "\xa2\x91\x2f\xfd\x04\xdd\x0c\x12\xf9\xb1\x44\x31\x9a\x46\x5a\xf5\x72" + "\xb4\xbe\x8a\xa0\x91\x6b\xaa\xa6\xed\xb9\x62\x05\xee\x8b\xb1", + "\xff\xc2\xfd\x8c", + "\x2d\x8b\x37\x38\x32\xd0\x29\x3f\x10\x37\x40\x9f\x6a\xd0\xc4\xf1", 1, + 32, "\xac\x26\xc6\xc0\x1e\xc2\xdf\x59\x07\xb7\x1a\xf2", 96 }, + { 256, 128, 281, + "\x45\x76\xc1\xe2\x47\x0b\x55\x7f\x85\xae\x2d\xef\xc0\x78\xcb\x9e\xea" + "\xc5\x2b\x32\x35\x13\x7f\x8b\xce\xfd\x9c\x11\x6b\x80\x4d\x2c", + "\x4e\x28\x4a\xb6\xf6\xe4\x9b", + "\xeb\x91\x0a\xfe\xc0\x79\xb3\xda\x46\x01\xfe\x6c\x9d\xe3\x29\x69", 1, + 56, "\x20\xd2\x9d\xd5\x28\xb2\x6a\x71\x6b\x11\x22\xd4", 96 }, + { 256, 128, 282, + "\xbe\x85\x5e\x5f\x44\x77\xa5\xf0\x6e\x64\xe9\x69\xcf\x0f\x7a\xad\x23" + "\x99\xd1\x0a\xc6\x30\x1b\xd2\x96\x18\xb5\x30\x31\x53\x04\xaa", + "\xb3\x5e\xa0\x0e\x01\xf3\x04\x38", + "\xba\xdb\xcd\xfb\x66\xe6\x8b\x94\x9b\xcb\x31\x4b\x36\x7c\x82\xcf", 1, + 64, "\x1d\xc8\xf6\xcb\xa3\x82\x8c\xb8\x86\x2e\xa9\x0d", 96 }, + { 256, 128, 283, + "\x30\xf1\x06\x09\xfc\xf8\xae\x39\x8f\xa5\x0f\xb2\x80\x7e\x0b\x60\x5f" + "\xe1\xf5\x96\x2f\x7c\xd7\xd6\x63\xfa\x2c\x50\xbb\x0e\xd5\x37", + "\xc6\x6f\x86\xfb\xee\x5e\xda\x6a\x0a\xc4\xf6\x4d\x7e\xf4\xb8", + "\xb7\xe5\xec\xd2\x35\xa4\x06\x15\x2e\xdd\x49\x09\xf1\x63\x52\x06", 1, + 120, "\xf2\x51\x34\x5f\x80\x11\xb4\xf7\xfd\xd5\x9a\xa2", 96 }, + { 256, 128, 284, + "\xb5\xca\x16\x87\x71\x37\xe3\x59\x5d\x05\x60\x01\xb1\x82\xeb\x16\x51" + "\xe4\xae\x6a\xf0\x80\xce\x7e\xe0\xa0\xc5\x65\x1a\x09\x46\x03", + "\xea\x94\xe0\x62\xb1\x0e\x5d\xfd\x2e\xa9\x93\xcb\x6a\x10\x3d\x98", + "\x21\x43\x9f\xb4\x07\x24\x7a\xc0\xa9\x8a\x30\x2d\x6c\xff\x4b\x0f", 1, + 128, "\x1f\xc2\x12\x8c\xf2\x47\xfe\xcb\x74\x68\x59\xfc", 96 }, + { 256, 128, 285, + "\xb5\x68\x42\x55\x46\x3e\x57\x88\xbe\xc2\xd7\x5c\x8c\x46\x3a\x65\x8c" + "\x79\x42\x8d\x49\xfb\x2a\xf5\xf1\x25\x6c\x6b\xb1\x71\x1f\x33", + "\x32\x9f\x30\x4b\x5d\x32\xe4\x81\x86\x82\x23\x06\xd6\x64\x1c\x09" + "\x0a", + "\xff\x54\xec\xd2\x6a\xa9\x4a\xac\xd6\xd9\x2b\xd9\xf6\x32\x3f\xf9", 1, + 136, "\x93\x88\xc4\xbf\x74\x15\x7c\x59\x01\x80\xe0\xc1", 96 }, + { 256, 128, 286, + "\x76\x07\x4f\xef\xee\x14\x8c\xd8\x87\x3d\x23\x51\x17\x5b\x8f\x0b\x46" + "\xba\x38\x81\xf0\x7d\x5b\xd7\xe6\x7a\x65\x9e\x01\x83\x0a\x40", + "\x8c\x03\x37\xfc\x94\x01\xe6\xa5\x8e\x40\x8a\x11\xdf\xf5\x7a\xb2\xdd" + "\xc1\x7c\x16\x82\x77\x8f\xa9", + "\x36\x86\xf1\xf8\xc1\x8a\x94\x4c\xb9\x70\xf0\x89\xc9\x39\xcc\xd5", 1, + 192, "\x0c\x5b\x68\xa8\xc3\xfd\xf6\x45\xe3\xac\x1e\x56", 96 }, + { 256, 128, 287, + "\xe2\xa7\x2f\x64\x30\x1e\x4d\xb6\xe8\xc0\x5c\x31\x2f\x2c\xea\x92\xfb" + "\x8b\x06\x3e\xb7\xfe\xc2\x3e\xa4\xe9\x8c\x46\xfd\x04\x84\xd6", + "\xc7\x0f\xb4\x3d\xe3\xaf\xf1\x1d\xbc\xab\x9b\x6c\x26\x7c\x4b\x5b\x35" + "\xcd\x9c\x08\xec\x69\x99\x41\x9a\x67\x71\xbc\xc7\x34\xae\x86\x50\xa5" + "\x6a\x42\x99\xc2\x10\x5b\x32\xbe\x02\x18\x1e\xaa\xeb\xe7\x9f\x07\x47" + "\x76\x15\x73\x95\x9d\x2b\xd7\x1b\x08\x74\xde\x54\xec\x2f\xd1\x7b\xfd" + "\x87\x1a\xcd\x76\x6d\x53\x13\xcb\xff\xbe\x26\xce\xd0\x83\xf5\x22\x4f" + "\x27\x77\xcd\x65\xac\x4d\x2d\xb0\x8f\x21\x3c\x0f\x7a\x5b\xcc\xbc\x19" + "\xb9\xbd\x42\xab\x64\x11\x61\x42\xf4\xd9\x4b\x09\x28\x0e\xe5\x84\x74" + "\x40\x55\xf8\x1e\x2f\xef\x29\x1b\x36\xae", + "\x54\x54\x83\x20\xe2\x80\x1d\xad\x45\x68\x32\x97\x56\x58\x67\x06", 1, + 1032, "\x89\x98\xac\x05\xe1\x1c\x96\x4f\x22\x09\x0c\xe7", 96 }, + { 256, 128, 288, + "\xd8\xd2\x86\xaf\x2b\x74\xab\x17\xbe\x1b\x23\x94\x84\x32\x82\xd2\x4a" + "\x19\xa2\x72\xa7\x1b\x0a\xc9\xb0\x5a\xbe\x82\x6e\xc7\xb9\xcd", + "\xa9\xdb\x39\x76\x5a\xd3\x0e\xa7\x18\xa2\xf7\x46\xe7\xe5\xc9\xc2\xb5" + "\xbb\xc8\xb5\xd7\x5f\x83\xde\xb6\x6d\x79\x73\xc9\x7f\x93\x62\x27\x48" + "\x71\x91\xb7\x18\x99\xc1\xaa\x7a\x32\xf9\xf4\xfd\x69\x97\x9a\x9d\x17" + "\xe8\xe3\xbf\x18\xa2\x8a\xc2\x74\x88\xdf\xa8\xf1\xc6\x96\x12\xbd\x3a" + "\x5d\xe0\x7e\x1d\x6b\x57\xb6\xab\x4a\x1c\xef\x60\xb8\x04\xac\x64\x6d" + "\x8f\x22\xcc\x47\xa1\x53\x99\x05\x86\xab\xdd\x61\xce\x8b\x8f\xb8\x4d" + "\x05\x1e\x56\x37\x81\xb5\x32\x21\xc2\xe8\x30\x22\x84\x4a\x1f\xb4\x22" + "\x5e\x60\xe8\xd7\x43\x6f\xf2\x05\x5b\x83\xec\x55\xcd\xa7\xb8\x4c\xb3" + "\x6e\x8a\x92\x62\xa5\x5e\x08\x07\x42\xc2\xda\xa9\x40\x1e\x51\x1f\x6c" + "\xec\x98\xce\xaf\xe3\xae\xa6\xe1\x07\xa6\x51\x8c\x76\x3e\xa1\x07\xd2" + "\x54\x9b\xa3\xf6\xa6\x6e\x64\xd3\x21\x0f\x96\x29\x79\x60\xf7\xac\xde" + "\x14\xb4\x67\x7a\xc3\x38\x5d\x26\x2a\xe9\x87\x0a\xe9\x73\x71\xba\xac" + "\x2e\xc7\xe9\xad\x98\x1f\x4f\x37\xaa\xdb\xff\x8d\x2f\xaf\xd1\x61\xa6" + "\x8e\x5f\x16\xb8\x21\x9f\x15\x72\xf8\x39\x07\xc4\x46\x44\x25\xf1\x43" + "\x3b\xec\xdc\xaf\x29\x09\xc5\xb6\x02\xf9\xe2\x82\x2e\x59\x5e\x8c\x9d" + "\xfc", + "\x1c\xaf\x54\x2b\xee\xa4\xbb\x14\x6d\x98\x93\xa4\xf0\x44\xf6\x93", 1, + 2048, "\x14\x3e\x28\x5a\x5c\x31\x0a\xc1\xe9\xa0\x18\x1a", 96 }, + { 256, 128, 289, + "\xdd\xba\xfd\xc9\x5b\x4c\x1c\x19\x2d\x3a\xc6\x8b\x03\xd5\x67\x4a\x36" + "\xef\x50\x77\x17\x49\xf4\xfc\xef\x6d\xf9\xac\xaf\x2e\xd0\xc2", + "\x87\xed\x96\x03\x9e\x80\xae\xe7\x91\xdc\x89\x10\xb8\xfe\xbc\x4f\x51" + "\xf2\x97\x79\x4a\x6a\x47\x38\x0b\x80\x1d\x45\x5e\x89\x37\x90\x67\xb6" + "\x9e\xe5\x6a\x52\x85\x5d\x5e\x35\xd4\x21\x12\x0b\xe0\x62\xf5\xa9\xf9" + "\xf4\x9a\x82\x9a\x9a\x19\x62\x63\x11\x63\x2e\x6a\x16\x67\x25\x79\x5b" + "\x66\x46\x57\x6d\x48\x12\x47\x0a\x9f\x20\xf0\x81\x6d\xf2\x32\x0a\x47" + "\xae\x77\x74\xc5\x3b\xda\xc5\xc9\xa1\x3a\xaa\x91\x5e\x4a\xb8\xac\x8c" + "\x9e\xb4\x5f\x06\x9f\x05\xb1\x11\x43\x10\xb7\xb2\xc2\xc8\x48\x04\x07" + "\x65\xd8\xea\xa9\xe9\x11\x22\xbb\x77\x01\x88\xeb\x14\xb1\x69\xc0\x0d" + "\x14\x11\x0d\x98\x3b\xd2\x98\xce\xbb\x26\x8c\x01\x64\xcb\xfd\x75\x70" + "\x4e\x83\xa0\x59\x7c\x0c\xcf\x5b\x83\xcb\x58\x05\x68\x98\x47\xb4\xec" + "\xa6\x45\x7e\xc1\x2e\xd7\x4d\xd7\xd2\x61\xdb\xb0\x6f\x11\x40\x14\xaf" + "\x14\xab\xc1\xd7\x6b\xd3\x0a\xb6\x9b\xe2\x80\x87\x29\x48\x96\x7d\xca" + "\xc0\xb5\x5a\x02\x08\x3a\xfc\xc4\x02\x0c\x94\x6e\x97\x09\xc3\x99\x47" + "\x70\x36\xef\xb4\xb8\xc5\x4f\xc0\x0a\x35\xfd\xcd\x28\x83\xcc\xd5\xdf" + "\x6f\x03\x3d\xc0\x13\x0a\xda\x4b\x9a\xd3\xae\xe6\x97\xa7\x00\x36\xb5" + "\xf3\x04\x83\xa0\xef\x8c\xfa\xe4\x9a\xb4\x91\xff\xee\x22\xf5\xad\xb0" + "\x24\xed\x3a\x18\xea", + "\xd8\xee\x23\xf7\x79\xc0\xb5\x4b\x7a\xb3\x83\xcb\x10\x7b\x00\x98", 1, + 2216, "\xcb\x86\x6b\x2f\x56\x88\x38\xb9\x3c\xec\xec\x38", 96 }, + { 256, 128, 290, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa6\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 291, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 292, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa5\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 293, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x53\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 294, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x27\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 295, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd1\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 296, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc3\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 297, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x95\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 298, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x81\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 299, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x14\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 300, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd1\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 301, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x22\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 302, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd2\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 303, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x21\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 304, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x99\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 305, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\xb6\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 306, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x86\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 307, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd7\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 308, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x07\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 309, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\x56\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 310, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\x89\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 311, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x4e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 312, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa8\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 313, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x05\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 314, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8e\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 315, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x53\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 316, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8d\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 317, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x50\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 318, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x0f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 319, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\xd2\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 320, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x17", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 321, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x10", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 322, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x14", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 323, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x13", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 324, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x56", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 325, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x51", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 326, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x19\x87\xa9\xa9\xc6\x8f\xe6\x85\x96", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 327, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\x36\xd6\x6e\x04\xb3\x52\xa9\x63\x91", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 328, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa6\xc2\xf2\x01\xd0\xdf\xea\x19\x86\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 329, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x94\xf6\x94\x23\x7f\x3f\x36\xd7\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 330, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x81\xd0\xdf\xea\x99\x87\xa9\xa9\xc6\x8f\xe6\x85\x16", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 331, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x14\x23\x7f\x3f\xb6\xd6\x6e\x04\xb3\x52\xa9\x63\x11", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 332, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa7\xc2\xf2\x01\xd0\xdf\xea\x99\x87\xa9\xa9\xc6\x8f\xe6\x85\x96", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 333, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x51\x94\xf6\x94\x23\x7f\x3f\xb6\xd6\x6e\x04\xb3\x52\xa9\x63\x91", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 334, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x58\x3d\x0d\xfe\x2f\x20\x15\xe6\x78\x56\x56\x39\x70\x19\x7a\xe9", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 335, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xae\x6b\x09\x6b\xdc\x80\xc0\xc9\x29\x91\xfb\x4c\xad\x56\x9c\xee", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 336, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 337, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 338, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 339, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 340, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x27\x42\x72\x81\x50\x5f\x6a\x99\x07\x29\x29\x46\x0f\x66\x05\x96", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 341, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd1\x14\x76\x14\xa3\xff\xbf\xb6\x56\xee\x84\x33\xd2\x29\xe3\x91", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 342, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xa6\xc3\xf3\x00\xd1\xde\xeb\x18\x86\xa8\xa8\xc7\x8e\xe7\x84\x17", 0, + 64, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 343, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x50\x95\xf7\x95\x22\x7e\x3e\x37\xd7\x6f\x05\xb2\x53\xa8\x62\x10", 0, + 128, "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b", 96 }, + { 256, 128, 344, + "\x79\xed\xd4\x40\xcc\x8a\x32\xb8\xb6\xd4\x63\x05\xc4\xce\x36\x34\x65" + "\x85\x0e\x28\xba\xc4\xb5\xf6\x40\x0f\xcc\xa9\x0e\x3a\xf0\xaf", + "\x81\xdb\x9c\xba\x83\xd9\x70\x4f\x30\xdb\x67\x32\xa0\x08\x21\xbd", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, "\x30\x57\x26\x28\x5d\x7b\x1f\x62\x2a\x37\x91\x07", 96 }, + { 256, 128, 345, + "\x79\xed\xd4\x40\xcc\x8a\x32\xb8\xb6\xd4\x63\x05\xc4\xce\x36\x34\x65" + "\x85\x0e\x28\xba\xc4\xb5\xf6\x40\x0f\xcc\xa9\x0e\x3a\xf0\xaf", + "\x37\x78\xdb\xc5\xc2\xe8\xc0\xe0\x62\x24\x9c\x79\x21\x1e\x64\x52", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, "\x30\x57\x26\x28\x5d\x7b\x1f\x62\x2a\x37\x91\x07", 96 }, + { 256, 128, 346, + "\x71\x08\x96\x35\xa9\xd7\xd3\x56\x6e\x8e\xe2\x22\x1c\xd1\x59\x60\xbd" + "\x83\xd1\x8c\x30\x34\x52\x36\x01\xed\x51\x48\x3c\x79\x28\x10", + "", + "\x16\x7b\x6a\x32\xc4\xac\x98\xfe\x73\xbf\xaf\xe4\x64\xba\x8c\xd2", 1, + 0, "\xe7\xe2\xe5\x65\xcb\x4c\x29\x91\x6c\xf5\x23\xbf\x32\xf9\xc8\xf8", + 128 }, + { 256, 128, 347, + "\x26\x2f\x1c\x95\x04\xff\x79\x67\xac\x50\x30\x15\xe3\x97\xff\x30\xab" + "\x0e\x88\xff\x74\xbc\xd0\xdf\x34\x41\x1f\x11\x0f\x60\xcb\xae", + "\x2c", + "\x96\x6a\x58\x80\x2c\x30\x57\x28\x12\x78\xa8\x75\x20\x86\x21\x90", 1, + 8, "\xdb\xc0\x1d\xa6\xe5\x11\x48\x18\x7e\x95\x38\x36\x25\xec\xb0\xf1", + 128 }, + { 256, 128, 348, + "\xb8\x0e\x88\x77\xe3\xf0\x2e\x5a\x1b\x92\xee\x64\xb8\x0a\x7a\xeb\x1b" + "\xe3\xb0\x18\x48\xaf\xbd\x8b\x10\xac\xf3\x46\x8f\xa7\x1c\x95", + "\x40\x12", + "\xa1\x57\x98\xc2\x75\x14\x59\xce\x9c\x1e\x7e\x35\x8e\xac\x0b\x06", 1, + 16, + "\x26\xba\x26\x1f\xd1\xe8\x95\x48\xd8\x19\x08\xe5\xa7\x4c\x11\x11", + 128 }, + { 256, 128, 349, + "\x3a\x4a\xff\xa4\x4c\x51\xed\xb8\x1d\xe1\x67\x68\xdc\x8d\x6d\x99\xbd" + "\x36\x42\x31\xc4\x45\xb0\xfa\xac\x6e\xa2\x87\x8d\x41\xf8\x25", + "\x33\x7c\x5b\xa3", + "\x86\x3f\x2e\x17\xd4\x14\x3a\x34\x88\x3e\x0f\x5c\xa9\xb0\xd6\xc5", 1, + 32, + "\x10\x80\x9d\xea\x9b\x6b\xf5\x9a\x16\xf7\x44\xe3\x69\x9e\x06\x82", + 128 }, + { 256, 128, 350, + "\xc1\x14\x77\x48\x32\xbb\xec\x17\x02\x74\x55\x74\x54\x0b\xb1\xd0\x04" + "\x48\x51\x94\xb8\xcb\x2c\x4c\x15\x9c\x26\xef\x21\x48\x18\xc6", + "\xdd\x19\xd0\x7e\x15\xcc\xd8", + "\x5f\x94\xe8\xf4\x41\xc7\x22\xb8\x57\x74\x60\xd8\x31\x33\x93\x0d", 1, + 56, + "\xb6\x44\xc2\x94\xe1\x50\x21\xd1\xbc\x95\x44\x78\x59\xce\xc0\xbf", + 128 }, + { 256, 128, 351, + "\x9e\x98\xbf\xfd\x38\x39\x22\x68\xa9\xea\x7c\x1b\x8a\xed\x18\x58\x66" + "\x66\xcc\x41\x9c\x03\x86\x09\x0f\xf8\x70\xc4\x59\x7e\x1a\x51", + "\x3f\x9c\x92\x3b\xf3\x83\x43\xa8", + "\xde\x5b\xa4\x4c\x9f\x61\x7a\x16\x32\x2f\x08\x7a\x49\x21\xad\x26", 1, + 64, + "\x9e\xc3\xb1\xf1\xcf\xa9\x7b\xa6\x0f\xae\xe6\xcf\x12\x02\x4f\x68", + 128 }, + { 256, 128, 352, + "\xe6\xd5\x91\x3e\x00\xcd\x10\xf2\xd1\x16\x8d\xc6\x6f\x45\xaf\x51\x1d" + "\x16\x11\xcc\x17\x31\xc0\xc5\x38\x9a\x99\xaf\x8a\xd7\xc4\x06", + "\xab\xb1\xbd\x62\x14\x0a\x6e\xeb\xff\x9a\x18\x62\xbc\x37\xd1", + "\xc4\x79\xcf\xd3\xc3\x41\x50\x94\x39\x76\xec\xdd\x53\x94\xd4\x92", 1, + 120, + "\x04\xf4\x49\xb5\xf0\x94\xc7\x21\xd9\xaa\x2f\x97\x02\xfe\x74\xb2", + 128 }, + { 256, 128, 353, + "\x84\x67\x5e\xd6\xeb\xa9\x25\x69\x0f\x1c\xc2\x1a\x69\xc8\x94\x3b\x82" + "\xe3\x44\x3a\x0b\x28\x23\x01\xbc\x7b\x6b\xf2\xba\xf6\xb2\x29", + "\x99\x22\x19\xd1\xbc\x60\xba\x0e\x1b\xa7\x2d\xb8\x57\xc9\xcf\x80", + "\x81\x3f\x5b\xe9\x5d\x2c\x66\x41\x2d\xf7\x12\x2f\xfc\xc9\x81\xa7", 1, + 128, + "\x02\x6e\xa8\x7e\x04\xe0\x4e\xad\xaf\xba\x49\xca\x4e\xac\x61\x0e", + 128 }, + { 256, 128, 354, + "\xaf\xf9\x02\xdd\xaf\x53\xdb\xb0\x7e\xc0\xd0\x61\xbb\x26\x66\xb7\x84" + "\xdb\xf8\x38\x66\xa4\x0f\x09\x8d\x53\xbc\x6b\xa3\x21\xc2\x31", + "\x60\x75\x42\xe0\xf7\x36\xfd\x4c\x8a\xd3\x49\xcf\xb1\x73\x07\xb1" + "\xc6", + "\xd9\x1a\x6f\xc3\x11\x0c\x4f\x82\xad\x5f\x31\x78\xe0\x6e\x72\x4d", 1, + 136, + "\x8c\xb2\x53\xaf\x95\x90\x90\x83\xa5\x73\x6c\x71\x00\xc1\xf2\x97", + 128 }, + { 256, 128, 355, + "\x49\x07\x7e\xe5\x6e\x7a\xad\x04\xd9\x02\x75\xb6\x86\xd9\xcc\x3a\x99" + "\xaf\xbb\xbf\xad\x5c\x0a\xf1\xd1\x18\xdb\xd9\xbb\x6b\x04\xbf", + "\x8a\xfc\xba\xc9\x7b\xa9\xe9\x9c\xe0\x12\x6b\xa0\x18\x69\x93\x1d\x04" + "\x1c\xa8\x1f\x49\xd8\x09\x3a", + "\xd9\x4e\x5c\x57\xfc\x36\x42\x22\xad\x35\xbe\x5b\x9e\x3d\x9a\x87", 1, + 192, + "\xa2\x0c\x60\xc2\x60\xf1\xae\xcb\xb2\xd2\x2e\xc8\x82\x9b\xf9\xf6", + 128 }, + { 256, 128, 356, + "\x7e\x4c\x3e\x47\xb9\x7f\xa6\x36\x2f\xe7\xf6\x60\xd2\x91\x67\x2e\x66" + "\xb5\x53\x58\x9e\x19\x40\x4b\x5c\x90\xa2\xff\x43\x84\x69\x14", + "\x0e\xf9\xd4\xbb\xb9\xed\xd5\x2c\x6e\x4d\x2e\x1d\xf0\x84\xc3\xda\x04" + "\x11\xfb\xd9\x60\xf9\x79\x3f\x87\x5b\xd8\xc1\xbc\x75\x1c\x7d\x78\xc0" + "\x07\x18\xce\x34\x4e\x11\xa6\xfb\x14\x87\x63\x4e\xdb\xfe\x52\xaf\x80" + "\x4c\x65\xa4\x6e\x91\x8b\xdc\x6a\xee\x98\xbe\xe3\x1f\x0b\x98\x7a\xde" + "\x33\xa0\x97\x6e\x1d\xb2\x66\xb2\xc1\x33\x45\x3e\x02\x7b\xd8\x65\x19" + "\x98\x17\xdf\x26\x89\xac\x09\xa7\x59\xd3\xbe\x19\x5c\xd4\x56\x52\x8a" + "\xbd\xef\x2f\x69\x38\xf8\x71\xe2\x55\xd0\xd0\x4e\x13\x0e\x19\x0b\x93" + "\x45\x21\x98\xc8\x5b\x0e\x0b\x9f\xc0\xce", + "\xdd\x37\xb1\xcc\x99\xa4\x81\x45\x41\x63\x36\xb6\x1d\x3b\x1f\x39", 1, + 1032, + "\x28\x00\xc4\xab\x25\xd2\xb5\x62\xce\x76\xa3\x5c\x03\x54\x2f\x66", + 128 }, + { 256, 128, 357, + "\x3c\xfa\x7a\xe7\x45\x0c\xa9\xd5\x14\x50\xd4\x81\xf6\xa8\xd4\x2a\x42" + "\x54\x07\x0d\xd8\x8d\xf3\x4e\x9d\x43\x02\x8f\xad\x1d\xad\x90", + "\x03\x92\x1a\xc5\x7a\x10\x12\xfa\x1f\x5b\x99\xfc\x96\xf1\x81\xd7\xc7" + "\x17\x29\x03\xe8\x0b\x52\xe7\xb9\x68\xa2\xdc\x96\x05\xec\x39\x07\x83" + "\xcf\xb8\xbe\x86\x7e\x6e\xab\xfc\xa3\x9c\xef\x4d\x1c\x53\xd4\x3e\xe6" + "\x1b\x18\x96\x35\xd3\x9a\x77\x9e\x00\xd4\xdc\x54\x28\x11\xc1\x1a\x3b" + "\x88\x03\xe5\x72\x99\x31\x4e\xf0\xb3\x11\x99\x3f\xa1\x57\x0e\xc2\x8a" + "\x9a\x83\xfe\x2c\xc9\x59\xc6\xd3\x0e\x1d\xaa\xa7\x1c\x81\x73\x54\x84" + "\xc6\x1a\x05\xf9\x45\x1e\x49\x41\x1b\xc6\x33\xb5\xe7\xe6\x38\xba\xe7" + "\xe3\xc9\x07\x46\xe8\xe1\xef\x79\x6d\x34\x9e\x5c\x90\x00\xc8\x7e\xfa" + "\xbf\x33\x73\xa9\xb5\x2c\xea\x26\x99\x5a\xf3\xcd\x7c\xf9\x84\xb7\x9a" + "\xcb\x79\x37\xf4\x1c\xd4\xaf\xe8\x67\xc3\x77\x81\xdb\xa2\x75\xc4\x17" + "\x6e\xe1\xa3\xfe\x7d\xe1\x1b\xb8\x6e\x28\x72\x46\x5e\xe8\xb4\x94\xca" + "\x64\x98\xc2\x34\x78\xe3\x1b\x45\x1f\x38\x28\x88\x2b\xcc\xf0\x4d\x83" + "\x29\xfb\x7d\x35\xa4\x90\x39\xc4\x54\xb6\x8f\x8f\xd2\x7c\xeb\xee\x84" + "\xcc\xd7\x19\xc5\x74\x1f\x57\xe6\xa9\x35\xfe\x3e\x95\xef\x47\xfd\xa7" + "\xb0\x90\x72\x8f\x81\x1b\x0b\x06\x3e\x50\x87\x61\xd4\x9f\x26\x3f\xfc" + "\xf2", + "\x3b\xf9\xef\xd9\x51\x02\xa4\x19\xd0\x6d\x75\xd3\xaf\x5d\xa6\x3c", 1, + 2048, + "\x8b\x71\x1c\x1b\x5b\x50\x14\xf3\x1c\x3f\xb5\x18\x3e\x09\x6f\xcd", + 128 }, + { 256, 128, 358, + "\x9c\xe7\xb1\xd9\x3f\x41\x1a\x38\xd4\x13\xf6\x33\xf9\x0c\xd4\x49\x34" + "\xa0\x76\xd6\x46\x34\xb4\x7b\x9f\x1b\xcb\xed\xc4\x07\xe3\xb5", + "\xe8\x4a\xa2\x87\x13\xdc\x0f\x02\x98\x26\x99\x8b\x18\xcc\xe8\x95\xcc" + "\xfe\xf5\xde\x60\x1c\xa1\xbc\x5a\x6f\x4b\x86\x1a\xe3\x85\x03\x1f\x89" + "\xdd\x32\x52\xc0\xaf\xcd\xfd\x5d\xce\x87\x03\xef\x12\x52\xbc\x9c\xf5" + "\x33\x85\x7e\x16\xe2\x80\xbd\x8f\x0c\x1c\xcf\x1d\xe2\xcf\x88\xe5\x3e" + "\x0f\x27\x54\xca\xd2\x61\xf1\x52\xb6\x44\xc6\xdb\x09\x87\xc8\xb1\x05" + "\x9c\x1e\xf3\x4a\xaf\x57\x30\xec\xea\x80\xbe\xec\x84\x26\x2e\xbd\xf9" + "\x72\x62\x19\x3a\x4e\x04\x30\x5d\x82\x13\x2e\x80\xf3\x30\x32\xd6\xac" + "\x43\x88\x6f\x39\xd7\xc6\x86\x50\x95\x52\xa8\xa7\x2e\xfa\x65\x81\x1a" + "\xee\xfb\x82\x06\xfd\x02\xbd\x8c\xd9\xfa\x90\xf7\x36\xa2\x9c\xc1\xe7" + "\x8f\xa7\xb3\x27\x70\x7e\xb4\x75\xd7\x0c\xa9\x2a\x4e\x39\x66\x7b\xed" + "\x7f\xed\x3c\xeb\x60\x85\xae\xf1\x30\xa2\xc0\xb0\xf9\x82\xeb\x01\xfd" + "\xcf\x9c\xdd\xbf\x33\xa6\x79\x45\x5d\x16\x90\x5a\x0f\xd6\x4d\x52\x71" + "\x06\x59\x0e\xb4\x15\x1a\x62\xf3\xc6\x5f\xb9\x1f\x5f\xe0\x02\x62\x46" + "\xa5\x18\x2f\xca\x77\x8e\x43\x04\x3f\x66\x18\xe5\x73\x7e\x8f\xe8\xd8" + "\x82\xd2\xa1\x16\x2c\x5f\x65\xf8\x20\xce\x99\xad\x44\x32\x7d\xcf\xcb" + "\x54\xbb\xd5\x89\xfa\xa3\x29\xad\x29\x57\x00\x72\x4b\xf7\x45\x5d\x97" + "\xfe\xbf\xb2\x5d\xb9", + "\x97\x0b\xf2\x36\xc7\x57\x6c\x97\x7f\xdc\xf7\x2c\x5d\xb6\x27\x76", 1, + 2216, + "\x6e\xee\x90\x6c\x6f\xe2\x6b\xcb\x3e\x63\x0e\x49\xb7\x51\x76\x33", + 128 }, + { 256, 128, 359, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4d\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 360, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbb\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 361, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4e\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 362, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xb8\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 363, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xcc\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 364, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3a\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 365, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe3\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 366, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb5\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 367, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x19\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 368, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x8c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 369, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x74\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 370, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x87\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 371, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x77\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 372, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x84\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 373, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x86\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 374, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\xa9\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 375, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x51\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 376, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x00\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 377, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\xd0\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 378, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x81\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 379, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xe6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 380, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x21\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 381, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x40\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 382, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xed\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 383, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa7\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 384, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7a\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 385, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa4\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 386, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x79\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 387, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\x26\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 388, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\xfb\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 389, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x62", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 390, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x65", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 391, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x61", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 392, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x66", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 393, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\x23", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 394, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\x24", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 395, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x06\x50\xc6\x41\xaa\xa6\x70\x63\xe3", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 396, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\x29\x01\x01\xec\xdf\x7b\x3f\x85\xe4", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 397, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4d\xe2\x08\x99\x75\xdb\xbd\x06\x51\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 398, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbb\xb4\x0c\x0c\x86\x7b\x68\x29\x00\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 399, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x19\x75\xdb\xbd\x86\x50\xc6\x41\xaa\xa6\x70\x63\x63", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 400, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x8c\x86\x7b\x68\xa9\x01\x01\xec\xdf\x7b\x3f\x85\x64", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 401, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4c\xe2\x08\x99\x75\xdb\xbd\x86\x50\xc6\x41\xaa\xa6\x70\x63\xe3", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 402, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xba\xb4\x0c\x0c\x86\x7b\x68\xa9\x01\x01\xec\xdf\x7b\x3f\x85\xe4", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 403, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xb3\x1d\xf7\x66\x8a\x24\x42\xf9\xaf\x39\xbe\x55\x59\x8f\x9c\x9c", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 404, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x45\x4b\xf3\xf3\x79\x84\x97\xd6\xfe\xfe\x13\x20\x84\xc0\x7a\x9b", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 405, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 406, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 407, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 408, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 409, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\xcc\x62\x88\x19\xf5\x5b\x3d\x86\xd0\x46\xc1\x2a\x26\xf0\xe3\xe3", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 410, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3a\x34\x8c\x8c\x06\xfb\xe8\xa9\x81\x81\x6c\x5f\xfb\xbf\x05\xe4", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 411, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07", + "\x4d\xe3\x09\x98\x74\xda\xbc\x07\x51\xc7\x40\xab\xa7\x71\x62\x62", 0, + 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 412, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbb\xb5\x0d\x0d\x87\x7a\x69\x28\x00\x00\xed\xde\x7a\x3e\x84\x65", 0, + 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + 128 }, + { 256, 128, 413, + "\xaf\xb8\x06\x35\x37\x60\xe5\x9a\x08\xbd\x78\x70\xed\xeb\xce\xd4\xb8" + "\x72\x3e\xe3\x1d\x7d\x4f\x96\x1e\x4e\xf2\x18\x6a\x7d\x3a\xbf", + "\x9c\x33\xf8\x42\xdb\xa5\x15\x3e\x8d\x65\xb9\x17\x77\x11\x2c\x69", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 1, + 128, + "\x3b\x44\x52\xec\xb8\xa0\xb1\x77\x2c\x0e\x50\xc1\xf5\x94\x09\xbf", + 128 }, + { 256, 128, 414, + "\xaf\xb8\x06\x35\x37\x60\xe5\x9a\x08\xbd\x78\x70\xed\xeb\xce\xd4\xb8" + "\x72\x3e\xe3\x1d\x7d\x4f\x96\x1e\x4e\xf2\x18\x6a\x7d\x3a\xbf", + "\xba\x32\xdf\xc0\x83\x55\xf8\xde\x70\xb6\xab\xf7\x32\xf2\x88\xd0", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 1, + 128, + "\x3b\x44\x52\xec\xb8\xa0\xb1\x77\x2c\x0e\x50\xc1\xf5\x94\x09\xbf", + 128 }, + { 0, 128, 415, "", "\x62\xe8\xe0\xda\x60\x00\xc6\x42", "", 0, 64, + "\xa2\x3c\xe8\xec\xb4\x4b\xab\x25\x46\xf3\x94\xc7", 96 }, + { 0, 128, 416, "", + "\x64\x46\xb9\xa0\x2b\x16\x76\xbe\x12\x46\x5f\xad\x38\x8e\x39\xfe", + "", 0, 128, "\xc9\xae\x3d\xc4\x39\xf2\xcd\x95\x4f\xff\xd0\xf8", 96 }, + { 0, 128, 417, "", "\x43\xf6\x3d\x68\x0d\x70\x12\xfe", "", 0, 64, + "\xc8\x46\xfa\x33\x6e\x6c\xe3\xf2\x04\x2f\xc9\x1b\x4a\x27\xb0\xfb", + 128 }, + { 0, 128, 418, "", + "\x8a\x27\x59\xaf\xa8\xa7\x77\xc4\x15\x27\xa6\x0b\xc5\x45\x5e\xfd", + "", 0, 128, + "\xf3\x7b\x6d\x4c\x49\xbd\xda\xc0\xbc\xe7\xf7\xdf\xb2\x76\x06\x9f", + 128 }, + { 8, 128, 419, "\x64", "\x22\x8b\xd5\xce\x6a\x59\x3d\xcd", "", 0, 64, + "\xf9\xd8\xb3\xd1\xaa\x47\x75\x5c\x9f\x31\x2a\x00", 96 }, + { 8, 128, 420, "\x23", + "\x8f\x5e\x20\x0b\x5c\x89\xad\x35\x8e\xed\xa4\xd0\xa3\xd4\x89\x17", + "", 0, 128, "\x1a\x72\x79\xd7\x41\x9a\x6a\xaa\x02\x64\x20\x4d", 96 }, + { 8, 128, 421, "\x96", "\xc2\x58\x27\xed\xb2\xf2\xee\xeb", "", 0, 64, + "\xf7\xf4\x52\xbf\xc3\x1b\xe5\x01\x2f\xbc\xdc\x79\xbd\xb5\x77\x82", + 128 }, + { 8, 128, 422, "\x71", + "\x3e\xb1\x29\x97\x19\xb9\x14\x3e\xed\xf2\x38\xe4\xc2\xde\x30\xf4", + "", 0, 128, + "\x97\xf0\xfc\x55\x34\x07\xfb\x52\xfa\xc2\x5a\xb5\xc6\x46\xfc\xf8", + 128 }, + { 64, 128, 423, "\x20\x93\xeb\x4f\x07\x02\x15\x15", + "\x2b\x1c\x35\xc5\x9f\x83\x59\x10", "", 0, 64, + "\x7c\xaa\x60\x94\xbc\xc0\x3e\x30\xdc\xb6\x63\x8a", 96 }, + { 64, 128, 424, "\x1d\x24\x5f\xe8\xb1\x2c\x12\x04", + "\x6c\xdc\x1a\xd1\x1e\x26\x57\x66\x20\x2b\xed\x63\x03\xf4\x5c\xc3", + "", 0, 128, "\xb0\x21\xda\x22\x34\xab\xde\x1c\x6e\x0b\xd4\x81", 96 }, + { 64, 128, 425, "\x8c\x4d\xcd\x56\xf3\x28\x0b\x42", + "\x65\xba\x8a\xb5\x45\xd1\xf7\xfb", "", 0, 64, + "\x62\x9c\x59\x6e\x66\x31\xcd\x62\x3d\xf9\x8f\x72\xcc\xf0\x94\xf1", + 128 }, + { 64, 128, 426, "\x5c\xda\xe7\x09\xce\x51\x7e\x61", + "\x9c\xa3\xd0\x42\xb9\xd9\x1c\x6a\xf0\xd2\x47\xcf\x28\x9f\x20\xd8", + "", 0, 128, + "\x1a\x96\x86\x28\xa0\x67\xbc\x3a\x1d\xc9\x5b\x06\x4d\x78\xbc\x3d", + 128 }, + { 160, 128, 427, + "\x1e\x09\x98\x1a\xd6\x52\x0d\xf2\x78\x96\x2c\x22\x5d\x14\x65\x2a\x14" + "\x4b\x04\x8b", + "\xa3\x4e\xc8\x9b\x01\x7a\x10\xd3", "", 0, 64, + "\xb8\xe4\xbd\xcd\x90\x1f\x5f\x56\x24\x7f\x27\xe4", 96 }, + { 160, 128, 428, + "\x09\x7f\x44\x2a\xcb\xba\xc6\xe3\xa8\x58\xc9\xbd\xf7\x6a\x36\xf0\x6a" + "\x10\x37\x06", + "\x6e\x9e\xb6\xa5\x8f\x91\xf9\x23\x1a\x94\x94\x3d\x78\x5c\x67\xa3", + "", 0, 128, "\x09\x8d\x00\xea\x1d\x92\x96\xd9\x3a\x03\x1d\x63", 96 }, + { 160, 128, 429, + "\xf1\xc1\x0c\x5a\x1c\x9a\x0a\x4a\x3c\x07\x43\x6f\xa6\xcf\xa9\xd4\xd8" + "\x78\x85\xb2", + "\x38\x05\x7f\x87\x9c\xa7\xcc\xe9", "", 0, 64, + "\x0a\xb9\xbb\xf8\xc2\xc3\xbf\xbd\xac\xd9\x56\xd8\x7b\xcb\xc5\x9c", + 128 }, + { 160, 128, 430, + "\x8e\x8d\x6c\x52\x26\xaa\x3c\x39\x83\xcb\x6a\x8e\x21\x1a\x5d\xca\x49" + "\xcb\xb3\x70", + "\x7e\xb1\xcf\xab\x0d\x9d\xea\x4d\x04\x1f\xc5\x90\x58\x55\xa3\xe4", + "", 0, 128, + "\x25\x49\xf2\x0a\x79\xac\x25\x67\x21\x47\x20\x16\x3b\x6c\x68\xef", + 128 }, + { 320, 128, 431, + "\x01\xe4\x6c\xc7\x91\x6f\xec\x9b\x53\x29\xb4\x22\xfc\xe3\x9b\x5e\xd1" + "\x4d\x21\x50\x7d\x75\x66\xf4\x98\x9d\x08\x7d\x5b\x00\xe7\x5a\x9c\xfa" + "\x03\x5f\xde\x39\x8b\x1c", + "\x76\xb0\xa2\x2e\x13\xf7\x3e\x7e", "", 0, 64, + "\x0d\x3a\xbf\xc7\x18\x44\x31\x47\x35\x52\x48\x9a", 96 }, + { 320, 128, 432, + "\x0d\x4d\x91\x11\x88\x54\x34\x2e\x7c\x26\x18\x66\xe9\xd4\x9a\x22\xfd" + "\xee\x0f\x28\xc5\xe5\xba\xa2\xcd\x74\xc9\xfd\x67\xbe\x9d\x3f\x14\xfa" + "\xaf\x0c\x60\x7c\xea\x94", + "\x91\x23\x96\x98\xee\xd6\xa6\x67\x12\x3c\xd0\xd9\xae\x85\xba\xf8", + "", 0, 128, "\xd3\x61\xf8\xeb\xfd\x7f\xb3\x43\x76\x3f\x5e\xdd", 96 }, + { 320, 128, 433, + "\x72\xdf\x7d\x65\xab\xa3\xb9\x7d\x2b\x31\xec\x7f\x39\x6c\xf2\x98\x3f" + "\x7b\x27\x97\xe3\x66\xb1\x3c\x5b\x0e\xf4\x46\x7b\xa7\xb7\xb8\x4a\xc7" + "\xc1\x82\x64\x43\x75\xee", + "\x9a\xdf\x89\x89\x56\x5b\xa6\x26", "", 0, 64, + "\x71\x08\x0a\xae\xef\x17\x2a\x0e\x3b\xf7\x32\xe5\xc4\x66\x60\x61", + 128 }, + { 320, 128, 434, + "\x85\xc3\x62\xd4\xc0\x53\xef\xcb\x5a\x72\x89\xad\x80\xfa\xf2\xa2\x78" + "\xf5\x1b\x83\x6b\x2a\x7d\x16\x41\x0b\x9f\xbc\xed\xe3\xea\xd6\x9a\x9b" + "\x1d\x3c\x96\x65\xd8\x77", + "\x06\x4d\x36\x5c\xd0\x6b\xc6\xc3\x35\x23\x62\x32\xe7\x2e\xae\x49", + "", 0, 128, + "\x22\x0f\x44\xcb\xe8\x3b\x7f\x75\x70\x24\x14\x29\x3b\x4b\x97\xed", + 128 }, + { 128, 128, 435, + "\x1a\xbc\x93\x3c\x4f\xe2\x3a\x4b\x49\x60\x5c\x3a\x9d\x30\x99\x7d", + "\x76\xc2\xb2\xb1\xd6\x92\xbc\xed\xe9\x11\x52\xb3\x45\xf6\x88\x15", + "\x36\x79\x73\xf1\x93\xfc\xc3\x9e\x23\x03\xca\x01\x93\x9b\x4e\x77", 0, + 128, "", 0 }, + { 192, 128, 436, + "\x1e\x39\xd9\x1e\xf4\xd0\x5e\xd9\xbd\x5d\x8d\x88\x6f\xbb\x93\x7e\x35" + "\xa5\x44\x73\xdf\x5d\x0c\x25", + "\xec\xf0\x19\x90\x48\x0f\xdc\xe0\xd2\x8c\x15\x53\xb8\x0e\xe1\x28", + "\x73\x2c\x22\x4d\x31\xb6\x1b\xe2\xe4\xdb\x36\x5d\x5a\x53\x3c\x1c", 0, + 128, "", 0 }, + { 256, 128, 437, + "\xc2\x92\xbb\xa4\x54\xcc\x13\xfc\x32\x4e\x19\xff\xf7\xa4\x63\x53\x4b" + "\xa9\x3a\xcb\x98\xe4\xc1\xb5\x1c\x21\xd5\xb7\xef\x53\x9f\x3a", + "\xc1\x60\xf1\x7c\x5a\xa9\x2c\xa6\x8a\x83\x7d\xc7\x51\xc0\x6f\x10", + "\x5a\xb3\x90\xc6\x7b\xde\x51\x0c\xf2\x7f\x4c\x77\x42\x5b\xff\x5a", 0, + 128, "", 0 }, + { 128, 128, 438, + "\x4f\x5d\xcb\xc6\xa2\xd7\x74\x40\xba\x17\x98\xc9\xb7\xf3\x02\xeb", + "\xd1\x0d\x2a\x06\xb3\xaf\x2e\xc1\x3b\xff\x20\xaa\xb2\x93\x97\x3b", + "\x6e\x9d\xe0\xb3\x12\x89\x25\x40\x43\x6e\x93\x59\x09\xa3\xfb\x48", 1, + 128, + "\xf1\xac\xc9\x29\x9a\xfe\x3c\x45\x48\x95\xfe\x6a\x42\x77\x11\xc0\x3f" + "\x15\xd8\xaa", + 160 }, + { 128, 128, 439, + "\x74\x00\xc7\x2f\xb3\x4b\x80\xd3\x45\xad\x4d\x17\x95\x7e\x7f\x96", + "\x6e\x0a\x7d\x3d\x9e\xda\x7d\x71\xdc\x47\x69\x81\x85\x24\x04\xdb", + "\x45\x27\x54\xf4\x89\x7f\x23\x58\xa0\xa4\xab\x02\x47\x67\xf3\x9c", 1, + 128, + "\xf8\xcb\x7e\x3d\xb0\x96\x2c\x17\xf9\x27\xde\x88\xfc\x90\xe4\xe6\xf8" + "\x06\x4f\xf0\xaf\x0d\x9d\xa3\x9d\x59\xf1\x92\xdd\x56\x13\x7e", + 256 }, + { 128, 128, 440, + "\xa9\xff\xf5\x5a\x0d\x06\x75\x32\xb2\x23\x36\xee\x01\xd2\xa7\xfc", + "\x7d\x77\xd4\x88\x4a\x13\x21\xd6\x58\x8d\xd1\xd4\x8d\x6e\x5b\xeb", + "\x68\xc9\xb2\x0e\xcf\x36\xd5\x66\xa4\x63\x07\x03\x57\x42\x39\x23", 1, + 128, + "\xef\x1d\xc5\xdc\x06\xc3\x21\x1a\x36\x66\xb6\xf1\xbc\xac\x32\xf0\x0d" + "\x00\x0f\xa2\xce\x9f\x39\xfe\xb2\x94\xe3\x1f\xb6\xc9\xd9\x83\xfa\x9e" + "\x4e\x2f\x24\x42\xf1\x79\xfa\x65\x28\xc4\x02\x64\x09\xd7\xd4\x62\x9a" + "\xdf\x38\x14\x4d\xc0\x60\x88\x7e\xf7\x2b\x5a\xd6\x82", + 512 }, + { 128, 128, 441, + "\x08\xfc\x93\x83\xee\x79\x86\xa1\xf0\xf3\x8d\xdd\x4e\x5b\xde\x90", + "\x72\xea\x2e\x0d\x0c\x5d\x7d\xc7\x63\x4e\xe0\x93\x59\xe9\xd9\x81", + "\xfb\x89\x5c\x06\xb6\xa7\xa5\x1e\xbc\x9e\xb1\x29\xd0\xeb\x9e\x1a", 1, + 128, + "\xee\xf7\x29\xba\xec\xd8\x92\x78\xda\x7c\xb9\x39\xb3\x45\xc2\x87\x2e" + "\xfc\x44\x3c\x22\xad\xdf\xb6\x8e\x42\x91\x63\x65\x6c\x56\xb3\xa3\x92" + "\x00\xd9\xe0\x78\x71\x01\x33\x00\xbb\xa9\xf9\xb0\xad\xa6\xd3\x91\x64" + "\x77\x16\x1d\xe0\x8a\xdc\xbc\x4f\xdc\x47\x67\x96\x15\x90\x1e\xfa\xf6" + "\xc2\xfd\xac\x7e\x02\x03\xca\x35\x2f\xc8\x44\x07\xb6\xa3\x76\x21\xcc" + "\x0d\x8a\x4c\x75\x26\xf5\x0f\xb2\x71\x9f\xad\x08\x56\x7e\xae\xa5\x6c" + "\x59\x88\x05\xa3\x36\xa0\xcc\x3d\x43\x09\xa1\xe2\xa2\xb8\x6d\x8d\x39" + "\x39\xce\x2c\xf3\x13\x8f\xff\xa2\x01", + 1024 }, + { 192, 128, 442, + "\x8e\xe3\x34\x45\x5b\xf9\x6b\x75\x1a\x6e\xbb\x9a\x97\x82\xfc\x0e\x46" + "\x9f\x5c\x69\xc2\x42\xad\x23", + "\x4b\xe6\x12\xa3\x5b\x8c\x98\x50\x2d\xad\x7c\x2d\x40\xc5\x1f\x34", + "\x59\xd3\x16\x61\xd8\xd4\x96\xa9\xea\x9b\xd3\x41\x2a\xe3\x3d\xa6", 1, + 128, + "\x15\xfc\xd4\x0d\x5d\x37\x43\x6e\xf3\x71\x4e\x25\x20\x6e\xfc\x23\xe4" + "\x63\x47\xce", + 160 }, + { 192, 128, 443, + "\x8a\x95\x0a\x59\x1d\xca\x1a\x24\x61\x50\x0c\x1b\xe5\x4a\x9f\x35\x40" + "\xdd\x79\x30\x0c\xd3\x21\x0b", + "\x59\x13\xaf\x7d\xe6\xbd\x1a\x2a\x64\xc6\x07\x37\x97\x8b\xfb\xaa", + "\x40\x76\x03\xda\x8b\x15\xea\x69\xc9\x51\xb2\xfb\x6d\xbf\xcb\x03", 1, + 128, + "\xa5\xa6\xa2\xbc\xee\x80\x2e\x8d\x49\xa7\x2f\x53\x6a\x49\x2c\x1a\x0a" + "\x9b\x8e\xf5\xd7\x9f\xe8\x11\x67\x24\x3f\xe0\x73\x04\xdd\xa4", + 256 }, + { 192, 128, 444, + "\x6a\x7c\x14\xce\x86\x05\xc9\x97\x8a\x0a\x7d\x9e\xba\x80\xa3\xb3\x72" + "\x39\xdb\x12\x9d\x2e\x0a\xbf", + "\xce\xfd\x73\xac\xfa\x57\xf8\xd8\x7c\xa4\x7d\xc7\x38\xa3\x0c\xdb", + "\x1a\x33\x5a\x24\x69\xd1\x54\xb0\x43\xd8\xa3\x90\x40\xd0\x71\x23", 1, + 128, + "\x04\x21\x59\xc9\xaa\xbe\xc0\x64\x41\xca\x01\xea\xc9\x27\x2d\xa4\xec" + "\x40\xb3\x9a\xa9\x6a\x53\xf0\xad\xaf\xad\x5a\x6f\xff\x86\x3a\x12\x6d" + "\xb9\x15\x0a\xd7\xa0\x26\x26\x72\xa4\xa5\x50\xfb\xcb\x10\x29\x95\xd4" + "\x86\x4e\x12\x4c\x6a\x9b\xd3\x3d\xaa\x37\xb1\xb1\x17", + 512 }, + { 192, 128, 445, + "\x24\x37\x2b\xbb\x35\xd3\xe0\xda\xfc\xf4\x45\x5a\x47\xa0\x41\x2b\x15" + "\x24\xcf\x8d\xbf\xdd\xa5\xf7", + "\x12\x1b\xf5\xdb\xe3\x61\x0a\xd5\xa0\x68\xfb\xa0\xaa\x7f\x60\x5a", + "\x5b\x46\xdd\xbf\x3f\x1e\xc8\xb1\x63\xdf\x0c\x72\xff\x47\xa8\xdc", 1, + 128, + "\xae\xd0\xf6\x78\x72\x17\xaa\x66\xb3\x8a\xac\x94\x8b\x57\x4d\x99\xf1" + "\x99\x2b\x62\x1e\x6e\x6b\xa0\x1f\x9b\x3e\xa1\xe3\x1c\x90\x16\xb9\x75" + "\x54\xca\x9a\xfd\xe1\x1d\xb8\x31\x02\xed\xbb\x76\xcd\x67\xeb\x13\x96" + "\x3a\x97\x6e\x10\x92\x99\xf7\x46\xc2\x8e\x91\x78\x32\x51\x75\xfc\xd1" + "\xac\x76\x69\x77\xf2\x97\x97\xff\x5f\xc0\x6a\x93\x4a\xed\x8e\x7e\x61" + "\xbb\x72\x3f\x89\x97\x17\x1d\x73\xdc\x61\x8e\xa6\x99\x80\x68\x97\xa3" + "\xf8\x96\x0e\x78\xe6\xfc\x14\x93\x23\x79\x0c\xf1\x05\x6a\xd3\xf1\x3f" + "\xb7\xd8\x07\x9a\xfb\x33\xf7\xe8\x36", + 1024 }, + { 256, 128, 446, + "\xa6\xe7\xfd\xbc\xd0\x72\x5e\x1d\x5d\x0f\x9d\xae\x2a\xa4\x4c\x0e\x8a" + "\x54\x2f\x4b\xf6\x25\x08\xf4\xd4\x27\x75\x08\x70\xb9\xb0\x88", + "\x91\x8d\x49\x1e\x78\x50\x2d\x60\x56\x22\x58\x25\xe7\xf7\x9f\x71", + "\xd2\x6d\x71\xab\x70\x69\x85\x52\x7a\x53\x56\xb9\xde\x56\x29\x77", 1, + 128, + "\xb4\x27\x89\x8e\xd3\xec\x7b\xf5\x2b\x62\x74\x06\x3d\xa8\x74\xc2\x5f" + "\x7d\xb0\x60", + 160 }, + { 256, 128, 447, + "\xb9\x42\x4e\xff\xc7\x76\x51\x0c\x1c\x49\x5a\x0a\x93\xd3\x7f\x3b\x1c" + "\x23\xb5\x44\x1d\xd0\x23\x51\xf2\x8c\x13\x75\xd8\x49\xe3\x43", + "\xea\x62\x21\xae\xc1\xe2\xb4\x35\x80\x65\xbf\xa1\xd8\xce\x1e\x9a", + "\xc3\x55\x60\x3c\x17\x89\x2b\x89\xd3\x0b\xed\x73\x91\x50\xad\xf1", 1, + 128, + "\xf6\xfd\xd3\x87\xe2\xec\x76\x89\xd7\x24\x18\xcb\xba\xa1\x40\x2f\x39" + "\xd9\x19\x2f\x01\x73\x46\x75\xc0\x46\x59\x70\xb6\x61\xbd\x69", + 256 }, + { 256, 128, 448, + "\x9b\xff\x9b\x26\xdc\xd6\xe6\x20\x72\x64\xea\x43\x3b\xfc\xb6\x13\xcf" + "\xa0\x02\x70\x42\xac\x7e\xc1\xd9\x86\xbb\xfc\xf4\x65\x38\xa2", + "\xc7\x7f\x51\xe5\x76\x66\xed\x25\xb4\x68\x4a\xa2\x0c\xef\x34\x91", + "\x05\xdc\x92\xc1\xe2\xc8\x0e\x0c\x7c\x9f\x1e\x82\xa8\x7d\x5b\x18", 1, + 128, + "\xa3\xc3\x85\xd0\x96\x88\x34\x4b\x76\x34\x8d\x17\xb0\x6a\xfc\x76\x4d" + "\x27\xcd\xec\xd2\x79\xcb\x6f\xab\x5e\xd2\xbc\x3c\xad\x1d\xe6\x51\xc1" + "\x43\x4b\xf3\x52\xe7\x02\xa4\x0f\x9b\x03\x38\x47\x74\x19\x93\x80\xcb" + "\x03\x79\xb6\x56\x0d\x77\x3b\xef\xd7\x8b\x11\x60\x92", + 512 }, + { 256, 128, 449, + "\x27\xef\x59\x44\x06\xaf\xad\xb5\x72\x6d\x85\xa6\xba\x85\x49\x6e\xd7" + "\xc2\xbc\xb6\xa1\xb1\x23\x65\x56\x55\xd0\xf6\xfe\x0e\x60\xb5", + "\x97\x00\x3e\x85\x74\xcd\xaa\x71\x8d\x50\x9e\x53\x32\xf2\xa5\x18", + "\x46\x5d\x99\x83\xcd\x5f\x73\x2b\x47\x5a\x9f\x72\x79\x25\x44\x33", 1, + 128, + "\xe0\x74\xf3\x26\xd3\x42\x79\x9d\xec\xe4\xe1\xff\xed\xf5\x95\x4c\xd8" + "\x0f\x4d\x45\x39\x41\x05\x06\xc3\x0c\x50\x97\x53\x45\x0e\x07\x30\x7e" + "\xae\xb4\x20\x3a\x73\x71\x75\xf6\xad\x81\xd9\x3e\x62\x5b\xc6\x0d\x78" + "\x63\xf5\x01\xac\xff\xb8\x9a\x5d\x8a\x31\xe0\xa4\x8e\x07\x37\x97\x1d" + "\x05\x7f\x24\x49\x71\x7c\x9b\x7b\xab\x24\xc9\xe4\xa8\xff\xff\x0b\x03" + "\xd9\x67\x1e\xf3\x7b\x80\x90\x32\x7d\x31\x86\x96\xc2\xb9\x6c\x40\x31" + "\x06\x4f\x95\x09\x94\x26\x17\xea\x7a\x03\x2d\x54\xbb\xc9\x7b\x82\xfa" + "\xc5\xb6\x54\xdd\x34\x89\xad\x13\x23", + 1024 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/hmac_sha1_test.json.c b/test/wycheproof/hmac_sha1_test.json.c new file mode 100644 index 00000000..d0c1d36e --- /dev/null +++ b/test/wycheproof/hmac_sha1_test.json.c @@ -0,0 +1,1119 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* HMACSHA1, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha1_test_json[] = { + { 160, 160, 1, + "\x06\xc0\xdc\xdc\x16\xff\x81\xdc\xe9\x28\x07\xfa\x2c\x82\xb4\x4d\x28" + "\xac\x17\x8a", + "", + "\x7d\x91\xd1\xb4\x74\x80\x77\xb2\x89\x11\xb4\x50\x97\x62\xb6\xdf\x24" + "\x36\x58\x10", + 1, 0, NULL, 0 }, + { 160, 160, 2, + "\x4c\xd6\x4e\xfd\xb7\x6d\xf5\xa8\x5d\xce\x3d\x34\x70\x12\xca\xd0\x6b" + "\x0c\x3d\xb4", + "\x6c", + "\x6d\x3d\x37\xaf\x55\xc7\x5d\x87\x2d\x2d\xa0\x7b\x9b\x90\x7b\xa2\x2a" + "\xd4\x87\xd4", + 1, 8, NULL, 0 }, + { 160, 160, 3, + "\x52\xe1\x99\x50\x25\x29\x7f\xe7\xb7\x93\xdc\x8e\x1e\x4f\x7d\x31\x2f" + "\xee\x27\x00", + "\x29\xdf", + "\x82\xcb\x24\xbf\xa3\x8f\xbd\xc9\x1d\x1e\xea\x2d\x2d\xc1\xce\x6e\x60" + "\xff\x88\x1e", + 1, 16, NULL, 0 }, + { 160, 160, 4, + "\xf3\xed\xfa\x00\x3d\x89\xc4\xe2\xa6\x42\x2e\x77\xa0\x1b\x8a\xdb\xd7" + "\xac\x26\xe4", + "\xb0\x15\xb7", + "\xcb\x24\x4c\xa6\xad\x23\x39\x47\x37\x84\x36\x07\x6f\xbf\xd2\x0c\x9c" + "\x8b\x84\x2b", + 1, 24, NULL, 0 }, + { 160, 160, 5, + "\x4b\x07\xed\x4e\x0c\x8d\xda\xa1\xf7\x6c\xf0\x01\x07\x28\x67\x9c\x88" + "\x57\xe1\x8b", + "\x3b\x2c\x1a\xfe", + "\x92\x41\x25\x53\x2e\x6b\x62\x5e\x7c\x5a\x8d\xcd\x16\x14\xe0\x43\x34" + "\xc0\x67\xcd", + 1, 32, NULL, 0 }, + { 160, 160, 6, + "\x7f\x53\x2c\x8e\xc8\x3c\xb2\x1d\xc9\x8a\xf7\x73\x4c\x64\xf5\xfd\x91" + "\x67\xec\x30", + "\xa3\x3c\x6f\x98\x26", + "\x0d\x25\xbc\x40\xf6\x0f\xbe\xd3\x6d\x8d\x7a\x10\x45\xff\xa6\x0d\x88" + "\x48\x4d\x56", + 1, 40, NULL, 0 }, + { 160, 160, 7, + "\x99\xe6\x0c\x1f\xc0\xcb\x3e\x6e\xd8\x36\x61\x97\x75\xe3\x7b\xf1\x5b" + "\x2c\xb9\x3f", + "\xb1\x29\xbb\x88\xce\xaa", + "\x69\x24\xd8\x33\xa3\xe7\x4b\x48\xf9\x91\xe6\xc4\x41\x73\x56\x5f\xdf" + "\x8c\x74\x70", + 1, 48, NULL, 0 }, + { 160, 160, 8, + "\x53\x84\x5f\x10\x34\x4b\x7f\x39\xed\xdb\xd3\xe4\x42\x31\xfa\x80\x2d" + "\x7e\x1a\xca", + "\xc6\xf5\xb1\xce\xe3\x10\x33", + "\xfd\x4b\x28\x27\x3d\x3e\xe8\xcc\x24\xde\x2d\x8d\xad\x23\xad\x4f\x35" + "\x52\x40\xc7", + 1, 56, NULL, 0 }, + { 160, 160, 9, + "\xe3\x22\x07\x00\xce\x24\xa0\x10\xcf\x62\x3f\x60\x89\x1e\x4f\x29\x8f" + "\xf2\x6b\x11", + "\xc9\x7a\xfb\x50\x63\xa9\xdd\x0d", + "\x38\x3b\x10\x3c\xe9\x05\x4c\xb7\x4a\x04\x31\xd1\x6d\xa9\x9d\x82\x33" + "\xe9\x4f\xc2", + 1, 64, NULL, 0 }, + { 160, 160, 10, + "\x46\x6c\x06\x1d\xdc\xf3\xd9\xb2\x85\xa2\x90\x0f\x87\x25\x97\x1b\x73" + "\x3f\x85\x0f", + "\x89\x02\x4c\xed\xa7\xde\x3c\x11\x4e", + "\x1b\x81\xf1\x12\x76\x35\x23\x33\x83\xb6\xea\x5b\xa8\xfd\x68\xeb\x51" + "\x12\xef\x0a", + 1, 72, NULL, 0 }, + { 160, 160, 11, + "\xa8\x1f\x9f\x51\xb0\x41\xff\x29\xb8\xd7\x05\xbb\x40\x8f\x85\x4c\xcb" + "\xd7\xe5\xab", + "\x03\x2d\x86\x6a\x27\x07\x62\xcb\xae\x24", + "\xb7\x2b\xa0\xc8\x9d\x01\x02\x15\xa8\xf2\x80\x61\x6a\xcb\xd8\x64\x0f" + "\xe8\x6c\xec", + 1, 80, NULL, 0 }, + { 160, 160, 12, + "\xe6\x0d\x0b\x14\x88\x6f\xe6\xfa\x2c\x83\x93\x29\x20\x4d\x84\xd8\x10" + "\x26\xb7\xab", + "\xfd\xa7\xf4\x8c\x11\x10\x12\x55\xe0\x2c\x8d", + "\x2b\xe7\xbb\x54\x1c\xed\xe9\x78\xf5\x41\xe2\xca\xc0\xab\x64\x51\x06" + "\x0e\x3e\x83", + 1, 88, NULL, 0 }, + { 160, 160, 13, + "\xc0\x90\xef\x12\x2a\x29\x34\x87\x40\xcc\xd5\x71\xd9\x84\x07\x76\x4b" + "\x2a\xda\xac", + "\xd1\xfe\x3d\xfa\x80\xad\xe7\x08\x7e\xfa\xbb\x52", + "\xe6\xc1\xe0\xc3\xeb\xb7\x75\x0d\x66\xa5\x0b\x6a\xbc\xcf\xde\xf9\xc2" + "\x59\x90\x08", + 1, 96, NULL, 0 }, + { 160, 160, 14, + "\x56\x4a\x56\x29\x0e\x1a\xea\x05\x22\xf1\x90\x88\xa8\x8a\xb4\xdc\xe4" + "\xc7\xcd\xf2", + "\x42\x13\xbd\x3c\xda\xeb\xbb\x1e\xc1\xcc\x81\x86\x6a", + "\x6d\x1d\x58\x08\xc0\x85\xad\x51\x24\x87\xde\xbb\x57\xfb\x93\x51\x4b" + "\x20\x50\x75", + 1, 104, NULL, 0 }, + { 160, 160, 15, + "\xf8\x98\x45\x9d\x27\x2f\xd5\xe4\x3b\x06\x21\x56\xf4\x49\x58\xd8\x5d" + "\x97\xea\x3f", + "\x5e\x86\xb0\x55\x22\xeb\x65\xa4\xfb\x7b\x93\x2c\xec\xd5", + "\xb7\x78\xf4\x21\xc2\xd1\xe2\x70\x1e\x75\xda\x6b\xd1\xbc\x65\x37\x9b" + "\x80\xe8\x79", + 1, 112, NULL, 0 }, + { 160, 160, 16, + "\x7d\x5c\xc5\x3f\x46\x4e\x75\x94\x38\xee\x90\xb4\x7f\x2f\xe6\x7a\xa8" + "\x3d\x6b\x52", + "\x9f\x38\xea\x80\x12\x2b\x40\xf7\x42\xa0\x0c\x2e\x83\xe0\x85", + "\xc8\x0c\xe6\xd3\x3f\xe8\x68\x43\x2c\x26\x27\x66\xfd\x23\xbf\x43\x1e" + "\x31\x38\x82", + 1, 120, NULL, 0 }, + { 160, 160, 17, + "\x33\xe9\x14\x01\x75\x51\x9b\x2f\x16\x19\xb4\x48\x48\x33\x17\x63\xc7" + "\x56\xfa\xd4", + "\x7d\xe0\xfc\xcc\x83\xb5\x1c\x29\xe5\xeb\x1b\x65\x8c\x10\x24\x38", + "\x34\xde\x6b\x8f\x47\x95\x23\x87\x0b\x8f\x90\x56\x84\x67\x26\x17\x66" + "\x9b\x06\x07", + 1, 128, NULL, 0 }, + { 160, 160, 18, + "\x0e\xf2\x9e\x7c\x96\x1d\xa3\x7a\xfa\xea\x81\x82\xf2\x87\x38\xd2\x2c" + "\x34\x02\x32", + "\x16\x5b\xb8\xe5\xc6\xf0\xa3\xae\x40\x94\x6d\xc8\x07\xae\xe8\x46" + "\x45", + "\x78\xe6\xfa\x53\xec\x21\x3e\x90\x19\xd4\x7e\xe7\x52\x9d\x96\x3a\x8a" + "\x25\x29\x42", + 1, 136, NULL, 0 }, + { 160, 160, 19, + "\x20\x3c\xfa\xd9\x21\xe6\x05\xc8\x0d\x7a\xa8\xb6\x4d\x3b\xf1\x83\x28" + "\xb7\xa7\xa0", + "\xd2\x89\xc7\xcd\x10\xd9\x96\xd5\xda\xca\x14\x10\xc3\x78\x15\xb2\x37" + "\xf7\x49\x29\x58\x8c\x5a\xe4", + "\x27\xd9\x6d\xa4\x18\x95\xbf\x53\xd1\x50\xac\x15\xe7\xc3\x18\x53\xf5" + "\x6a\xe3\x63", + 1, 192, NULL, 0 }, + { 160, 160, 20, + "\x8e\xb7\x41\x6e\xfd\x0c\x73\xc8\x6b\x91\xdf\x0d\x58\x89\x1f\xdb\x73" + "\x8f\x40\xdf", + "\xb4\x15\xcb\x7c\xd3\x84\xa1\x03\x5d\x2b\xac\x1f\x7b\x96\xae\x85\x8d" + "\xfd\x44\xc4\x67\x03\x0f\x30\x4e\x81\x7d\x11\xb9\xf9\xc6\x06", + "\x24\xcb\x16\x32\x3b\x7e\xc4\x7e\x3a\xdd\x8f\x55\xcb\x99\x20\xaa\x7c" + "\x16\x55\xde", + 1, 256, NULL, 0 }, + { 160, 160, 21, + "\x6f\xbb\x3c\x55\xe9\x35\xe0\xa0\x02\xc1\x70\xa9\x12\x2f\x1f\x70\x37" + "\xbc\x0c\x59", + "\x36\xac\x9a\x8c\xf0\x22\x3c\xcf\x5d\x90\x48\xbe\x9a\x65\xdf\x4a\x1f" + "\x40\xaa\xa8\x57\xce\x13\xd6\x21\xf6\x01\xbd\xee\x1f\xbe\x80\x31\x71" + "\x00\x2d\x1f\xa6\x34\xa1\x97\x7d\xc2\x3d\x9a\xa8\xfd", + "\xe2\xfe\x8b\x34\x3c\xef\x4b\x97\x54\x30\x84\x08\x93\x05\x26\x15\x95" + "\x37\xec\xc8", + 1, 376, NULL, 0 }, + { 160, 160, 22, + "\xe4\x0f\x62\x06\x10\x5f\x78\x00\xa1\xf1\x90\x60\x2b\xb6\xdd\xe8\x05" + "\x7c\x3a\x87", + "\x10\x46\x3b\x77\x1f\xa5\x86\xc5\xed\x5c\x1f\x64\x88\xd7\x93\x29\x9d" + "\xb4\x0f\xdd\x4f\x3e\x53\x33\x4a\xe3\xff\x8e\x09\xe5\xa8\x79\xda\x06" + "\xeb\x46\xd2\x10\xee\x0a\xf0\xc8\x25\x1e\x6c\x07\xaa\x1d", + "\xb5\xdf\xce\x59\x98\xd2\xe3\x21\x80\x0e\x0e\x42\x76\x2e\x62\xec\x7a" + "\x81\x44\x8f", + 1, 384, NULL, 0 }, + { 160, 160, 23, + "\xf0\x4c\xc6\x41\xff\x67\xab\xa4\xac\x2d\x17\xe6\xa0\x42\xb6\xcc\xf8" + "\x6a\xe1\xd2", + "\x73\xc1\x79\xac\xc2\x6f\xfd\x07\x10\xb6\xcb\x3f\x73\x57\x07\x02\xc9" + "\xc0\x59\xbf\x68\x56\x14\xbb\x0b\xa7\x97\x3a\xb8\x75\xff\x88\x2d\x9a" + "\xee\xce\xa4\xef\x45\x2c\x88\x93\x22\x44\x72\xcf\xa5\xb6\x1c", + "\xb7\xd1\xe6\x3f\xae\x54\x63\x80\x82\xa9\xcb\x58\xc6\x9f\xac\x9e\xfc" + "\xbe\xe1\x74", + 1, 392, NULL, 0 }, + { 160, 160, 24, + "\xf6\x1c\x1a\x87\x85\x50\xd2\x7a\xa4\x59\xb3\x01\x6b\x31\x73\x1b\x89" + "\x63\x0d\x36", + "\x82\xb3\x78\xd4\x0c\xa0\x4a\xd4\x78\xa9\x80\xd7\xb4\x6e\x56\xc9\x96" + "\x7b\xc4\xe1\x10\xa7\xad\xd8\xbc\xbd\xa4\x11\xc1\x2d\xe3\x84\xf4\x13" + "\x24\xe9\xdf\x88\x8d\x81\x70\x2f\xf2\xb9\xe8\x75\x29\x86\xba\x08\x13" + "\x63\xea\xcc\x2e\x39\x6f\x6b\x5f\xb0\x1b\xf8\x42\x35\x8f\x01\x45\xd5" + "\x69\xd3\x4f\xb3\xb4\xe2\x4e\xe9\xdc\x91\x03\x28\x4d\x74\x3c\x52\xea" + "\x86\x61\x50\x4b\x2d\xb4\x2f\x22\x1b\x6d\x49\xb6\x05\xfd\xe3\x4a\xa5" + "\x55\xe3\x3a\xb0\xa1\x40\xf6\x1f\x3c\xda", + "\xe6\x88\x19\x94\x89\xc9\xd3\x93\x8f\x2e\x33\xd7\xcb\x3f\xc8\x1b\xad" + "\x4f\xfb\x8c", + 1, 896, NULL, 0 }, + { 160, 160, 25, + "\x8c\x29\xeb\x66\x1f\xb6\x33\x08\x7f\x24\x52\xd0\x57\xf9\x8d\x55\x3d" + "\x28\x46\xf1", + "\x02\x1b\x96\x8c\x4c\xe3\x37\x59\x51\x54\xd9\x0e\x44\x22\x99\x80\xf0" + "\xe2\xb6\x47\x76\xf5\x62\xea\x25\xb2\x48\x81\x63\x7b\x44\x37\x5b\xde" + "\x65\xe5\xf9\x41\x8b\xf1\x63\xe2\xaa\xcd\x37\xbd\x10\x31\x97\x29\xac" + "\x59\x66\x15\xa3\x5c\xb6\x32\xe0\xff\xc3\x16\x93\x6a\x68\xac\xf4\xc7" + "\xae\x3a\xd3\x60\x26\x12\x4c\xee\x6d\x20\x4f\x10\x43\x2f\x08\x15\x7c" + "\xc3\x2c\x5f\x4b\xca\xda\xee\x67\xbd\x42\xbb\xeb\x82\x6a\x9e\x9c\x8a" + "\xf9\xf5\x54\xf7\x41\x9f\xb2\x65\x33\x8d\x22\xba\xe2\x19\x0b\xb6\x44" + "\xb3\x2f\xe9\xbb\x6a\x22\x87\xaa", + "\xf9\x40\xdf\x33\xb0\x99\x65\xa3\x11\x8c\x84\x7c\x2a\xe1\x59\x16\x90" + "\xd0\x40\x5f", + 1, 1016, NULL, 0 }, + { 160, 160, 26, + "\x63\x16\x29\x8f\x3a\xad\xad\xc6\x64\xed\xa2\xce\xdf\x17\x66\x9b\xc8" + "\x0d\x44\xae", + "\xba\xb8\x07\xdf\x54\xc0\x09\x61\x0a\x5c\x3f\x1e\x81\x60\x5f\x6b\xf7" + "\xd7\x6b\x29\x9d\x7e\xbd\xef\xa7\x0f\x5e\x2e\x0b\x97\x90\x11\xd1\x91" + "\xea\xd3\x9c\x3b\xbe\x5d\xd2\x65\x83\x47\xeb\x17\x29\x50\xa1\xe0\x3a" + "\x01\x55\x2b\xb3\x8a\xdd\x33\xba\xc8\x32\xb7\x17\x7a\x77\xb0\x8e\xb1" + "\x1c\xc1\xaf\xe3\xae\x84\xda\xff\xe4\xc4\xe8\x8b\xc4\x41\xe5\x4e\x4d" + "\xcb\xae\x3e\x0d\x56\x39\xf6\x35\x22\x8d\x81\x1f\x0a\x04\x3b\x13\xd5" + "\xc9\x18\x99\xc2\x6b\xce\x2d\xa2\xdd\xab\xd2\x1b\x2e\xe6\x68\xa2\x1b" + "\x45\x49\x28\x91\x5d\x65\x85\x40\x8d", + "\x23\xb9\xa6\xd6\xa9\xc7\xce\xf6\xdc\xe5\x37\x72\x2f\x45\x57\xb6\x5d" + "\xcd\xde\x99", + 1, 1024, NULL, 0 }, + { 160, 160, 27, + "\xf2\x91\x69\x6b\xf4\xf9\x65\x5a\x00\xc9\xa2\x38\x2b\xd1\x48\x73\x42" + "\x35\x87\x14", + "\x32\xa6\x50\xb5\x30\x7d\x94\xb2\x31\x39\xbe\x64\xd4\x70\xef\x14\x92" + "\xd5\x7c\xa7\xaf\x98\x20\x5b\xf9\xbc\xe8\x85\x4b\xa8\xf5\x20\x48\x80" + "\xb2\xe9\xd5\x8d\xdb\xe2\xe7\xbb\x21\xe6\xe0\x67\x3f\x5e\x1a\x39\xf5" + "\x05\x90\x92\x27\x47\x5e\x41\xc1\xd5\x9c\x73\xa9\x33\xb1\x3f\x4b\x07" + "\xa7\x5c\xb9\xf3\x27\x9c\xc3\xbf\x61\xa6\xc0\x9e\x3b\x9f\x75\x59\x07" + "\x49\x1b\x9e\x74\x5b\xfd\xa5\x8a\xd4\xe2\x30\x4f\xf7\x52\x5b\x41\x50" + "\x7a\x51\xa2\xfd\x66\x4a\x2e\xe9\x8c\xea\x00\x14\x8a\x36\x63\xd7\x7c" + "\x47\xbe\xb0\x55\xbd\x45\xe7\xdf\x48\xf6\xa0\xce\x66\xc0\xa2\xd9\xa8" + "\x48\x76\x1a\x45\x75\xd1\x95\xd7\x4e\xef\x5b\xb7\x8c\x09\x93\x55\x7a" + "\x25\xad\x7c\xa3\x2e\x0a\x96\xb2\x51\x8d\x9d\x8a\x18\x0f\x35\x74\x02" + "\xa4\x42\x17\xf1\xe3\x6a\x91\x38\xc0\x90\x9f\xaf\xfd\x0e\x9a\x90\x70" + "\x48\x58\x4b\xb0\x3a\x4e\x06\xfc\x69\xc4\x63\xf3\x95\x42\xdd\x2c\x7c" + "\x81\x46\x7d\x37\x28\x48\x1b\xbf\x6b\xb6\x02\x59\x60\x4a\xa3\x3a\x2d" + "\x4c\x61\x95\x01\x2f\xdc\x7a\xec\x99\xe2\x17\x5a\xeb\x2d\x0c\x1f\x68" + "\x09\x64\xd6\x3e\xe1\x14\x18\xcd\x4d\x26\xe7\x7e\xc1\x31\x10\x84" + "\x17", + "\x33\x46\xbf\x23\xe5\x22\x31\xa4\xae\xd7\x73\xfd\x73\xe5\x8d\x91\x85" + "\x80\xed\xe5", + 1, 2040, NULL, 0 }, + { 160, 160, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x07\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x04\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe6\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x86\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x64\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe9\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x65\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\xd0\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x80\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfd\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xcf\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfe\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xcc\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x02\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x92\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x37\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x07\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\xb6\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x86\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x41\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x1d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd8\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xeb\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe3\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x06\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe0\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x05\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\x62\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x87\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd8", + 0, 0, NULL, 0 }, + { 160, 160, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x21", + 0, 128, NULL, 0 }, + { 160, 160, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xdb", + 0, 0, NULL, 0 }, + { 160, 160, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x22", + 0, 128, NULL, 0 }, + { 160, 160, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\x99", + 0, 0, NULL, 0 }, + { 160, 160, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x60", + 0, 128, NULL, 0 }, + { 160, 160, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\x59", + 0, 0, NULL, 0 }, + { 160, 160, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\xa0", + 0, 128, NULL, 0 }, + { 160, 160, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x07\xe8\xad\x50\xfc\x10\x35\x82\x37\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x64\x16\x00\xce\xdd\x7e\x12\x07\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\xd0\xfc\x10\x35\x02\x36\x61\xd9\x79\xe2\x96\x89\x68\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x80\xce\xdd\x7e\x92\x06\x3d\xea\xea\x07\x88\x78\x5f\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x06\xe8\xad\x50\xfc\x10\x35\x02\x36\x61\xd9\x79\xe2\x96\x89\xe8\xce" + "\xcd\x03\xd9", + 0, 0, NULL, 0 }, + { 160, 160, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x92\x06\x3d\xea\xea\x07\x88\x78\xdf\x56" + "\x11\x35\x20", + 0, 128, NULL, 0 }, + { 160, 160, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\xf9\x17\x52\xaf\x03\xef\xca\x7d\xc9\x9e\x26\x86\x1d\x69\x76\x97\x31" + "\x32\xfc\x26", + 0, 0, NULL, 0 }, + { 160, 160, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x1b\x9b\xe9\xff\x31\x22\x81\xed\xf9\xc2\x15\x15\xf8\x77\x87\xa0\xa9" + "\xee\xca\xdf", + 0, 128, NULL, 0 }, + { 160, 160, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00", + 0, 0, NULL, 0 }, + { 160, 160, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00", + 0, 128, NULL, 0 }, + { 160, 160, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff", + 0, 0, NULL, 0 }, + { 160, 160, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff", + 0, 128, NULL, 0 }, + { 160, 160, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x86\x68\x2d\xd0\x7c\x90\xb5\x02\xb6\xe1\x59\xf9\x62\x16\x09\xe8\x4e" + "\x4d\x83\x59", + 0, 0, NULL, 0 }, + { 160, 160, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x64\xe4\x96\x80\x4e\x5d\xfe\x92\x86\xbd\x6a\x6a\x87\x08\xf8\xdf\xd6" + "\x91\xb5\xa0", + 0, 128, NULL, 0 }, + { 160, 160, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", + "\x07\xe9\xac\x51\xfd\x11\x34\x83\x37\x60\xd8\x78\xe3\x97\x88\x69\xcf" + "\xcc\x02\xd8", + 0, 0, NULL, 0 }, + { 160, 160, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x65\x17\x01\xcf\xdc\x7f\x13\x07\x3c\xeb\xeb\x06\x89\x79\x5e\x57" + "\x10\x34\x21", + 0, 128, NULL, 0 }, + { 160, 80, 82, + "\x5e\xce\x07\x69\x74\x2f\xea\xbb\x66\x44\x46\x9c\x9b\x26\x43\x26\xb3" + "\xde\xb1\x26", + "", "\x34\x4f\x83\x51\xf1\xd2\x77\x3c\xae\x9e", 1, 0, NULL, 0 }, + { 160, 80, 83, + "\x4e\xe9\xf9\xa9\x3b\x2d\xdf\xe5\x51\x28\x1b\x39\x7c\xce\xf8\x44\xfc" + "\x21\xaf\x3a", + "\x2d", "\x3a\xab\x1a\x2c\x9a\x2f\x2b\x8a\xc8\x40", 1, 8, NULL, 0 }, + { 160, 80, 84, + "\xdf\xee\xbe\x9a\x5c\x18\x1a\xfc\x60\x5f\xf6\x3b\x22\xbf\x34\x9e\xbd" + "\xb6\xc7\xfb", + "\xa5\xf3", "\xc3\xb7\x15\x22\x30\xda\xca\xe4\xef\x48", 1, 16, NULL, + 0 }, + { 160, 80, 85, + "\xc5\x14\x7e\xcd\x59\xb7\xd4\x23\x15\xd5\xe3\xa5\x5e\xc8\xb3\xa3\x20" + "\xc8\xd6\x15", + "\x37\x17\x77", "\x0a\xbb\x78\xa2\xc6\x7c\x56\x5f\x89\xb1", 1, 24, + NULL, 0 }, + { 160, 80, 86, + "\xdb\x0d\xa5\x65\x9b\xa6\x9c\xe1\x95\xa6\x95\x24\x50\x8e\x43\x7c\x68" + "\x8f\x71\x47", + "\x4e\xf4\xec\x44", "\x40\x62\xfa\xeb\x0b\x40\x66\x98\xb7\x40", 1, 32, + NULL, 0 }, + { 160, 80, 87, + "\x49\x5d\x6c\x11\x91\x85\x2e\xca\xf0\x57\x3e\x6a\x77\x61\x0c\x32\xac" + "\xf5\xa1\x17", + "\x15\x66\xae\x63\xce", "\x45\xcb\xef\xcd\x9c\x7e\xee\xe3\x7a\xe5", 1, + 40, NULL, 0 }, + { 160, 80, 88, + "\xe5\xcf\xe7\xdc\x67\x51\x4c\x4c\x75\xd2\x8b\xb8\x05\xd0\x70\x0b\xde" + "\xa0\xd6\x69", + "\xaf\x1a\x6b\x15\xb6\x22", + "\xe8\x67\x26\x9d\x50\x6a\x37\xe8\xa6\x2d", 1, 48, NULL, 0 }, + { 160, 80, 89, + "\xcf\x1b\x8b\x90\x25\x12\x18\x6e\x38\xc3\x81\x65\xd6\xe5\x87\xbc\xee" + "\xca\xe8\x7f", + "\xa5\x95\x12\x15\x2c\x72\x21", + "\x1a\x12\x5b\x21\x41\x2c\xdc\x59\x68\x94", 1, 56, NULL, 0 }, + { 160, 80, 90, + "\xd0\x15\x7f\xb4\x0c\x77\x39\xae\x50\x6a\xad\x7d\xe6\x0f\x32\xcc\xc3" + "\x32\x55\x83", + "\x12\x92\xdf\x8d\x53\xd1\x6f\x3c", + "\x92\x24\xf1\x1b\xff\x0e\x49\xb9\xaa\x95", 1, 64, NULL, 0 }, + { 160, 80, 91, + "\xfd\x55\xb8\x1e\xdd\x55\xa1\x5b\xff\x40\x91\x29\xe9\x93\x0f\x1b\xa1" + "\x76\x3c\x33", + "\x05\x22\x0a\x69\x97\x53\x3c\x69\x9b", + "\x3b\x2d\x07\xdc\x8c\xa2\x06\xba\x16\xc4", 1, 72, NULL, 0 }, + { 160, 80, 92, + "\x07\x99\x37\xcf\x3b\xd4\x28\x64\xd5\xb1\x5c\x62\xbd\xd9\x2f\x27\x55" + "\x97\x31\x6d", + "\x5a\xc1\x3c\xe1\xb1\xf7\x77\x24\xe2\x81", + "\x0c\x24\xaf\xcd\xad\xd8\x53\x89\x77\xb1", 1, 80, NULL, 0 }, + { 160, 80, 93, + "\x54\x5b\x13\xe1\xf3\x9f\x0b\x7c\xa9\x25\x2b\xc5\x96\x27\x72\x78\x16" + "\x6a\xd4\x10", + "\x08\xa7\xbc\x90\x73\x2d\x54\x38\x1b\x6e\x30", + "\x9b\xf7\xf1\x21\x36\x5a\x82\xc2\xac\x69", 1, 88, NULL, 0 }, + { 160, 80, 94, + "\x25\x38\x5e\x9f\x89\xb6\x60\x98\xee\x81\x62\xae\xca\x03\xbb\x45\xb3" + "\x13\x56\x1f", + "\xbf\xf2\x36\xaa\xd7\x1f\xb5\xda\xf7\xfc\x43\xb8", + "\x44\xdb\x86\xe7\xa1\x47\x62\x26\xdd\x86", 1, 96, NULL, 0 }, + { 160, 80, 95, + "\x9b\x68\x13\x9d\x93\xa8\x8f\xe3\x4c\xf9\xf8\x30\x06\xc0\x3b\x31\x64" + "\xb6\x04\x68", + "\xad\x67\x2b\x97\x19\xc1\x08\x63\xfd\x6f\xa8\xdb\x88", + "\x2d\x17\xa8\x8d\x87\xaa\xe7\xdb\xac\xed", 1, 104, NULL, 0 }, + { 160, 80, 96, + "\xc3\xb7\x85\x91\x5e\x13\x75\x44\xda\xc5\x42\xcb\x4b\xdb\x16\xd5\x30" + "\x36\xfb\x11", + "\xf8\xc1\xed\xb4\x69\xb9\x3c\x07\x3b\x6f\x6b\xf7\x4c\xca", + "\x16\x71\x3d\x61\xfb\xb4\x14\x9f\x50\x0b", 1, 112, NULL, 0 }, + { 160, 80, 97, + "\xda\x67\x47\x51\x85\xb3\x61\x50\x55\xf9\x71\x81\x9d\xb2\x78\x71\xb2" + "\x3c\x75\xd0", + "\xa1\x76\x53\x33\x19\xbe\xe5\xe4\x3d\x8f\x0e\xaf\xb7\x7b\xb3", + "\xfc\xee\xce\x89\x28\x52\xd4\xa2\x60\x70", 1, 120, NULL, 0 }, + { 160, 80, 98, + "\x0c\xf1\x46\xca\x7a\x25\x4d\xb1\xe0\x01\xa2\x9a\xd0\x3c\x5e\x6d\xcb" + "\xe7\x14\x0a", + "\xa8\x3d\xf5\xd0\x99\x85\x4e\xb6\xea\xd7\x03\x1c\x51\x46\x03\x57", + "\x9b\xcf\x75\x13\x20\x6e\x27\xa4\x69\x7d", 1, 128, NULL, 0 }, + { 160, 80, 99, + "\xe0\x38\xdf\xf0\x28\x22\x7d\xc4\xb4\xd7\x45\x3d\xb3\x07\x01\x08\x46" + "\x5d\xd5\xb2", + "\x7a\xe4\xe3\x08\x34\xdb\x44\x9e\x42\x44\xa9\xfc\x03\x22\x19\x3e" + "\x7a", + "\x9a\xa8\x54\x4a\x9a\xfd\xd9\x20\xc0\xf2", 1, 136, NULL, 0 }, + { 160, 80, 100, + "\xb3\x99\xfc\xfd\x1a\xd3\x21\x40\x87\x9a\xa0\x55\x6a\xc3\x4d\x8b\x5a" + "\xc2\x67\xf2", + "\x0e\x3f\x0f\xc5\xcb\x14\x56\xfe\xde\x99\xf8\x6a\x05\x6f\x64\x0b\x8f" + "\x5e\x5e\x1b\x61\x2f\x25\xf6", + "\xfa\x09\x5c\x6f\xae\xd0\xf0\x86\xb2\x15", 1, 192, NULL, 0 }, + { 160, 80, 101, + "\x2b\xf7\xd2\x01\xef\x44\x24\x1a\x22\xae\x4b\x81\xaa\xb9\x10\xd2\x2c" + "\x2d\xb9\x18", + "\xaa\x0a\xfa\xf3\xaf\x36\x54\x82\x27\x34\x9a\xdc\xfc\xb6\xbf\x99\x8a" + "\x7f\xa7\x8d\x29\xb8\x7a\x0f\x50\x60\x9c\x42\xed\xcd\xb3\xdd", + "\x70\x8e\xc4\x5d\x41\x0b\x1f\xe0\x75\xc8", 1, 256, NULL, 0 }, + { 160, 80, 102, + "\x48\xcc\xc3\x90\x7c\x36\x12\xa1\x82\x94\xfd\xdf\x26\x60\xe3\x3d\x9c" + "\xb7\x87\xfc", + "\xed\xbb\x68\x02\x43\xa8\x25\x06\x8e\xef\xe5\xba\x18\x4e\x5e\xed\x4b" + "\x7f\x85\xca\x3b\x51\x1a\x42\xd6\x55\xbe\x3e\x05\xd8\xff\x12\x45\x41" + "\xb3\xd5\x6a\x10\xa3\x5c\xff\x8d\xa8\xb6\x22\x9a\xc1", + "\x3c\xb7\xfc\xe2\x0d\xf8\x38\x5c\xf6\xbb", 1, 376, NULL, 0 }, + { 160, 80, 103, + "\x22\x7d\x79\x6b\x78\x67\x40\x9d\xb3\xde\x1f\xfa\x3c\xfe\x37\x67\x04" + "\x04\x4f\x01", + "\xb6\x39\x3a\xb1\x83\x76\xc0\x25\xe2\xd8\xe0\x0c\xa7\x74\xa5\x1a\xec" + "\x19\xdc\x4a\x89\xcf\x6a\x9f\x8f\xc4\xab\xa8\x1d\x73\xb3\x90\x7e\xfe" + "\xf1\xa0\xd0\x18\xa5\x3c\xb8\xb8\xca\x10\x32\xe3\x15\x83", + "\x52\x53\x87\xc8\x1c\x2a\xb6\x7a\xca\x74", 1, 384, NULL, 0 }, + { 160, 80, 104, + "\x57\x18\xe7\x00\xc4\x8a\x79\x71\x35\x0d\x8a\x11\xb3\x77\x54\xae\x55" + "\xa9\xaa\xd2", + "\xcc\x95\x28\x6e\x9b\x3c\xa9\x36\x19\x1a\xff\x87\x31\xe6\xa1\x78\x06" + "\xa0\x95\x8b\x0b\x1a\x39\x97\x7c\x46\x39\x52\x40\x64\x1e\x97\xd5\x39" + "\x5a\x9c\x8a\x9d\x36\x28\x1e\xba\x82\x5a\x94\xe8\xb1\xad\x79", + "\xe5\x47\x82\x11\x0d\x40\xef\xb5\x43\x43", 1, 392, NULL, 0 }, + { 160, 80, 105, + "\xf5\x6a\xa7\x92\x79\x5e\xb0\x3a\xe0\x99\x04\x40\x71\x4a\xb1\x6c\xc4" + "\xad\x18\xc3", + "\x7a\x02\x4d\x99\x5a\xdd\xd3\x8d\x96\x7b\xc3\xb4\x16\x41\x73\x8b\x69" + "\x89\x7d\x8c\x52\xb7\xaf\xf9\x61\xa7\x00\xcb\x68\xfa\x74\x81\xda\x0a" + "\x36\x90\xa1\x51\xce\x09\xc9\x5b\x4d\xa6\x0f\x7c\xf2\x89\x90\x01\x72" + "\x92\x89\x3b\xbb\x2f\x81\xa4\xdd\xa4\x5f\xe8\x63\x98\x77\xac\x5a\xba" + "\xeb\xbe\x00\xc1\xfd\x17\x9e\xaa\xf7\xdf\xb4\xd5\x09\x29\x37\x1b\x9a" + "\xb8\xb7\xd3\x53\x1a\x63\xab\x18\x8d\x7b\x99\x16\x00\x60\x47\x5c\x33" + "\xe8\x33\x51\xf6\x5d\x5e\x32\x9e\xe8\xbf", + "\x1e\x22\x05\xd1\x7a\xd4\xde\x3f\x1e\xc1", 1, 896, NULL, 0 }, + { 160, 80, 106, + "\x00\xbe\x00\x34\xd3\x26\x99\xb1\x33\x5d\x8d\x4e\x50\x62\x35\xee\x4f" + "\x07\xbe\xf6", + "\x63\x9e\x82\x8d\x88\xbf\x06\x42\xbe\x0a\x54\x1b\x1c\x3f\xca\x07\x60" + "\x9e\xb9\x8d\x23\xa8\xb2\xcd\x4e\x60\xe1\x39\x51\x5e\x4f\xf4\x40\xdf" + "\xda\x1b\xc1\x93\x92\xfe\xff\xed\x74\x16\x4d\x6a\x9d\x8f\x5b\xaf\xe5" + "\x3f\xd3\x97\xcb\x5e\xe1\xdc\xdf\x9b\xfc\x86\x16\x9f\x1b\xc3\x8b\xa5" + "\x7f\x88\xd7\xe8\xc6\x72\x8c\x35\xfc\x07\x12\x8a\xb6\xc3\x96\xbb\x3e" + "\xf3\xc1\x4d\x13\xa0\x5f\x8c\x34\x53\x35\x3e\x85\x0d\xc1\xb2\x91\xac" + "\x70\x61\xab\x52\xf1\x21\x66\x3f\x18\xb0\x24\xe5\xcc\x00\x68\x32\x8c" + "\x88\xf5\x2c\x20\xcd\x21\x79\x3a", + "\x6e\x98\x97\x3d\x3a\x77\x5a\xc5\x08\xe7", 1, 1016, NULL, 0 }, + { 160, 80, 107, + "\xc5\xba\xa7\x50\xa8\x42\x44\x50\xf1\xb4\xd4\x53\xc5\x8e\x29\xc4\x62" + "\xe5\x26\x39", + "\x13\x7c\x72\x27\xa1\x92\xbe\xd2\x6d\x08\xda\x88\x64\x30\xf0\x10\x09" + "\x42\x43\xb5\xc4\x68\x6e\x68\x31\xe4\x8d\xb4\x50\x04\x5a\xa1\xd7\xe3" + "\xae\xcf\x19\x3e\xaa\xa1\xa7\x39\x05\xf5\xf1\x19\x06\x59\xa4\x3e\xd4" + "\xd1\x0b\xfc\xa5\x66\x8e\xbe\xb3\x43\xb2\x1f\xf7\x1d\x07\x37\xf8\x1f" + "\x67\x39\x2b\x64\x59\xaa\x95\xf9\x44\x1f\x69\x9b\xf4\x5f\xee\x24\x86" + "\x7a\x98\xa8\xa6\xc5\x7f\x97\x2a\xbe\x3e\x40\x0f\xd6\x4c\xe3\xe5\xb4" + "\x86\x22\xa0\xe9\x9e\x08\xd4\x24\x25\x0f\xa0\x0e\xd0\xdf\xa1\x19\x3f" + "\x93\x6c\x78\xaf\x27\x6a\x4b\x44\x2b", + "\x49\x0d\xfd\x2d\x5e\x6e\xa1\x30\xf6\xa1", 1, 1024, NULL, 0 }, + { 160, 80, 108, + "\x6b\xd4\x86\xce\x93\x4c\x2f\x5f\xe3\x8a\x19\x42\x3d\x25\x7b\xc5\xd8" + "\x08\xe3\x67", + "\xb8\xa6\x84\xad\xa0\xa0\x14\x05\x61\x4b\x1f\xa6\x6e\xba\xb8\xb0\x35" + "\x6e\x33\xb8\x89\xb8\x1b\x3e\xb6\x8d\x13\xb0\x5c\x4e\x60\xc7\x24\x78" + "\x5e\x63\x4c\x4e\xc0\x08\x1c\xc6\xbb\xdf\x21\x3d\xb7\x25\x4f\x92\xc0" + "\xa8\x58\xbf\xcc\x3d\x63\xa4\xe4\xdf\xd9\xe7\x5b\xd4\x83\x9a\xc0\x57" + "\x51\xc2\x3c\xb5\x99\x45\xf4\xc3\x66\x0d\x2b\x30\x09\xf5\xb0\x8a\x59" + "\x6b\xdc\x33\x07\x02\x44\xbc\xee\xa1\x11\x80\xac\x09\x06\x40\x45\x18" + "\xb0\x9d\x5d\x86\x12\xe0\xd8\xe6\x9f\x4b\x9e\x55\xbb\xc0\x53\xb5\x65" + "\x74\x71\x1b\x02\x95\x6d\xb3\xf3\xbd\x8f\x6c\x42\x06\x58\x71\x25\x58" + "\x54\xa1\x61\xe5\x71\x00\xad\xb8\x2c\xba\x79\x89\x3a\xad\x71\x5d\xc3" + "\xdf\x14\x88\xb3\xed\xb5\x6e\x58\xb8\x9c\x0b\xe3\xcf\xab\x09\xa3\xdf" + "\x40\x52\x4d\x2d\x32\x51\xb0\xfc\xf7\xfa\xab\xfc\x75\xf5\x00\x26\x79" + "\x50\x60\xc1\xd6\x28\x72\x57\x4a\x76\x9e\x3d\xa0\xe1\x9a\xf1\xb5\xe2" + "\x55\x14\xae\x17\xa1\x60\xc8\xd1\xeb\x25\x3c\x9f\x66\xec\x3d\xf7\x89" + "\xec\x0c\x6c\x70\x4a\x9e\x2f\xe8\xef\x7e\x9b\xf8\xe8\x16\x4f\x86\xd0" + "\x9d\x2a\x23\x69\x87\x33\xa8\xe4\x0a\x27\x9c\xd5\xfe\x02\xc2\x95" + "\xff", + "\xa5\x94\xd2\x6d\x98\xb5\x3b\x40\x63\xb2", 1, 2040, NULL, 0 }, + { 160, 80, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x07\xe8\xad\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x04\xe8\xad\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe6\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x86\xe8\xad\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x64\x64\x16\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe9\xad\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x65\x16\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xac\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x17\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xaf\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x14\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\x2d\x50\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x96\x00\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\xd0\xfc\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x80\xce\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfd\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xcf\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfe\x10\x35\x82\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xcc\xdd\x7e\x12\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x02\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x92\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x37\x61", 0, 0, NULL, 0 }, + { 160, 80, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x07\x3d", 0, 128, NULL, 0 }, + { 160, 80, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\xb6\x61", 0, 0, NULL, 0 }, + { 160, 80, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x86\x3d", 0, 128, NULL, 0 }, + { 160, 80, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x60", 0, 0, NULL, 0 }, + { 160, 80, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3c", 0, 128, NULL, 0 }, + { 160, 80, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x63", 0, 0, NULL, 0 }, + { 160, 80, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x3f", 0, 128, NULL, 0 }, + { 160, 80, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x41", 0, 0, NULL, 0 }, + { 160, 80, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x1d", 0, 128, NULL, 0 }, + { 160, 80, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\x21", 0, 0, NULL, 0 }, + { 160, 80, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\x7d", 0, 128, NULL, 0 }, + { 160, 80, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\x50\xfc\x10\x35\x82\x36\xe1", 0, 0, NULL, 0 }, + { 160, 80, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x00\xce\xdd\x7e\x12\x06\xbd", 0, 128, NULL, 0 }, + { 160, 80, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x07\xe8\xad\x50\xfc\x10\x35\x82\x37\x61", 0, 0, NULL, 0 }, + { 160, 80, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x64\x16\x00\xce\xdd\x7e\x12\x07\x3d", 0, 128, NULL, 0 }, + { 160, 80, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x06\xe8\xad\xd0\xfc\x10\x35\x02\x36\x61", 0, 0, NULL, 0 }, + { 160, 80, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe4\x64\x16\x80\xce\xdd\x7e\x92\x06\x3d", 0, 128, NULL, 0 }, + { 160, 80, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\xf9\x17\x52\xaf\x03\xef\xca\x7d\xc9\x9e", 0, 0, NULL, 0 }, + { 160, 80, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x1b\x9b\xe9\xff\x31\x22\x81\xed\xf9\xc2", 0, 128, NULL, 0 }, + { 160, 80, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, 0, NULL, 0 }, + { 160, 80, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, 128, NULL, 0 }, + { 160, 80, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, 0, NULL, 0 }, + { 160, 80, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, 128, NULL, 0 }, + { 160, 80, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x86\x68\x2d\xd0\x7c\x90\xb5\x02\xb6\xe1", 0, 0, NULL, 0 }, + { 160, 80, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x64\xe4\x96\x80\x4e\x5d\xfe\x92\x86\xbd", 0, 128, NULL, 0 }, + { 160, 80, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "", "\x07\xe9\xac\x51\xfd\x11\x34\x83\x37\x60", 0, 0, NULL, 0 }, + { 160, 80, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xe5\x65\x17\x01\xcf\xdc\x7f\x13\x07\x3c", 0, 128, NULL, 0 }, + { 80, 160, 159, "\x1d\x95\x35\xa0\xda\xea\x9d\xfe\x44\x3a", "", + "\x74\x36\x08\x9f\xed\xe3\x29\x1c\x0c\x42\x1c\x9a\xd1\x3c\x35\x7e\xc8" + "\x66\x0b\xae", + 1, 0, NULL, 0 }, + { 80, 160, 160, "\xf5\xc2\xc4\x20\xc6\xf0\x56\x46\x7f\xca", + "\xbc\x8a\x29\xf5\x2e\x57\x58\x1c\xb8\x9a\x86\xe5\xd6\x44\xa1\x4d", + "\x1e\xb7\x6d\xf7\x23\x5c\x52\x37\x1d\x86\x11\x3f\x54\x23\x62\x8b\x2e" + "\xb7\xc3\xd5", + 1, 128, NULL, 0 }, + { 80, 160, 161, "\xe5\x9b\x02\x76\xe2\x7a\x0a\xbc\x75\xf1", + "\x06\x1c\xa1\xa1\xaf\x51\xc5\x13\x37\x28\xc4\x14\xf9\x64\x6b\x3f\x50" + "\x22\x3e\x9b\x20\x55\x70\x70\x32\xe7\x54\xdc\x1d\x31\x96\x4b", + "\x20\x81\x26\x0f\x65\x31\x6d\xf2\x95\x6a\xac\x72\x3a\x9b\xd7\xd2\x22" + "\x5a\x86\x69", + 1, 256, NULL, 0 }, + { 80, 80, 162, "\xb1\x8a\xba\x11\x71\xcc\x2f\xfc\x7d\x58", "", + "\xde\xeb\x3d\x6d\x81\xe3\x3d\x1c\xad\x21", 1, 0, NULL, 0 }, + { 80, 80, 163, "\x23\x08\x20\x66\xe8\xc4\x5d\xa8\x2f\xc6", + "\x06\xc1\x9c\x6e\xe4\xd2\xf0\x15\x76\x9f\x6d\x46\xeb\x46\xd6\xb4", + "\x74\x7c\xd9\x28\xe8\x83\x19\x17\xc8\x55", 1, 128, NULL, 0 }, + { 80, 80, 164, "\xa6\xfa\x1e\x04\xdf\x38\xa7\x86\x67\xeb", + "\x51\xa4\xea\x38\xe5\x56\x6d\x6f\xd8\x03\xae\xc5\xe0\x73\xe0\x87\xe9" + "\xae\x00\xd3\x7d\x4a\x98\xd5\x59\x07\x4e\xbf\xfc\x76\x58\xb7", + "\xec\x8c\x20\x0c\x1d\xda\xd6\xd3\xaa\xd0", 1, 256, NULL, 0 }, + { 520, 160, 165, + "\xab\x92\xe2\xcd\x40\xe0\x0b\x40\xc4\x44\x2d\xd7\x67\x1c\x06\x7c\x77" + "\x92\xaf\x28\xe6\x0f\x25\x85\xe8\x7f\x16\x3b\xf3\xbd\xfc\xa7\xf5\x53" + "\xce\xc7\x1b\x00\x65\x02\x55\x00\xc4\x8e\x20\x70\x98\x4a\xd9\xe2\x4e" + "\x73\x31\x07\xeb\xfd\xe2\x71\x64\xa4\x82\x89\x81\xac\x20", + "", + "\x7a\x29\xb4\x7f\xf6\xae\x90\xc9\x95\x73\xd8\xc9\x22\xa2\x3e\x83\xa6" + "\x2b\x66\xbc", + 1, 0, NULL, 0 }, + { 520, 160, 166, + "\x3b\x47\xa5\xd5\xb7\x2b\xab\xe1\x16\xe6\x19\x19\x60\x0c\xb9\x80\xc9" + "\x04\xc2\x98\xab\x91\xfa\xe3\xdb\x9c\x82\xb0\xf3\x8a\x18\x88\x8b\xc0" + "\x5a\x41\x8d\x65\xd6\x8f\x88\x50\x93\x75\x59\xbb\x37\x32\x5b\xce\x04" + "\xd0\xe5\xd1\x75\xa2\x4f\xea\x30\x98\x95\xf5\x70\x5a\xd7", + "\xd7\x18\x62\x02\x8f\xca\xf1\x34\x22\xbf\x32\xac\x0c\x5f\x07\x9b", + "\x80\x19\x23\x1e\x77\xac\xa6\x45\x18\x26\x70\xca\xdf\x88\x7a\xfd\x4b" + "\x41\x15\xa7", + 1, 128, NULL, 0 }, + { 520, 160, 167, + "\x6e\x6c\x43\xdf\x9b\xb6\xc6\xb8\xfe\x41\x4a\x18\x3e\x73\x85\x08\xf0" + "\xac\xa4\x1d\x5b\xee\xf6\xdf\x1b\x26\x0c\x39\xe1\x97\x9b\x54\x68\x36" + "\x22\xa4\xd5\x33\x54\x23\x1b\xef\x6c\x35\xe1\x29\xf8\x5f\x82\x2b\xa0" + "\x91\x98\xaa\x30\xc6\x5e\xe6\x0e\x42\x02\xde\x8c\xd1\x02", + "\x98\xf0\xa4\xb9\xa3\x6e\x17\x3d\x89\x73\x0a\x3b\x37\x07\x77\xc4\x99" + "\xb4\xcf\xf2\x84\x6f\x50\xbf\xb8\x8f\xbb\xbc\x54\x7c\xba\xe4", + "\xd1\x65\x3c\x90\xfc\x59\x1e\x3a\x3c\x28\x5a\x3b\xe8\xb1\x2c\xa9\xb2" + "\x12\x1e\x88", + 1, 256, NULL, 0 }, + { 520, 80, 168, + "\x4f\x00\xfd\x17\xae\x82\xa6\x25\x2a\xda\x98\x28\x0b\xbd\x89\x5d\x74" + "\x3f\xc4\xc2\x0b\xc9\xe6\x15\xd8\xa7\x86\xc7\x9e\x45\x4c\x2b\x13\x41" + "\xe2\x42\x54\xfa\x03\x71\xfa\xc8\x6e\x7c\x0e\xf1\xa7\xdf\x5c\x16\xf3" + "\xb3\x56\x9f\xda\x11\x2c\xca\x86\x85\xfa\xec\xbb\x89\x23", + "", "\x68\x02\xca\x52\xbe\x05\x6d\x66\xb9\xa0", 1, 0, NULL, 0 }, + { 520, 80, 169, + "\xd2\x2e\xc5\x68\x90\x99\x90\xc2\x13\x67\x9f\x70\x72\xea\xf1\x97\x63" + "\x50\x8e\xbd\xe6\x96\x2c\x75\xe7\x42\x9c\x5f\x24\x54\xd4\xb5\x47\x28" + "\x11\xee\xa8\xe0\x2f\xdc\x89\xec\x38\x6b\xc6\xf4\x1d\x2a\xd8\xa9\x1d" + "\x11\x6b\x2c\xbc\x52\xb8\x0d\x35\x71\x27\xd1\x55\x5a\x66", + "\xf7\x1b\x43\xe0\xcc\x64\xb5\x40\x9e\x65\x01\xca\x55\xa8\xd4\x50", + "\xd4\x92\xa2\x96\x86\x0c\xc5\xa8\x9c\x5f", 1, 128, NULL, 0 }, + { 520, 80, 170, + "\xbc\xf6\xad\x6e\x5c\x7e\x22\x00\x29\x9e\xa8\x60\x2e\xfb\x42\xb4\x09" + "\x29\x23\x46\xf7\x8a\x0e\x57\xa7\x89\xba\x17\xb1\x7e\xd6\x08\xe8\x84" + "\x97\xe2\xbb\x4e\xbb\xbb\x3c\xe7\x75\x0d\x22\x2b\x3b\xdf\x84\x8d\x4d" + "\xc8\xd4\x9b\x5b\x60\x37\x8f\xb9\x3c\xe3\xf6\x6a\xb4\xeb", + "\x58\x6f\x5d\xdb\xc3\x72\xc0\x71\x1b\x77\xe4\xb8\x7d\x34\x5d\x62\xb6" + "\xde\x55\xa1\xce\x6f\xa1\x8d\xe3\x34\x6c\x86\xbe\x5c\xec\x6e", + "\x1d\xe9\xae\xfc\xc5\x31\x30\x24\x5a\x6e", 1, 256, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/hmac_sha224_test.json.c b/test/wycheproof/hmac_sha224_test.json.c new file mode 100644 index 00000000..c8724fce --- /dev/null +++ b/test/wycheproof/hmac_sha224_test.json.c @@ -0,0 +1,1219 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* HMACSHA224, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha224_test_json[] = { + { 224, 224, 1, + "\x7e\xef\x1e\x40\x25\x33\x50\xeb\x93\x07\xcc\x6b\xd8\xab\x8d\xf4\x34" + "\xbc\x2f\xaf\x70\x95\xe4\x5b\x50\xff\xdd\x64", + "", + "\x45\xb4\x66\x02\x12\x14\xd1\x92\x45\x50\x69\x00\x53\x2f\x52\x72\xf4" + "\x4b\x5a\xd9\xb3\xd8\x29\xf0\xf5\xc2\x10\x8c", + 1, 0, NULL, 0 }, + { 224, 224, 2, + "\x86\x48\xee\x93\x6c\x6e\xbc\x5a\xe4\xbb\x48\xc1\x13\x9a\x54\xe3\xac" + "\x5d\x89\x7b\xee\xc4\x92\xdc\x4d\x74\x07\x52", + "\x2e", + "\x5b\x72\xe3\x20\x86\x79\xe6\x3f\x92\x9e\x6e\xe1\x9a\x25\x7d\x05\x55" + "\xf2\x14\x84\xc7\xca\xac\x7c\x98\x61\xbe\x43", + 1, 8, NULL, 0 }, + { 224, 224, 3, + "\x22\x97\xd7\x8c\xc4\x5f\xaf\x9b\x88\x5b\x36\xac\x80\x20\x5c\xc0\x8e" + "\x1b\x73\x0f\x26\x4f\x23\xf4\xed\xbb\xb4\x06", + "\x32\x9f", + "\x2e\x7a\x81\xc4\xe2\x9a\x43\x5d\x91\xe9\x5f\x37\xfb\x0a\x62\xfb\xe9" + "\xa6\x9e\x06\x1f\x41\x6c\x1a\xd1\x7a\x7f\xca", + 1, 16, NULL, 0 }, + { 224, 224, 4, + "\x03\x61\xa9\x04\xf7\xcb\xd1\x07\xa6\x17\x61\x4a\xb6\x9d\x11\x20\x8e" + "\xe6\xd4\x23\xb3\xae\x90\xe2\xbb\x6d\x7e\x54", + "\xe6\xe7\x65", + "\xbb\xfa\x7f\xf9\x60\x93\x1e\x2f\x5e\xd8\xc9\x25\xcd\x74\x27\x29\x90" + "\xe7\x55\xf3\x14\x22\xe5\xc8\x58\x99\x5b\x73", + 1, 24, NULL, 0 }, + { 224, 224, 5, + "\x26\x4a\x8d\x21\x28\xe8\xfd\x09\x72\xd9\xac\xc6\x6d\xc2\x75\xb1\x28" + "\x6b\xee\xb0\xaf\xf7\xce\x8e\x97\xc7\xb9\x6c", + "\x25\x83\x8e\x50", + "\xb2\x5c\x33\xbb\xa1\xa9\x10\x24\xf4\x2c\xfb\x93\x23\x2a\xd6\x85\xd5" + "\x4b\xe2\xca\x31\x0b\x0f\xf9\xba\x51\x07\xb8", + 1, 32, NULL, 0 }, + { 224, 224, 6, + "\x6d\xde\x88\x28\xf0\x9b\x7a\xa9\x81\x08\x2a\xa1\x16\xfc\xa3\xb7\x34" + "\x17\x21\xc0\x44\x08\x03\xf5\x2c\xc9\x73\x2e", + "\xbe\x81\x60\x2d\xa7", + "\xe5\x10\xfb\xf1\x4b\xd7\x30\x1f\x75\x1c\xc0\xae\x89\xf8\x72\x5a\x76" + "\x54\xeb\xbb\xa6\xbb\x2f\x74\x16\x26\x47\x1d", + 1, 40, NULL, 0 }, + { 224, 224, 7, + "\x3b\xa1\x56\xff\xdc\x55\xd1\x55\xbd\x08\x51\x05\xac\xa6\x4d\x13\x04" + "\x4d\xb6\x0c\x82\xcf\x2c\xd9\xd6\x1d\x09\x8f", + "\x69\xc7\x6c\x89\x37\xa0", + "\xa9\xd3\x87\x40\x24\x50\x38\xd9\xc2\x3c\xbb\x59\xba\x65\x13\xf7\x03" + "\x4d\x80\x47\xa0\x7a\x90\x4a\x2a\x23\xd2\xfc", + 1, 48, NULL, 0 }, + { 224, 224, 8, + "\x9c\x27\x39\xba\xe2\xa8\x63\xfb\x02\x36\x46\x6b\xa3\x40\x8f\x4e\xec" + "\x8d\x43\x20\x6d\x56\xbb\x7a\xa2\xf8\xf7\x5e", + "\xaa\xf4\xc9\x14\x6d\xb9\x48", + "\x21\x10\x39\x3c\x6b\xa0\x1f\x53\xbe\x20\x35\x33\xfb\xc5\x47\x1f\xc8" + "\xf0\x49\x40\xfe\x91\x24\x11\x56\x4b\xa3\x6e", + 1, 56, NULL, 0 }, + { 224, 224, 9, + "\x31\xd9\xca\xe2\xc3\xdf\x06\x40\x18\x20\x9b\x12\x1f\x9e\x88\x39\x76" + "\xea\x75\x79\x42\xec\xda\x9d\x92\xfd\xad\xfd", + "\xb8\x44\x28\x95\x29\x20\x6f\x5a", + "\x1f\x1d\xdb\x86\x80\xb0\xd9\x98\x93\xc4\x98\xa7\x72\xa7\xbe\xa6\x3c" + "\x2e\x08\xc0\x25\x7a\x7f\x31\xe3\xdb\x2b\x88", + 1, 64, NULL, 0 }, + { 224, 224, 10, + "\x89\xa1\xb9\xe9\x00\x44\x44\xc1\xd4\xe9\x67\x57\x0c\x21\xa0\x55\x12" + "\xd3\xf6\x18\xec\x16\x8f\xc3\xe1\x3e\xa5\xa2", + "\x6b\x42\xeb\x6d\x84\xe9\x0c\x70\xc2", + "\x3b\x6f\x3b\x09\xe0\x34\x24\xc8\xad\xc2\x67\xfc\xce\xfa\xf6\x14\xdb" + "\x6d\x74\x97\x77\x54\xfc\xad\x8a\x8d\x1a\x9b", + 1, 72, NULL, 0 }, + { 224, 224, 11, + "\x43\x98\x73\x17\x52\xfd\x7a\xf1\xdb\x86\xeb\xcc\xbe\xe0\xad\x65\xeb" + "\x5f\xaf\x00\xac\xe6\xc9\xaa\x35\x44\x1f\xaa", + "\x1a\xe2\xe7\xd9\x17\xc4\x80\x26\x57\x0d", + "\x5f\x19\x48\x33\x69\x53\x33\x7c\x38\x1d\x44\x9c\x17\xab\x5c\x32\x7c" + "\x86\x12\x1a\x8b\x1e\x0d\xb1\x9f\x62\x4e\x3f", + 1, 80, NULL, 0 }, + { 224, 224, 12, + "\x33\x94\x60\xd6\xbb\x26\xca\x60\xeb\xce\xf1\x0c\x38\x58\x7b\x9e\x57" + "\x5c\x39\x84\x91\x78\x2c\xcf\x9e\x8f\x68\x03", + "\xca\x03\xeb\x4f\x37\x53\x6b\x23\x77\x73\x8e", + "\x51\xc5\x66\x1c\x31\xfc\x7e\xdd\x09\xde\x60\xc9\x19\x57\x03\x68\x24" + "\xa1\x97\x61\xbc\xc5\x4f\x1e\x93\xc4\x3c\x3c", + 1, 88, NULL, 0 }, + { 224, 224, 13, + "\x02\x5f\x83\x80\xd1\x0b\x82\x07\xb3\x62\x3e\x4a\x90\xf7\x9c\x3e\x75" + "\x3b\x1b\xe6\xa3\x5b\x88\xb6\x83\x30\xa4\x0c", + "\xe5\x7d\xae\xf9\xed\xe4\xe9\x15\xc3\xa9\xee\xce", + "\x8a\xfd\xb3\x71\x71\x4e\x9d\x60\x63\xec\x9e\x43\xc8\xcd\x55\xe1\xc0" + "\x32\xb2\xfd\xa5\x7f\x91\xe9\xec\x0f\x66\x01", + 1, 96, NULL, 0 }, + { 224, 224, 14, + "\x0b\xdc\x5f\x51\xf8\xa1\xa3\x5d\x75\x55\x4b\xe7\x0e\xfb\xcd\xf5\x1e" + "\x54\xf3\x0f\xa4\x69\x6f\x72\x74\x31\x94\x1f", + "\xcc\x3d\xd1\xeb\x06\x90\xf7\xaf\x09\xad\x40\x8f\x9c", + "\xc0\x91\x89\x51\xc3\x42\x2b\x48\x50\x26\x35\xb6\xe5\x8c\x5d\xce\xe9" + "\xfe\xa5\x1c\x9d\xce\x5c\x7c\x21\x5c\x9b\x93", + 1, 104, NULL, 0 }, + { 224, 224, 15, + "\x5a\xda\x97\xd9\x0a\x74\xa7\xd4\xa6\x8c\x54\x64\xff\xf2\x5a\x9b\x7f" + "\xa2\xe7\x5d\x6a\xcf\x0a\x59\xf1\x43\xa2\xe9", + "\x3f\xe4\xed\xe1\x58\xaf\x10\x8e\x09\xf5\x43\xe1\x4a\xb7", + "\x18\x0a\x6b\x88\x14\xae\x34\x22\x8a\xe9\xac\x76\xda\x83\x79\x37\x6a" + "\xae\x6f\x1a\xa0\x10\x2e\x8f\x06\xb0\x22\xdc", + 1, 112, NULL, 0 }, + { 224, 224, 16, + "\x00\x7a\xfe\x6b\x7c\x07\x01\xc3\x0c\xb7\x6b\x43\x1a\xfa\x35\x10\xc8" + "\xb3\x1d\x21\xcf\xe0\xbb\xaa\x52\x89\xcd\x08", + "\xc2\xcf\x80\x00\x5c\x59\x1c\x1f\x73\x73\x69\xfc\xc2\x12\xf0", + "\xfb\xfd\xb4\x50\xa4\x2f\x9a\x41\x54\x14\x6f\x73\xc5\x90\xa0\xee\x91" + "\x87\xaf\x85\x05\xd6\x07\x90\xa9\x61\x54\x47", + 1, 120, NULL, 0 }, + { 224, 224, 17, + "\x26\x49\x11\x68\xa3\x2c\xe8\xcb\xc4\xc0\xcd\x64\x10\x7e\x4f\xcc\x43" + "\x2f\x07\xd5\x9c\x99\x28\x62\xe1\xe5\x5b\x1e", + "\x15\xe5\x10\x91\xb4\xf4\x24\xba\x1f\xde\xcb\x5e\x2f\xba\x11\xf6", + "\x3f\xa9\x9e\xe1\x60\x32\x8f\xdd\xc4\x7a\x7c\x50\x43\xe9\xef\x64\x5b" + "\x8b\x07\x46\x2b\x71\xca\xd5\x8a\x02\x45\x17", + 1, 128, NULL, 0 }, + { 224, 224, 18, + "\x69\x78\xb6\xc1\x34\xdd\x69\x49\x83\x2d\x65\xe4\xcb\x9c\x1e\x1d\xc3" + "\x6b\xea\xe4\xa1\x34\x90\x7c\x80\xda\x0f\x44", + "\x66\x41\xd8\x34\xb3\xfb\xfd\xb5\xd1\x78\x00\x78\x01\xf7\xb4\xe7" + "\xb1", + "\x61\x38\x72\x30\x44\x6f\x31\xfd\xe8\x55\x2f\x22\xec\x52\xa7\xfe\xf8" + "\x2e\x16\xd0\xad\x39\x9d\xe9\x39\xd8\x22\x9b", + 1, 136, NULL, 0 }, + { 224, 224, 19, + "\x9f\x9f\xb2\x80\xad\xf1\x2e\x73\x95\x48\xb1\xd6\x76\xcb\x79\x4d\x68" + "\x5b\x91\x04\xe6\x3b\x61\x9b\x05\x5c\xb6\x0f", + "\x91\x51\x3d\xd6\xde\x40\xa1\xc2\x3f\x8d\x1e\xb0\xab\x8f\x5e\xa6\xf6" + "\x83\x55\x06\xec\x75\x08\x94", + "\xe6\xb9\x2f\x9c\x03\x02\x70\x89\x7c\x5d\x27\x16\x2a\x5d\x40\xf6\xd3" + "\x73\xff\x13\x61\x05\xd1\xa9\x0e\x0f\x9a\x60", + 1, 192, NULL, 0 }, + { 224, 224, 20, + "\x3b\x1b\x16\xe6\xdd\x2e\x69\x55\x9d\xbe\xb9\x64\xe1\x0f\xc9\x4c\x06" + "\x84\x71\xb2\x37\x4d\x3a\x2d\x24\xd2\xd4\x66", + "\x8e\xcd\x55\xb5\x6c\x66\x8d\xcb\x8e\x8b\x1e\xfd\x69\x9c\x0e\x4a\x46" + "\x42\x04\xd2\x9a\xf1\x40\xf8\x7d\x3f\x50\x75\x49\x53\x78\xa3", + "\x17\x58\x56\xb8\xf5\x6a\x8c\x6f\xbe\xbc\x36\x54\x17\x71\x54\x50\x46" + "\xbb\x41\x62\x54\xf0\x1f\xf1\x1a\x21\x8d\x2e", + 1, 256, NULL, 0 }, + { 224, 224, 21, + "\xfc\x29\x63\x98\x84\x50\x63\xe6\x61\xbd\xf3\x6f\xf3\x61\x59\x26\xea" + "\xcc\xbf\x06\x94\x7c\xd3\x1e\x66\x77\xf7\x10", + "\x62\xbd\x0a\xd7\x5d\x64\xc5\x54\xcb\x2c\xc1\x09\xc6\xe4\x01\x9f\xc6" + "\x01\xc6\x1c\xab\xdf\x99\xf8\xde\x87\x1e\xdc\x17\xa3\x01\xb4\xc1\xf5" + "\x5a\x15\xed\x66\xf9\x1e\xb4\x66\x6d\xd0\x8b\xc5\x9c", + "\xb7\xcf\x74\x1c\xf9\x6d\x6b\xf5\x7d\x21\x6c\x43\x61\x1c\x20\x86\x9c" + "\xa0\xd0\x08\xa4\x54\x2f\x5c\x85\x06\x05\xbc", + 1, 376, NULL, 0 }, + { 224, 224, 22, + "\x6c\x98\xd1\xfe\xaf\xff\x98\x61\x35\x19\x66\xbc\x6e\xd1\x9e\xd4\x67" + "\xf9\xdc\x76\x7f\xa0\xdf\x6b\x56\x95\x55\x54", + "\xe9\x9d\x51\xa1\xd9\xa2\x5c\x58\x42\x50\x1a\x53\x83\x13\x35\x78\xc8" + "\xde\xbe\x50\x15\x81\xb1\x61\x0f\x75\x75\x51\x9b\xbd\x26\xf0\x1a\xb7" + "\xcb\xe0\x69\xbf\xd5\xdf\x36\x99\xa2\xfe\xa5\xb4\x61\xa3", + "\x0f\xe6\x4f\xdd\x91\x29\x66\xa6\x54\x20\x69\xa2\x2b\xfd\x08\x4b\x48" + "\x4c\x01\x5c\xf4\x34\xd8\x6b\xca\x15\xcd\xb6", + 1, 384, NULL, 0 }, + { 224, 224, 23, + "\x42\xa1\x64\xf9\x4e\x33\xd5\x74\x11\x8e\x0f\x8c\x93\x8b\xbc\x28\x74" + "\xba\xb2\x19\xee\x7a\x17\x9f\x21\xe1\x3b\x02", + "\xe8\x95\x63\x96\x31\xf8\xb5\xd4\x8e\x3c\xe0\x0e\xb3\x10\xbf\x12\x99" + "\x76\xff\xce\xd9\x6a\x6f\x30\xa0\x9d\x6a\xc1\xc2\x91\xf7\x3e\x93\x69" + "\x05\x26\xd8\x6c\xc4\xd1\xa8\xe2\x1c\x11\xf5\xa8\x97\x93\x08", + "\x1e\xa9\x82\x22\x6e\x8d\x4c\xb7\xb0\x79\x22\x15\x8e\x53\x5a\xf2\x23" + "\x3b\x4c\x4d\x39\xd2\x6b\x06\x2d\x6d\x2a\xae", + 1, 392, NULL, 0 }, + { 224, 224, 24, + "\xc1\xb5\xb9\x12\x10\x66\x7e\x72\xaa\x51\x03\x46\xe1\x81\x13\x58\x81" + "\x5a\x33\x30\xc5\xed\x27\xa6\x95\xc3\x94\x51", + "\xbf\x10\x86\xc3\xea\x8b\x88\x40\x41\x8c\x69\x0c\x92\x15\x2c\x73\xa6" + "\x73\x0b\xd1\xa0\x21\x0c\x8b\x1d\x25\xc4\x3a\x21\x93\xe7\x39\x68\x4f" + "\x04\xa2\x5a\x52\xcc\x30\x55\x99\xf2\x2b\xa6\xf7\x0c\x8e\xd0\x0d\x10" + "\xb9\x14\xa9\x52\x2a\x25\xe0\x6c\x47\x1e\xbc\xa2\xff\x1b\xb4\xfa\x67" + "\x99\xb8\x51\x22\x02\x09\x78\xdf\xa6\x6e\xf1\x2e\xd2\x6a\xd3\x83\x31" + "\xb2\x6e\xaf\x59\x1a\xfc\xea\xc9\x6d\x8c\x77\x1e\xae\x50\xfb\x7f\x46" + "\x24\x23\x37\xdd\x00\x29\xf4\x81\x3b\x53", + "\x4f\x35\x5e\xdb\xe6\xa3\xc9\x3f\xa7\xad\xd3\x84\xbe\x89\x9b\xb4\xfb" + "\x55\x38\x5a\x78\x81\x2a\x26\xcb\x64\xe4\x4f", + 1, 896, NULL, 0 }, + { 224, 224, 25, + "\x4f\x09\xd1\x4d\x40\xe4\x75\xb6\x82\x88\xc0\x80\x66\x8e\xbb\x1b\xc8" + "\xc6\xbe\x31\x91\xf6\x66\x4d\x91\xa2\x3f\xcd", + "\xae\x8b\x6e\xcc\x21\x9b\x36\x8d\x22\xfb\x59\x6e\x42\x65\x2d\x0b\xff" + "\xee\x0b\x20\xd6\x9c\xfd\x08\x9c\xe3\xdc\x93\x03\xba\x2f\x05\x4c\xca" + "\xf5\xf5\x14\x7c\x79\x68\xa0\x28\xb1\x40\xf5\xe3\xc9\x27\x4e\xae\x2a" + "\xfc\x61\xc3\xbb\x62\x98\xdc\x59\x8d\xf7\x7d\xec\x1c\xd2\xdd\x84\x21" + "\x26\x93\xb0\x82\xb8\x13\x2a\xd0\xf0\xb1\x9f\x66\xdb\x69\xfa\x7f\x6b" + "\xf3\x52\xb4\xfe\xac\x72\x4c\xe0\x48\x44\x0d\x2a\x42\xb4\x4d\x53\xbb" + "\x62\xfe\x2a\xb2\x5f\x7f\x54\xbe\xdf\x9c\xe7\xdd\xaf\xd8\xe0\x93\x30" + "\xda\xcc\x6d\x52\xee\x9b\x65\xf5", + "\x29\xba\x26\x81\x03\x01\x9e\x15\x8a\x35\x61\x4c\x80\x78\x0f\xda\x3f" + "\x5e\xc3\xfc\x32\xc8\x0a\xaa\x27\xb4\x02\x5d", + 1, 1016, NULL, 0 }, + { 224, 224, 26, + "\x61\x3f\x41\x4c\xd9\x41\x30\xbb\x8a\x62\x43\xe1\x2e\xcc\xd9\x08\x36" + "\x80\x84\x28\xb4\xa7\x17\x78\x67\x93\x4d\xa0", + "\xf6\x96\xb9\x06\x3b\x64\x81\x6a\x45\x06\x4f\x48\xca\x05\xff\xe4\xd5" + "\xcc\x3d\x0b\x3b\xeb\x0d\xd4\x05\x7b\x6a\xda\x99\x49\x69\xbf\x03\x9b" + "\xfb\xb7\x2c\xe1\x97\x10\x1c\xc4\xe4\xb3\x95\x9b\x37\x02\xf0\x45\xaf" + "\xb7\xfb\x31\x13\xc9\x97\x60\x6d\xca\xf2\xaa\xab\x31\xe0\x2a\xc6\xee" + "\x59\x7d\xfc\x0f\x91\x43\xd0\xef\xfe\xdc\x9a\xe7\xea\x10\xe7\xdd\xb1" + "\xdb\x86\x0a\x91\xaf\xec\x62\xc4\x8e\xd9\xc0\xa6\xc1\x0b\x4d\xa1\xde" + "\x74\x8c\xaf\x7f\x7a\x5e\x01\x79\x9a\xc5\x70\x90\xda\xf4\xe3\x35\x2f" + "\xe8\x59\xc5\x13\x1c\x20\x5d\x26\x2d", + "\x81\x29\xe2\x09\x30\x70\x16\x8a\x20\x89\x97\x93\xa0\x44\x47\xa7\xef" + "\x01\xae\x72\x34\x19\x25\x6a\x8c\xb4\x2f\x6d", + 1, 1024, NULL, 0 }, + { 224, 224, 27, + "\x5b\x88\x27\x53\x07\xaa\xf6\x91\xa0\xcf\x0c\x51\xf5\x05\x53\xdd\xa9" + "\x72\xd1\x4f\x8a\xff\xf9\x8e\x62\xc2\xd9\x72", + "\x57\xe4\xef\xbd\xe1\xce\x9f\xee\x2e\x29\xdb\x19\xdf\xc6\xba\x3b\xcb" + "\x17\xf3\x37\x65\xaf\x7f\x20\x13\x3b\xbd\x19\x10\xd5\x42\x14\x5c\x7d" + "\xef\x18\x7a\x30\x45\x17\xb8\xd8\x95\x44\x54\xa9\x0a\x71\x7f\x67\xf9" + "\xc8\xcc\x58\x79\x65\xfd\x9b\x43\xf4\x1e\xcc\x50\xb3\x45\x8d\x8c\xe9" + "\xf6\x6b\x47\x5f\x1e\xae\xf4\xa2\x9b\xa8\x9a\x3d\x58\xe5\x01\x1c\x92" + "\xac\xd1\x53\x6f\xcd\x18\xab\xec\x29\x41\x1b\x38\x9b\x64\xf7\xf3\x44" + "\x77\x7e\xd6\xde\xae\x32\x12\x7a\xba\xa6\x9a\x50\xba\x22\xa1\x1d\x6e" + "\x59\x35\x4f\x2f\xf0\xe3\xc3\xe3\x29\x3c\xdc\x33\x54\x11\xcf\x55\xb1" + "\x80\xba\xb5\x9d\xa3\x69\x03\xa6\xfb\xa9\x1d\xf3\x4d\x2a\xad\xf7\x01" + "\x7f\xf4\x9a\x4f\xbd\x73\xc9\xc7\x44\x69\xf2\x25\xda\xfc\x0a\x0c\x70" + "\x48\xc2\xb8\x24\xcc\x0c\xba\x8c\xad\x8a\xce\xd1\x1b\x8c\xda\xc3\x24" + "\x3c\xdb\x5b\x65\x4f\x7a\x15\xce\x20\x14\xe9\x2e\xe2\x87\xd0\x69\x04" + "\xd7\x78\x51\x2a\x1b\x1f\x5e\xc0\xc9\xb0\x90\xb9\xab\x43\x9c\x44\x26" + "\x6b\x6b\xe3\xd6\xa9\x89\x47\xd2\x6d\x07\x9e\x4f\x7e\x84\x9f\x3c\x6d" + "\x93\xde\x98\x62\x4e\x6c\x5f\x53\xec\x02\xdb\xd3\x68\xbc\x24\xa3" + "\x00", + "\x65\x7d\xd0\x4b\x97\x02\x19\xed\xd6\x3a\xbf\x9d\x4a\xa1\x08\x47\x4a" + "\xa3\x16\xb6\xbb\x66\xbc\xa7\x6e\xd8\x06\xc6", + 1, 2040, NULL, 0 }, + { 224, 224, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6f\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6c\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0f\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\xee\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x98\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x20\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\xe2\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\x2e\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe4\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xde\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe7\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdd\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x13\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\x4d\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6c\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x50\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\xed\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\xd1\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x58\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xdc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb4\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xae\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x28\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x73\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x2b\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x70\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\xa9\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\xf2\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x13", + 0, 0, NULL, 0 }, + { 224, 224, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8b", + 0, 128, NULL, 0 }, + { 224, 224, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x10", + 0, 0, NULL, 0 }, + { 224, 224, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x88", + 0, 128, NULL, 0 }, + { 224, 224, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x52", + 0, 0, NULL, 0 }, + { 224, 224, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\xca", + 0, 128, NULL, 0 }, + { 224, 224, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x92", + 0, 0, NULL, 0 }, + { 224, 224, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x0a", + 0, 128, NULL, 0 }, + { 224, 224, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6f\x99\xe8\x62\xe5\x32\xe8\x93\x6c\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x21\x6f\xae\xdf\x30\x53\xcd\x50\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\xe2\xe5\x32\xe8\x13\x6d\x78\xb5\xf0\x29\x09\xb1\x30\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\x2e\xdf\x30\x53\x4d\x51\xfc\xaf\x41\x72\x22\xc8\xf1\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6e\x99\xe8\x62\xe5\x32\xe8\x13\x6d\x78\xb5\xf0\x29\x09\xb1\xb0\xab" + "\x09\x80\x6b\x2a\xf0\x2f\x7c\xb9\xd3\x9d\x12", + 0, 0, NULL, 0 }, + { 224, 224, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\x4d\x51\xfc\xaf\x41\x72\x22\xc8\x71\x44" + "\xab\xd5\xf2\xf7\xfa\x00\xab\x46\x67\xd8\x8a", + 0, 128, NULL, 0 }, + { 224, 224, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x91\x66\x17\x9d\x1a\xcd\x17\x6c\x92\x87\x4a\x0f\xd6\xf6\x4e\xcf\x54" + "\xf6\x7f\x94\xd5\x0f\xd0\x83\x46\x2c\x62\xed", + 0, 0, NULL, 0 }, + { 224, 224, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf2\xde\x90\x51\x20\xcf\xac\x32\xae\x03\x50\xbe\x8d\xdd\x37\x0e\xbb" + "\x54\x2a\x0d\x08\x05\xff\x54\xb9\x98\x27\x75", + 0, 128, NULL, 0 }, + { 224, 224, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 224, 224, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 224, 224, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 224, 224, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 224, 224, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\xee\x19\x68\xe2\x65\xb2\x68\x13\xed\xf8\x35\x70\xa9\x89\x31\xb0\x2b" + "\x89\x00\xeb\xaa\x70\xaf\xfc\x39\x53\x1d\x92", + 0, 0, NULL, 0 }, + { 224, 224, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\xa1\xef\x2e\x5f\xb0\xd3\x4d\xd1\x7c\x2f\xc1\xf2\xa2\x48\x71\xc4" + "\x2b\x55\x72\x77\x7a\x80\x2b\xc6\xe7\x58\x0a", + 0, 128, NULL, 0 }, + { 224, 224, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", + "\x6f\x98\xe9\x63\xe4\x33\xe9\x92\x6c\x79\xb4\xf1\x28\x08\xb0\x31\xaa" + "\x08\x81\x6a\x2b\xf1\x2e\x7d\xb8\xd2\x9c\x13", + 0, 0, NULL, 0 }, + { 224, 224, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x20\x6e\xaf\xde\x31\x52\xcc\x50\xfd\xae\x40\x73\x23\xc9\xf0\x45" + "\xaa\xd4\xf3\xf6\xfb\x01\xaa\x47\x66\xd9\x8b", + 0, 128, NULL, 0 }, + { 224, 112, 82, + "\x26\xf3\x14\x17\x0b\x05\x4d\xae\xf5\x34\x98\x04\xda\x18\xf9\x69\xc9" + "\x41\x74\xba\xca\x2b\xee\xb0\x09\xd4\x7a\x23", + "", "\x81\x6d\x7a\xf2\x47\x5e\x94\x71\x3f\x2d\xc3\xaa\x30\x69", 1, 0, + NULL, 0 }, + { 224, 112, 83, + "\x17\x42\x9a\x62\x2d\xc1\x8d\x38\x71\x5b\x31\xf8\xf2\xb9\x63\x10\x8e" + "\x95\x2a\x67\x08\xf3\xe5\x2d\x5b\x25\x84\x8a", + "\xda", "\x26\x63\x07\x77\xd8\x5f\x77\x71\x87\x63\x0b\xb9\x46\x74", 1, + 8, NULL, 0 }, + { 224, 112, 84, + "\x0a\xcf\xe1\x2d\x89\xac\xd7\xd9\xca\x49\xba\xe6\x31\x8f\x35\xb2\xfb" + "\xbf\xc8\x4e\x5d\x2c\x9d\x49\x54\xbe\xde\xd7", + "\x03\xa8", + "\xaa\x4c\x4b\xb6\x3c\xad\x66\xac\x67\x51\x50\xf7\x18\xb2", 1, 16, + NULL, 0 }, + { 224, 112, 85, + "\x5a\x06\x80\xf1\x12\x35\x4b\xd4\x67\x86\x5b\x19\xae\x95\x6b\x27\x19" + "\xe2\x1e\xce\xe1\xa9\x13\xbd\xca\x29\x43\x39", + "\xa0\xfb\x73", + "\x36\xc7\xcd\x3f\x29\x0d\x1d\x7d\x33\x2b\x95\x1a\xa4\x71", 1, 24, + NULL, 0 }, + { 224, 112, 86, + "\x46\xfa\x59\xaa\x52\x4f\xe3\x0a\x0f\x4e\x39\x56\x1b\x56\x66\x85\x44" + "\x40\xdb\xd9\x70\xbb\x59\x92\x5c\xe0\xae\x1a", + "\xc8\xb2\xf5\x57", + "\xc1\xa8\xa7\xd4\x3d\xf3\x4d\x91\x7f\x0c\xb5\x12\xc5\x7d", 1, 32, + NULL, 0 }, + { 224, 112, 87, + "\x29\xef\xc5\xab\x5d\x30\xe5\x35\x35\x76\x03\xf2\x71\x1b\x6e\x0a\xa6" + "\xcf\x46\x13\x54\x6c\x23\x14\x44\x36\xd2\x13", + "\xc8\xd9\xf5\xb3\x73", + "\xa8\xcc\x7b\xeb\xef\x4c\xfc\xd5\xac\x2f\x40\x1a\x37\x2d", 1, 40, + NULL, 0 }, + { 224, 112, 88, + "\xfe\x60\xe0\x32\x20\x35\x53\x8f\x2b\x1d\xe9\xde\x38\x0c\xde\x35\xf2" + "\x91\xde\xeb\x6e\x02\x7b\x5d\x82\x9e\xcd\x1e", + "\x18\x5e\x4c\xad\xa4\xf4", + "\x32\xfa\xa1\x54\x39\x6b\x0b\x62\x43\x6e\x6b\xf9\x37\xee", 1, 48, + NULL, 0 }, + { 224, 112, 89, + "\x1b\xf7\xfc\xdf\x37\x42\xfa\x77\x99\x15\x28\xcc\x1c\x67\x8b\x98\xbe" + "\x98\x76\xa8\xc8\xc5\xb8\x09\xbe\xab\x7d\x9c", + "\x9c\x0f\x34\xa5\x65\x42\x79", + "\x08\x61\x70\xc4\x6d\x2b\x0a\x76\xc6\x15\x27\xc2\xd0\x52", 1, 56, + NULL, 0 }, + { 224, 112, 90, + "\x32\x53\x3c\x16\xf7\x92\xed\x0a\xcf\x8e\x9e\x60\xf5\x4a\xa1\x73\x93" + "\x7c\x71\x94\xb8\x82\xec\xc3\xe6\x71\x00\x9f", + "\xf9\x68\xdc\x7a\x19\xaf\xe3\x39", + "\x3f\xb4\xeb\x44\x50\xac\x4b\x26\xa7\x14\xbc\xfb\x22\x4c", 1, 64, + NULL, 0 }, + { 224, 112, 91, + "\x3c\xf2\x8a\x47\x6c\xe7\xea\xec\xfc\x3f\xbf\x1b\x08\x59\xa0\x42\xa5" + "\x68\x74\x0a\x58\x4c\x77\xcb\x8f\x96\x03\xac", + "\xdb\xca\x9e\x4b\xdd\x84\xb3\x89\x34", + "\x2c\xf1\x4e\xb8\xf4\xc7\x53\x7e\x98\x31\x98\x3b\xb5\xaf", 1, 72, + NULL, 0 }, + { 224, 112, 92, + "\xa2\xa8\x09\x0a\xef\x69\x27\x7f\x92\x83\x0e\xc7\x40\x4c\x03\x2f\x8f" + "\xde\xbf\xbc\xea\xbb\x9e\x59\x09\x68\xa7\x7f", + "\x6b\x79\x0a\x94\x6a\x83\x36\x4c\x79\xd7", + "\xd4\x67\x20\x9f\x63\xa9\xbd\x3d\x2c\x53\x98\xc3\x05\xda", 1, 80, + NULL, 0 }, + { 224, 112, 93, + "\x6f\x99\x99\x29\xe9\x16\x72\xba\xc3\x5e\xa7\x0f\x8f\xf8\xb9\xae\xef" + "\xa5\x48\x94\x93\xc9\x9b\x0d\x27\x79\x72\x07", + "\xb7\xda\xbb\x23\x7a\xea\xe2\xbe\x8b\x5e\x19", + "\x09\xb2\xbb\x6e\xae\xda\x5f\x02\x29\xb8\xc3\x5a\x2f\x54", 1, 88, + NULL, 0 }, + { 224, 112, 94, + "\x45\x25\xb9\x6c\x26\x3e\x4d\x2d\xab\x28\x90\xaa\x55\xf3\xcc\x50\x3d" + "\xc1\x20\x6d\x9f\x19\x15\xa6\xfb\xa5\xae\x61", + "\xef\x85\x8f\x49\x6f\xcb\x7c\x3f\xab\xbf\xb5\x2e", + "\x6f\x5c\xa7\xef\xcb\x9a\x70\xd0\xab\xf8\x42\x5f\x42\xca", 1, 96, + NULL, 0 }, + { 224, 112, 95, + "\xf8\x94\x01\xac\xb0\xa6\x0d\x07\xfd\x73\x3e\xd5\x63\xf2\xee\x24\x1f" + "\x4e\xcf\xea\x81\x14\x58\x7a\x44\xdf\xdb\x0c", + "\x7d\x3c\x09\x18\x08\x59\x84\xdf\x95\x09\x7a\xfa\x81", + "\xfc\x22\x7f\x29\xb5\x1f\x9c\x85\x53\x43\xdc\xd0\xea\x11", 1, 104, + NULL, 0 }, + { 224, 112, 96, + "\x58\xbc\xe8\xc0\xd1\x7f\xc7\x13\x1d\x2f\xa2\x26\x24\x09\xbb\x14\x66" + "\x3a\x6e\x68\x01\x9f\x88\x29\x99\x87\x89\x3e", + "\x1c\xa5\x0c\xd6\xc3\xf1\x22\x5e\xb6\xc4\xec\x4d\x6a\x90", + "\x53\xe1\x03\xbb\xde\xd7\xb8\x25\xaf\xfa\x24\x0f\x85\x78", 1, 112, + NULL, 0 }, + { 224, 112, 97, + "\x65\x8e\x51\x0f\xba\x4e\x22\x08\xaf\xac\x98\x33\x3f\x9e\x24\x2b\xc1" + "\x18\xf6\xe7\x9e\xf0\x66\x1d\x61\x9d\xd3\x2b", + "\x32\xc3\x85\xb7\x5a\xe8\x45\x58\xca\x30\x28\x81\xc5\x16\x39", + "\x48\x5f\x35\x1e\x2a\x9a\x82\x91\x0c\x3c\x94\x9e\x32\xb8", 1, 120, + NULL, 0 }, + { 224, 112, 98, + "\x6a\x41\xcc\x3c\xa7\x14\x2a\xe1\x4e\x6d\x97\x9a\x3f\x89\x0a\x33\x15" + "\x97\xe5\x92\xdd\x74\x52\x0c\xe4\xea\x66\x0f", + "\x78\xe3\xa7\x70\xa8\xaa\xaf\x03\x9f\xd4\xc9\xb6\xa1\x78\x04\x11", + "\x33\x1a\x58\xed\x96\xfc\x8b\x9e\x68\x4a\xb0\x5f\x63\x6c", 1, 128, + NULL, 0 }, + { 224, 112, 99, + "\xb8\x97\x2b\x93\xb6\x83\x02\xcb\xaa\x08\xd3\x29\x04\xea\xe6\x37\x5a" + "\x66\xf3\x50\x8e\xce\x3c\x9b\x22\x38\x2c\x7e", + "\x36\x87\xe6\x28\x7d\x73\xc9\xe3\xf6\x79\xa5\x0e\x76\x71\x24\x71" + "\x27", + "\x27\xd8\x11\x39\x55\x02\x6d\x4d\x31\x80\x70\xfb\xfd\x8f", 1, 136, + NULL, 0 }, + { 224, 112, 100, + "\xbc\x57\x09\x32\xab\xfa\x11\x05\x0a\xd4\xfc\x80\xa6\xd5\xaf\xe3\x27" + "\x1d\x86\xaa\x29\xdc\x62\x73\x8b\x20\x7d\x14", + "\xd5\x32\x02\xac\xd2\xec\x74\xd7\x46\x53\x1b\xd9\xad\x30\x16\xd0\x98" + "\x0e\x01\x66\xfb\x42\x7a\x08", + "\x02\x0e\x3e\x0c\x29\x40\xce\x15\xee\xb6\x73\x92\x57\x0f", 1, 192, + NULL, 0 }, + { 224, 112, 101, + "\xc9\x2a\x06\x65\xc1\x2e\x87\x02\x6e\x1b\x34\x4f\x97\x1f\xdb\x0e\x47" + "\x4d\x45\x0c\xba\x83\x4a\xae\x40\xe2\xd2\x1e", + "\x4a\x3a\x85\xac\x09\xf5\x19\x0a\xb9\x4f\x73\xfd\x91\xd9\x8f\x05\x60" + "\x15\x26\x3c\x89\xed\x5d\xa2\x23\xfc\x46\x75\xca\xb2\x5c\xdd", + "\x92\x28\x53\xf1\x59\xc4\x2b\x9e\x27\x4f\xce\xf7\xbd\xf3", 1, 256, + NULL, 0 }, + { 224, 112, 102, + "\x6f\xbe\xf6\x7c\xfb\xac\xc9\x8c\x63\x25\x2b\x1c\xa0\x09\xa6\x0e\x8e" + "\x34\x79\x76\x9a\x2d\x44\x9f\xb4\x63\x90\x64", + "\x00\x6e\x17\x9e\xac\xfa\x9e\x1e\x62\x8b\xb7\x82\x3e\xe9\x60\x9a\xe7" + "\x96\x8b\x6d\xf9\x0e\x17\x6f\x77\x2a\x79\x08\x8d\x37\xe9\xb1\x5c\xab" + "\x31\x29\x22\xaa\xf8\xfc\x65\x83\xa3\x41\x00\x2b\xda", + "\x0a\x27\xa1\x2a\xfb\xb9\xc3\x13\x62\x02\xe0\x2a\xe3\xb2", 1, 376, + NULL, 0 }, + { 224, 112, 103, + "\x70\x0b\x09\x90\x81\x74\xf1\x07\x2e\x31\xae\x8c\xcb\xda\x1c\x44\x60" + "\xfc\xf2\x1f\xdf\x14\x6a\x11\x48\x2b\x21\x0d", + "\xf7\x72\x56\x4e\xcb\x10\x9e\x80\xee\xfb\x1d\x5a\x7f\x1c\x95\xe2\x03" + "\xba\x4c\x98\x02\x33\xdd\x8d\x13\xde\x30\x46\x07\x9a\x6b\x2c\xa2\x6d" + "\xc3\x52\x1e\x5e\x0c\x80\x7e\xae\x7a\x79\x87\x7c\x73\xe9", + "\x04\xc7\x18\xa4\xcd\x8b\x58\x3d\x5f\xfb\x81\x70\x27\x6c", 1, 384, + NULL, 0 }, + { 224, 112, 104, + "\xe1\x8a\x20\x24\x6e\xbe\x1b\x57\x96\xdb\xfe\x35\x11\x0e\xfc\x76\x37" + "\xd7\x4a\x35\x5f\x0a\x67\x58\xd4\xa0\x0b\x7d", + "\x77\x72\x0d\xde\x53\x0e\x6e\xea\xa0\xe9\xaf\x33\x11\xf7\xe9\x91\x89" + "\xd6\xc4\xf7\xd7\x1d\x0a\x42\x07\xd6\x2c\x76\x6b\xee\x32\x02\x0c\x92" + "\xf5\xd5\xd2\x8d\x5d\xe4\xd0\xd9\xc9\x4b\x57\xec\x05\xf0\xc3", + "\x25\xeb\xc8\x61\x1f\x4b\x63\x6d\x89\x2f\x11\xdf\x2b\x29", 1, 392, + NULL, 0 }, + { 224, 112, 105, + "\x3c\x45\x85\xa7\x75\xbe\xc7\x6c\x7d\x8b\x27\xb8\x7e\x70\xa5\x86\x3a" + "\x85\xe6\x11\x1f\x31\x61\xb3\x81\x5f\x59\xb4", + "\x62\x8c\x0f\xf8\xc4\x32\xd7\x4f\x4c\xfb\x77\xba\x46\xb7\xce\xf6\x7a" + "\x48\xac\x05\x3c\xf0\xc1\x8b\xe4\x16\x48\x73\x6a\xbc\xc8\xc6\xfb\xe4" + "\x98\x15\x29\xba\xbd\x4b\x27\x86\x6e\x34\xce\xd1\x6d\x8b\x0b\xec\x45" + "\x6e\x14\x65\x3a\x14\x22\xf5\xa6\x25\x56\xd2\x0b\x0f\xe4\xe0\x37\x49" + "\xd5\xf6\xe9\x86\x37\x50\x62\xdb\xdd\x82\xf6\xe9\xe1\xd4\xad\x54\x7c" + "\x31\x53\x0c\x2a\x31\x38\x3c\x25\xff\x57\xe8\x79\xea\xe9\x9d\x9b\x3a" + "\x0d\xa1\xf3\xc1\xda\xcb\x97\x50\x67\xac", + "\xde\xb9\x4b\x2d\x43\xe9\x89\x26\xaf\x51\xfc\x0c\x88\xdc", 1, 896, + NULL, 0 }, + { 224, 112, 106, + "\xac\xaf\x94\xcb\x1a\x8f\xf4\x67\x7f\xc5\x86\xd2\xbd\xf9\x81\xac\x3a" + "\x65\x6b\x20\x82\x15\xe0\xa7\x64\x7b\x42\x0f", + "\x31\x4c\x2c\x25\x46\x5d\xe3\x42\x72\x79\xdb\xc8\x94\x36\x50\x5f\xee" + "\x6d\x37\xd5\x6f\xbd\xa0\xe5\xe2\xa4\x94\x49\xd9\xdb\xf0\x03\x02\x7f" + "\x2e\x4e\xf5\xc5\x2f\x7a\xf9\x3f\xd8\x01\x55\xa6\x6a\x1c\xd6\xb9\x88" + "\x5b\x56\xd8\x28\x05\x8a\x0d\xe7\xd2\x47\xe1\x95\x80\xb2\xe8\xdc\xbd" + "\xef\x2a\xe4\x68\x40\x56\x5f\xd8\xb2\x76\x56\x9c\x19\xd7\xe1\x85\x11" + "\x6e\xa1\x1a\xd6\x7d\x5f\xc2\x7f\x4a\x68\x16\xba\x45\xbe\x5d\x14\xf3" + "\xba\x43\x15\xc7\x4d\x1e\xdb\x20\xf2\x17\xb1\x16\xbe\x85\x2b\x62\xa7" + "\xf4\xe3\x2b\x3e\x70\x8f\xf9\xf7", + "\x5b\x0d\x7a\xec\x7f\xbd\x19\x6e\xe6\x9e\xd3\x73\xe1\x31", 1, 1016, + NULL, 0 }, + { 224, 112, 107, + "\xe4\x90\x34\x8a\xd7\x8f\xd2\xcd\x5b\x51\xf2\x79\x5b\x79\xe5\x80\x5c" + "\xe1\xd9\xba\xf1\x15\x1d\xbd\xf9\x95\xe1\xb0", + "\xf6\xff\x18\x45\x84\x2b\x9e\x46\xf7\x9a\xdb\x10\x79\xaf\xf4\x73\x97" + "\x39\x1d\xc2\x69\xbc\x0c\x89\x9b\xa4\x08\x7b\x58\xa6\x76\xf5\x40\x8c" + "\x3f\x76\x37\xff\xc4\x77\x2a\xf3\xe4\x1b\x5c\xea\x51\x05\x8b\xc5\x28" + "\xea\x09\xbb\x4b\xd7\x97\x59\x4c\x79\x8b\x0f\x0f\xf8\x81\x69\x5e\x98" + "\xc0\x8b\xbb\x04\x0c\x12\xc5\xcb\xdb\x22\x8d\x61\xcc\x99\xe3\x32\xe9" + "\x63\x12\x8d\x06\xe9\x7e\xd2\xee\xfd\xed\x2e\x1b\x5a\x03\x5f\x3b\xea" + "\x68\x27\x3e\xfa\xc0\x3a\x89\x4d\xcf\x2f\xcc\x79\xa5\x69\x62\x18\x59" + "\x54\x04\xb2\x75\x8d\xeb\x9a\x80\xee", + "\x59\x07\x27\xf3\x44\xd8\xa5\x40\xe5\xc5\xe0\xf4\xda\xe9", 1, 1024, + NULL, 0 }, + { 224, 112, 108, + "\xc8\xe0\x99\xdb\xb6\x0a\x8f\x19\xd8\xb8\x68\x56\xb2\x1c\x55\xf3\x43" + "\x7a\xe2\x7f\x77\xdf\xf9\x80\x8f\x12\xa1\xb5", + "\xed\xbc\x10\x9b\xf2\x8c\x8a\xb3\x2b\x12\x38\xef\xf1\xcd\x14\x30\x8c" + "\xdd\x84\x5f\xa9\x19\xbf\xd8\xa0\x0c\x99\x1c\xf9\xa8\xd6\xb0\x5d\xd8" + "\xcc\x7d\x23\x93\x78\x29\x49\xc8\x99\xde\x79\xe7\x71\xef\x7d\x85\x67" + "\xf3\x22\x87\x62\x39\x63\x04\x8e\x6c\x80\xd9\x1f\x07\x78\xdd\x63\x31" + "\x11\x06\xe9\xd0\x91\x3c\x08\xb7\xa0\xb7\x25\x3f\xa3\xce\x30\x7a\xc4" + "\x0e\xc5\x5a\x4c\x44\x5f\x54\x55\xa5\x70\xfe\xc0\x90\xe2\x51\xe8\x64" + "\x6b\xad\xa1\xa4\x86\xd4\x1c\x37\x94\xbc\xe5\x63\x97\x32\xf2\xc6\xcf" + "\xd5\x80\x81\xc4\x79\xa6\x8c\x51\x5f\x5d\x47\xbc\x5b\x2f\x16\x22\xa0" + "\x8d\x38\xa5\x96\xa8\x17\xf3\xd4\xef\xef\x80\x03\xae\x43\x0e\x6a\xe9" + "\x3b\x0a\x3a\xe8\xfa\x95\xa2\xac\xe3\xd2\x4d\x90\xa9\xef\x86\x1d\xc0" + "\x4c\x13\xe3\x8f\x6e\x52\x4b\x3a\xbd\xf9\xcc\xe4\xfa\x49\x07\x07\xc8" + "\x0c\x16\xe2\x54\xb7\xa7\x1a\xf0\x0a\x12\xdb\xf4\x73\xb5\x0b\x9f\xe4" + "\x09\x7e\xc0\x0a\xb2\x7e\x66\xb6\xf3\x02\x2b\x0f\x10\x1e\xe1\xa9\xf7" + "\xfa\x86\x52\xe9\xf0\x95\xca\x24\x0a\x44\x60\x67\x44\x68\x67\xf7\x8e" + "\x83\x52\xc4\x11\x07\x94\xc2\xe3\x38\x3d\xfe\xdf\xb3\x5e\x74\xa3" + "\x3e", + "\x33\xe7\xdc\xb0\xfd\xe3\xb1\xc5\xb9\x25\x06\xe6\x35\xeb", 1, 2040, + NULL, 0 }, + { 224, 112, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6f\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6c\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0f\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\xee\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x98\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x20\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\xe2\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\x2e\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe4\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xde\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe7\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdd\x30\x53\xcd\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x13\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\x4d\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6c\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x50\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\xed\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\xd1\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x58\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xdc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb4\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xae\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x28\x09", 0, 0, + NULL, 0 }, + { 224, 112, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x73\x22", 0, 128, + NULL, 0 }, + { 224, 112, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x2b\x09", 0, 0, + NULL, 0 }, + { 224, 112, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x70\x22", 0, 128, + NULL, 0 }, + { 224, 112, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\xa9\x09", 0, 0, + NULL, 0 }, + { 224, 112, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\xf2\x22", 0, 128, + NULL, 0 }, + { 224, 112, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x08", 0, 0, + NULL, 0 }, + { 224, 112, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x23", 0, 128, + NULL, 0 }, + { 224, 112, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x0b", 0, 0, + NULL, 0 }, + { 224, 112, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x20", 0, 128, + NULL, 0 }, + { 224, 112, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x49", 0, 0, + NULL, 0 }, + { 224, 112, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\x62", 0, 128, + NULL, 0 }, + { 224, 112, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\x62\xe5\x32\xe8\x93\x6d\x78\xb5\xf0\x29\x89", 0, 0, + NULL, 0 }, + { 224, 112, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\xae\xdf\x30\x53\xcd\x51\xfc\xaf\x41\x72\xa2", 0, 128, + NULL, 0 }, + { 224, 112, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6f\x99\xe8\x62\xe5\x32\xe8\x93\x6c\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x21\x6f\xae\xdf\x30\x53\xcd\x50\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6e\x99\xe8\xe2\xe5\x32\xe8\x13\x6d\x78\xb5\xf0\x29\x09", 0, 0, + NULL, 0 }, + { 224, 112, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0d\x21\x6f\x2e\xdf\x30\x53\x4d\x51\xfc\xaf\x41\x72\x22", 0, 128, + NULL, 0 }, + { 224, 112, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x91\x66\x17\x9d\x1a\xcd\x17\x6c\x92\x87\x4a\x0f\xd6\xf6", 0, 0, + NULL, 0 }, + { 224, 112, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xf2\xde\x90\x51\x20\xcf\xac\x32\xae\x03\x50\xbe\x8d\xdd", 0, 128, + NULL, 0 }, + { 224, 112, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, 0, + NULL, 0 }, + { 224, 112, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, 128, + NULL, 0 }, + { 224, 112, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, 0, + NULL, 0 }, + { 224, 112, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, 128, + NULL, 0 }, + { 224, 112, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\xee\x19\x68\xe2\x65\xb2\x68\x13\xed\xf8\x35\x70\xa9\x89", 0, 0, + NULL, 0 }, + { 224, 112, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x8d\xa1\xef\x2e\x5f\xb0\xd3\x4d\xd1\x7c\x2f\xc1\xf2\xa2", 0, 128, + NULL, 0 }, + { 224, 112, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "", "\x6f\x98\xe9\x63\xe4\x33\xe9\x92\x6c\x79\xb4\xf1\x28\x08", 0, 0, + NULL, 0 }, + { 224, 112, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x0c\x20\x6e\xaf\xde\x31\x52\xcc\x50\xfd\xae\x40\x73\x23", 0, 128, + NULL, 0 }, + { 112, 224, 161, + "\x77\xb0\xde\x54\xe8\x93\x64\x2c\xae\xac\x34\xbf\xd1\xab", "", + "\x20\x14\xa9\xf2\x72\x37\x8f\xa1\xc9\xf6\x74\x4d\x4d\xb4\x86\x1b\x52" + "\xe6\x1a\x19\xeb\x28\x32\x0e\xbe\xe2\xd1\x74", + 1, 0, NULL, 0 }, + { 112, 224, 162, + "\x73\x46\xc7\xe4\xb1\x18\xb2\x4e\x51\xf4\x51\x2f\x90\x6a", + "\x50\x6d\x4f\xaf\x62\x4f\x92\x96\x5a\xa6\xb5\xc0\x1e\x0c\x80\xa8", + "\xc4\xe0\xad\x2f\x62\x27\x98\x98\xa7\xed\xe0\xf7\x09\xa1\xcc\xb8\xc1" + "\x00\x49\x41\xf3\xc5\x07\x43\x92\xe7\x95\x33", + 1, 128, NULL, 0 }, + { 112, 224, 163, + "\xca\xa8\x64\x17\x9f\x66\xe8\x26\xa0\xef\x3b\x5e\xdb\xe3", + "\x73\xf6\x42\x53\x70\x6c\xe6\xb5\x09\x4c\x24\xee\x01\x2e\xce\x9a\xc2" + "\x49\x52\x83\xdc\xd8\xc7\xf1\x11\x4e\x81\xe4\x58\x7d\x8e\xa4", + "\xa1\x22\x07\x45\xbb\x03\xd9\x82\x76\x3b\xfa\x7c\xe3\x52\xb8\xbc\x87" + "\x57\x6a\x0a\xd5\xd4\x6a\x0d\xa0\x8f\xf2\xd6", + 1, 256, NULL, 0 }, + { 112, 112, 164, + "\x66\x3a\x97\xd6\xb5\x49\x3d\xbf\xa6\x0c\x8d\xd0\x87\xed", "", + "\x0c\x6e\x21\xa8\x5e\x3c\xd2\xcd\x41\x3f\x36\x50\x7d\x6e", 1, 0, + NULL, 0 }, + { 112, 112, 165, + "\xb0\x8c\x34\x5a\x7c\x71\x66\xfd\xd3\x3c\xe7\x68\xc1\xdc", + "\x99\x64\xd8\x0e\xe2\x33\x8c\xff\xe2\x84\x83\xaa\x44\x6a\x6f\x76", + "\xaa\x00\x30\x15\x30\x9f\x2e\xd6\xfd\x77\x52\xe4\x9c\x31", 1, 128, + NULL, 0 }, + { 112, 112, 166, + "\xfc\x9d\x28\x83\xc6\x75\x34\xfe\xfb\xd6\xed\x4a\x97\x98", + "\xa4\x98\x20\xc1\x94\xa4\x3d\xee\xf1\x1f\x3a\x0f\x4e\xaa\x80\x42\x54" + "\x39\xfc\xa9\xd9\xf1\xd7\xc8\xe6\x65\xd6\xb1\x30\xe4\xe9\x08", + "\x1c\x2b\x96\x62\x3c\x91\xca\x9c\x50\x27\xf8\xf8\x1e\xde", 1, 256, + NULL, 0 }, + { 520, 224, 167, + "\xcf\xa6\x39\x65\x6c\xd4\x9f\x8d\x70\xf0\xb1\xa5\xa0\x56\xab\x4f\xc0" + "\xae\xee\xbc\x91\x33\x8d\x06\x7f\x36\xc4\x7b\x60\x12\xdc\x8d\x85\x6b" + "\x8a\xbc\xc4\xe1\xab\xff\xc9\x10\xae\xae\xe2\x1b\x4d\x36\x6e\x90\x74" + "\x88\xff\xd0\xca\x55\xb3\x6a\x62\x1a\xee\x0b\x2e\x9f\x0c", + "", + "\x0e\xf4\xfe\xda\xea\xab\x4a\xd5\x2c\x84\x36\x57\x04\x7b\x19\x78\x8a" + "\x9f\xa9\x10\x61\xb7\xa1\x4a\xdd\xa8\xc4\x90", + 1, 0, NULL, 0 }, + { 520, 224, 168, + "\xb3\x6d\x3d\x47\xa4\x58\x5b\x40\x1f\xc6\x4c\x98\xef\xf5\x62\x43\xd4" + "\xda\x78\x86\x30\x63\xd8\x14\xe8\x8f\x37\x0b\x92\x57\x64\x06\xd4\x47" + "\xfc\xf3\xd1\x29\xa1\xed\xe5\x7d\xdc\x56\xea\x3a\x0a\x1f\x10\x01\x05" + "\xa9\x5e\x83\x13\x8c\xdf\x45\xec\xf2\xa5\x99\x2a\xcf\x90", + "\x15\xc7\x5a\x64\xb0\x4d\x09\x7a\xf2\x37\x1a\xf3\x80\x07\x9e\xb8", + "\x4e\xcb\x2d\xaa\x5f\xb0\x8d\xbd\x83\x6e\x92\xa5\x1e\x20\x0b\xb2\x30" + "\xf5\x4a\xc2\xc9\x77\x8f\x52\x26\xb3\xab\xc9", + 1, 128, NULL, 0 }, + { 520, 224, 169, + "\xcf\x78\xb9\x91\x38\x2d\xb5\xe8\x66\x6c\xcb\x23\x33\xfb\x67\x21\x79" + "\xb1\x0a\x75\xcf\x9e\x5a\x76\x99\xae\x64\x00\x05\xe1\x97\x72\xef\x64" + "\x99\xa3\xbc\x97\xf1\x2e\x58\xe8\x35\xbb\x00\x17\xbb\x3b\x2e\x64\xc6" + "\xab\x44\xa0\xd6\x19\xdf\xa0\x36\x34\x84\xd1\xc9\x91\xe2", + "\xf6\x61\xe5\x98\xf1\x80\xf2\x5d\xc6\xdd\x76\xdb\x8a\x9e\x0e\x4c\x9c" + "\x27\x2b\x96\x65\xa6\xb1\x75\x65\x60\xc7\x23\xb8\xe0\x85\x95", + "\xcd\x55\xcd\xb0\xc4\xf0\x2b\x9f\x61\x48\x39\x29\x93\xb1\x8b\x4f\xf0" + "\x0a\x5e\x73\xb6\xf3\xfb\xf8\x3a\x85\x4a\xeb", + 1, 256, NULL, 0 }, + { 520, 112, 170, + "\x37\x72\xff\x6b\xb4\xe5\xb2\x81\x1c\xfd\x4d\x6a\x3d\x34\xdc\x74\xbc" + "\xa3\xdb\xf8\x9a\x58\x17\xb7\x9d\x84\x72\xa1\x38\x3b\x8c\x9a\xfb\x27" + "\xb3\x00\x61\x96\xce\x99\x66\x82\x9e\xae\x6a\x31\x3c\x2d\x72\x4d\x99" + "\x5f\x4d\xef\x17\x11\x7c\x09\xed\xcf\xc8\xc0\xcb\xbc\x93", + "", "\x40\xbe\xb1\xd3\xaa\xab\x25\xa4\x03\x22\x4e\x57\x77\x70", 1, 0, + NULL, 0 }, + { 520, 112, 171, + "\x2b\xa9\x10\xbc\x0b\xca\x90\x64\x4c\xb2\x1e\x96\x06\x3e\x2c\xd8\x5f" + "\x5d\xd0\x2f\xda\x75\xd3\x53\xc9\xb5\x1e\xaf\x45\xee\xe9\x4c\x16\x5c" + "\xa6\x59\x2d\x6c\xfd\xd9\x87\xbf\xdc\x1c\xba\x66\x36\x3d\x53\x5a\x14" + "\xb2\xf7\xea\xd8\x41\xb1\x7c\x4d\x76\xa5\x04\x91\x05\xf9", + "\x7b\xa4\x61\x04\x0d\xe9\xea\x3c\xef\xd4\x80\x91\x24\xf7\x8b\x39", + "\x4d\x28\xa9\x26\xdf\x1b\x18\x8e\x85\xd0\x92\xba\xcf\x11", 1, 128, + NULL, 0 }, + { 520, 112, 172, + "\x7f\xcf\x3c\xb1\xb1\xc5\xb5\x37\x49\x2a\xed\xe4\x68\x92\x84\xb5\x88" + "\x19\x35\xe3\x53\x7b\xb7\x30\x71\x98\xd6\x51\x8e\x7a\x6a\xab\xf7\x0b" + "\x50\xb4\x4e\x4a\x8d\xfe\xe3\x5e\x9f\x5c\xba\xda\x74\x47\xe5\x11\xa3" + "\x72\x09\x39\x0f\xcd\x17\x1c\x62\x07\x5c\x6a\x8b\xf1\xeb", + "\x83\xd2\x9c\x1c\x4d\x05\x9d\xdb\x0d\x2a\xca\x78\x7e\x5b\x70\x1b\xac" + "\x39\x53\xfb\x9b\xc7\x2d\xc8\x7b\x1e\xf9\x2a\x58\x2e\x97\x48", + "\x39\x2c\xe3\x8f\x78\x38\xb2\xf8\x71\x63\xee\xa0\x0b\x86", 1, 256, + NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/hmac_sha256_test.json.c b/test/wycheproof/hmac_sha256_test.json.c new file mode 100644 index 00000000..68c7406e --- /dev/null +++ b/test/wycheproof/hmac_sha256_test.json.c @@ -0,0 +1,1262 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* HMACSHA256, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha256_test_json[] = { + { 256, 256, 1, + "\x1e\x22\x5c\xaf\xb9\x03\x39\xbb\xa1\xb2\x40\x76\xd4\x20\x6c\x3e\x79" + "\xc3\x55\x80\x5d\x85\x16\x82\xbc\x81\x8b\xaa\x4f\x5a\x77\x79", + "", + "\xb1\x75\xb5\x7d\x89\xea\x6c\xb6\x06\xfb\x33\x63\xf2\x53\x8a\xbd\x73" + "\xa4\xc0\x0b\x4a\x13\x86\x90\x5b\xac\x80\x90\x04\xcf\x19\x33", + 1, 0, NULL, 0 }, + { 256, 256, 2, + "\x81\x59\xfd\x15\x13\x3c\xd9\x64\xc9\xa6\x96\x4c\x94\xf0\xea\x26\x9a" + "\x80\x6f\xd9\xf4\x3f\x0d\xa5\x8b\x6c\xd1\xb3\x3d\x18\x9b\x2a", + "\x77", + "\xdf\xc5\x10\x5d\x5e\xec\xf7\xae\x7b\x8b\x8d\xe3\x93\x0e\x76\x59\xe8" + "\x4c\x41\x72\xf2\x55\x51\x42\xf1\xe5\x68\xfc\x18\x72\xad\x93", + 1, 8, NULL, 0 }, + { 256, 256, 3, + "\x85\xa7\xcb\xaa\xe8\x25\xbb\x82\xc9\xb6\xf6\xc5\xc2\xaf\x5a\xc0\x3d" + "\x1f\x6d\xaa\x63\xd2\xa9\x3c\x18\x99\x48\xec\x41\xb9\xde\xd9", + "\xa5\x9b", + "\x0f\xe2\xf1\x3b\xba\x21\x98\xf6\xdd\xa1\xa0\x84\xbe\x92\x8e\x30\x4e" + "\x9c\xb1\x6a\x56\xbc\x0b\x7b\x93\x9a\x07\x32\x80\x24\x43\x73", + 1, 16, NULL, 0 }, + { 256, 256, 4, + "\x48\xf3\x02\x93\x34\xe5\x5c\xfb\xd5\x74\xcc\xc7\x65\xfb\x2c\x36\x85" + "\xaa\xb1\xf4\x83\x7d\x23\x37\x08\x74\xa3\xe6\x34\xc3\xa7\x6d", + "\xc7\xb8\xb2", + "\x6c\x13\xf7\x9b\xb2\xd5\xb6\xf9\xa3\x15\xfe\x8f\xd6\xcb\xb5\xcb\x81" + "\x7a\x66\x06\x87\x00\x9d\xec\xcd\x88\xc3\x77\x42\x9e\x59\x6d", + 1, 24, NULL, 0 }, + { 256, 256, 5, + "\xde\x8b\x5b\x5b\x2f\x09\x64\x5b\xe4\x7e\xcb\x64\x07\xa4\xe1\xd9\xc6" + "\xb3\x3a\xe3\xc2\xd2\x25\x17\xd3\x35\x7d\xa0\x35\x7a\x31\x39", + "\xcc\x02\x1d\x65", + "\xe8\x75\x38\xeb\x16\x7e\x62\xd7\xcb\x23\x66\x90\xff\x3f\x03\x4a\x9c" + "\x12\xd4\x17\xaa\x8d\xfa\x69\x4d\x74\x05\xf9\xe1\xf8\x5f\xe8", + 1, 32, NULL, 0 }, + { 256, 256, 6, + "\xb7\x93\x89\x10\xf5\x18\xf1\x32\x05\xca\x14\x92\xc6\x69\x00\x1a\x14" + "\xff\x91\x3c\x8a\xb4\xa0\xdc\x35\x64\xe7\x41\x8e\x91\x29\x7c", + "\xa4\xa6\xef\x6e\xbd", + "\x01\xa9\x3f\x4e\xd2\x16\xd0\xb2\x80\x89\x63\x01\xe3\x66\xaa\x67\xb2" + "\x5e\x6b\x6a\x5a\x6e\x84\xf2\x91\xa1\x33\x91\xc6\xe4\x96\xc5", + 1, 40, NULL, 0 }, + { 256, 256, 7, + "\x1b\xb9\x97\xff\x4d\xe8\xa5\xa3\x91\xde\x5c\x08\xa3\x3b\xc2\xc7\xc2" + "\x89\x1e\x47\xad\x5b\x9c\x63\x11\x01\x92\xf7\x8b\x98\xfe\x78", + "\x66\x7e\x01\x5d\xf7\xfc", + "\x06\xb5\xd8\xc5\x39\x23\x23\xa8\x02\xbc\x5c\xdd\x0b\x3c\x52\x74\x54" + "\xa8\x73\xd9\x65\x1c\x36\x88\x36\xea\xa4\xad\x98\x2b\xa5\x46", + 1, 48, NULL, 0 }, + { 256, 256, 8, + "\x32\xfd\xed\xa3\x9f\x98\xb4\xf4\x42\x6c\x2d\x2a\xc0\x0a\xb5\xdd\x4b" + "\xfa\xbb\x68\xf3\x11\x44\x72\x56\xed\x6d\x3d\x3a\x51\xb1\x54", + "\x41\x63\xa9\xf7\x7e\x41\xf5", + "\x1b\x01\x03\x72\x9f\x48\xc2\x77\x2b\xb1\x32\xae\xf9\xeb\xd6\xdd\x6a" + "\xaf\xc9\x14\x5d\xf6\xd5\xc5\x14\xb2\x33\xee\x92\xef\x4a\x00", + 1, 56, NULL, 0 }, + { 256, 256, 9, + "\x23\x3e\x4f\xde\xe7\x0b\xcc\x20\x23\x5b\x69\x77\xdd\xfc\x05\xb0\xdf" + "\x66\xf5\x63\x5d\x82\x7c\x66\xe5\xa6\x3c\xdb\x16\xa2\x49\x38", + "\xfd\xb2\xee\x4b\x6d\x1a\x0a\xc2", + "\x12\x0b\x26\xee\x13\x55\xc1\x34\xc2\x62\x51\x3c\x79\x22\xde\xb6\xc4" + "\xfd\x90\x30\x3d\xe4\xcd\x61\xb9\xf9\xcd\x08\xf2\x2d\x6e\x18", + 1, 64, NULL, 0 }, + { 256, 256, 10, + "\xb9\x84\xc6\x73\x4e\x0b\xd1\x2b\x17\x37\xb2\xfc\x7a\x1b\x38\x03\xb4" + "\xdf\xec\x40\x21\x40\xa5\x7b\x9e\xcc\xc3\x54\x14\xae\x66\x1b", + "\xde\xa5\x84\xd0\xe2\xa1\x4a\xd5\xfd", + "\x88\xbc\x22\x82\xe5\xfc\xe4\x7e\xc6\xd9\x89\x53\x95\xcd\x47\xff\xf9" + "\x1a\x0c\xdc\x58\x9a\x8f\xd5\x6d\x8d\x34\x46\x16\x53\x3a\x3d", + 1, 72, NULL, 0 }, + { 256, 256, 11, + "\xd0\xca\xf1\x45\x6a\xc5\xe2\x55\xfa\x6a\xfd\x61\xa7\x9d\xc8\xc7\x16" + "\xf5\x35\x8a\x29\x8a\x50\x82\x71\x36\x3f\xe1\xff\x98\x35\x61", + "\x18\x26\x1d\xc8\x06\x91\x3c\x53\x46\x66", + "\xf6\x78\xf0\x81\xd8\x3c\xf1\x26\xad\x6b\xd5\x2c\x2d\xff\xd7\x86\x21" + "\x4f\x51\x9c\x47\x45\x2b\x85\xa9\x74\x58\xd0\xc1\x0c\x3e\xe5", + 1, 80, NULL, 0 }, + { 256, 256, 12, + "\x83\x5b\xc8\x24\x1e\xd8\x17\x73\x5e\xc9\xd3\xd0\xe2\xdf\x4c\x17\x3e" + "\xe4\xdd\xed\x4a\x8e\xf0\xc0\x4a\x96\xc4\x8f\x11\x82\x04\x63", + "\x26\xf8\x08\x3e\x94\x4b\xac\xf0\x4e\x9a\x4d", + "\xe0\xe4\x6c\xd7\xd1\xa7\x5b\x3d\x10\x28\x93\xda\x64\xde\xf4\x6e\x45" + "\x53\x08\x76\x1f\x1d\x90\x87\x86\x62\x8c\xa7\xee\x22\xa0\xeb", + 1, 88, NULL, 0 }, + { 256, 256, 13, + "\x05\x5f\x95\xc9\x46\x1b\x08\x09\x57\x5e\xcc\xdf\xa5\xcd\xd0\x62\x75" + "\xf2\x5d\x30\x91\x5c\x4e\xb8\xdb\x40\xe1\xac\xd3\xab\x75\x91", + "\xbf\xb7\xd6\xa0\x8d\xba\xa5\x22\x5f\x32\x08\x87", + "\xe7\x6d\x5c\x8c\x07\x0a\x6b\x3c\x48\x24\xe9\xf3\x42\xdc\x30\x56\xe6" + "\x38\x19\x50\x9e\x1d\xef\x98\xb5\x85\xae\xba\x0d\x63\x8a\x00", + 1, 96, NULL, 0 }, + { 256, 256, 14, + "\xe4\x0f\x7a\x3e\xb8\x8d\xde\xc4\xc6\x34\x7e\xa4\xd6\x76\x10\x75\x6c" + "\x82\xc8\xeb\xcc\x23\x76\x29\xbf\x87\x3c\xca\xbc\x32\x98\x4a", + "\x7f\xe4\x3f\xeb\xc7\x84\x74\x64\x9e\x45\xbf\x99\xb2", + "\xaa\x57\xd0\x20\xaa\x24\xad\x82\x34\x72\xc2\xb8\x0f\xf2\xd0\xcf\x47" + "\x5f\x7d\xe0\x06\x8f\x9a\x59\xe8\x11\x2f\xed\xe5\x3a\x35\x81", + 1, 104, NULL, 0 }, + { 256, 256, 15, + "\xb0\x20\xad\x1d\xe1\xc1\x41\xf7\xec\x61\x5e\xe5\x70\x15\x21\x77\x3f" + "\x9b\x23\x2e\x4d\x06\x37\x6c\x38\x28\x94\xce\x51\xa6\x1f\x48", + "\x81\xc7\x58\x1a\x19\x4b\x5e\x71\xb4\x11\x46\xa5\x82\xc1", + "\xf4\x5c\x72\x60\x3c\xc1\x60\xc0\x76\x2f\x70\x34\x07\x84\x4a\x77\x81" + "\xdf\xe0\xf1\xdd\xf0\xaa\xf4\xcc\xd8\x20\x5e\x94\x46\x9a\xed", + 1, 112, NULL, 0 }, + { 256, 256, 16, + "\x9f\x3f\xd6\x1a\x10\x52\x02\x64\x8e\xcf\xf6\x07\x4c\x95\xe5\x02\xc1" + "\xc5\x1a\xcd\x32\xec\x53\x8a\x5c\xce\x89\xef\x84\x1f\x79\x89", + "\x2a\x76\xf2\xac\xda\xce\x42\xe3\xb7\x79\x72\x49\x46\x91\x2c", + "\x02\x26\xee\x13\xcc\x05\xe2\x34\x01\x35\xb3\xf4\xb2\x7a\x9d\xa1\xa1" + "\x60\xf6\x17\x0f\xe8\x05\xda\xdd\x98\xa3\x71\x1e\xc9\xc4\x21", + 1, 120, NULL, 0 }, + { 256, 256, 17, + "\x6f\xa3\x53\x86\x8c\x82\xe5\xde\xee\xda\xc7\xf0\x94\x71\xa6\x1b\xf7" + "\x49\xab\x54\x98\x23\x9e\x94\x7e\x01\x2e\xee\x3c\x82\xd7\xc4", + "\xae\xed\x3e\x4d\x4c\xb9\xbb\xb6\x0d\x48\x2e\x98\xc1\x26\xc0\xf5", + "\x9e\xd7\xf0\xe7\x38\x12\xa2\x7a\x87\xa3\x80\x8e\xe0\xc8\x9a\x64\x56" + "\x49\x9e\x83\x59\x74\xba\x57\xc5\xaa\xb2\xa0\xd8\xc6\x9e\x93", + 1, 128, NULL, 0 }, + { 256, 256, 18, + "\x53\x00\x48\x94\x94\xca\x86\x22\x1c\x91\xd6\xd9\x53\x95\x2a\xe1\xa5" + "\xe0\x97\x13\x9d\xc9\xcf\x11\x79\xc2\xf5\x64\x33\x75\x38\x24", + "\x90\xfe\xa6\xcf\x2b\xd8\x11\xb4\x49\xf3\x33\xee\x92\x33\xe5\x76" + "\x97", + "\x5b\x69\x2c\xba\x13\xb5\x4f\xff\xc3\xad\xcb\xb0\xe0\x15\xcc\x01\x1f" + "\xbf\xd6\x12\x35\x30\x3f\xf0\xad\x2a\x49\x77\x50\x83\xbf\x22", + 1, 136, NULL, 0 }, + { 256, 256, 19, + "\x38\x3e\x7c\x5c\x13\x47\x6a\x62\x26\x84\x23\xef\x05\x00\x47\x9f\x9e" + "\x86\xe2\x36\xc5\xa0\x81\xc6\x44\x91\x89\xe6\xaf\xdf\x2a\xf5", + "\x32\x02\x70\x5a\xf8\x9f\x95\x55\xc5\x40\xb0\xe1\x27\x69\x11\xd0\x19" + "\x71\xab\xb2\xc3\x5c\x78\xb2", + "\x4e\x49\x01\x59\x2b\xa4\x64\x76\x40\x8d\x75\x84\x35\xc7\xd1\xb4\x89" + "\xd2\x68\x9a\xfd\x84\xce\xaa\xee\x78\xbf\xb9\x1f\xd9\x39\x1d", + 1, 192, NULL, 0 }, + { 256, 256, 20, + "\x18\x6e\x24\x8a\xd8\x24\xe1\xeb\x93\x32\x9a\x7f\xdc\xd5\x65\xb6\xcb" + "\x4e\xaf\x3f\x85\xb9\x0b\x91\x07\x77\x12\x8d\x8c\x53\x8d\x27", + "\x92\xef\x9f\xf5\x2f\x46\xec\xcc\x7e\x38\xb9\xee\x19\xfd\x2d\xe3\xb3" + "\x77\x26\xc8\xe6\xce\x9e\x1b\x96\xdb\x5d\xda\x4c\x31\x79\x02", + "\x3f\xc1\xd7\x3d\xd4\xa8\x85\x8c\x1f\xc3\xd8\xc4\xa3\xf3\x3e\xd5\xad" + "\x0c\x70\x21\x00\x38\x39\x4a\x59\x02\xcb\x26\xfe\x28\x73\x48", + 1, 256, NULL, 0 }, + { 256, 256, 21, + "\x28\x85\x5c\x7e\xfc\x85\x32\xd9\x25\x67\x30\x09\x33\xcc\x1c\xa2\xd0" + "\x58\x6f\x55\xdc\xc9\xf0\x54\xfc\xca\x2f\x05\x25\x4f\xbf\x7f", + "\x9c\x09\x20\x7f\xf0\xe6\xe5\x82\xcb\x37\x47\xdc\xa9\x54\xc9\x4d\x45" + "\xc0\x5e\x93\xf1\xe6\xf2\x11\x79\xcf\x0e\x25\xb4\xce\xde\x74\xb5\x47" + "\x9d\x32\xf5\x16\x69\x35\xc8\x6f\x04\x41\x90\x58\x65", + "\x78\x8c\x05\x89\x00\x0f\xb7\xf0\xb5\xd5\x1f\x15\x96\x47\x2b\xc9\xec" + "\x41\x34\x21\xa4\x3d\xf9\x6e\xe3\x2b\x02\xb5\xd2\x75\xff\xe3", + 1, 376, NULL, 0 }, + { 256, 256, 22, + "\x8e\x54\x0c\xb3\x0c\x94\x83\x6a\xe2\xa5\x95\x0f\x35\x5d\x48\x2a\x70" + "\x02\xe2\x55\x20\x7e\x94\xfd\xa3\xf7\xef\x1a\x09\x90\x13\xa0", + "\xd6\x50\x0f\x95\xe1\x12\x62\xe3\x08\xbf\x3d\xf4\xdf\x4b\x85\x5f\x33" + "\xe8\x57\x56\x3d\x45\x43\xf1\x95\x63\x9a\x0a\x17\xb4\x42\xeb\x9f\xdc" + "\xc1\x36\x7d\x2e\xee\x75\xc8\xf8\x05\x73\x0b\x89\x29\x0f", + "\x39\x69\x7e\x70\xce\x74\x1f\xeb\x33\xde\xdc\x06\x9f\x00\xb5\x62\x7f" + "\xd9\xb8\x37\xd1\x0c\xbd\xd5\xb6\xd1\x9c\xfb\xd5\x11\xdd\x2c", + 1, 384, NULL, 0 }, + { 256, 256, 23, + "\x69\xc5\x0d\x52\x74\x35\x81\x88\xcf\xf4\xc0\xfa\xe7\x42\x24\x3d\x4e" + "\x8a\x5e\x5b\xa5\x5d\x94\xff\x40\xed\xd9\x0f\x6a\x43\xdd\x10", + "\x1a\xc5\x25\x5a\xff\x05\x28\x28\xd8\xea\x21\xb3\x76\xf1\xeb\xdd\x4b" + "\xb8\x79\x94\x99\x13\x90\x04\x05\xae\xbc\xe8\x3e\x48\xfe\xb6\x81\x3b" + "\x5e\x9c\x89\xf9\x45\x01\xa8\xad\xe4\x1b\x26\xb8\x15\xc5\x21", + "\x4b\x0b\x4d\x04\x16\xfa\x2e\x11\x58\x6f\xbf\xa7\xfb\x11\x26\x1e\x69" + "\x99\x1d\xfa\x34\x01\x9b\x98\x93\xd6\x9a\x2b\xe8\xc1\xfc\x80", + 1, 392, NULL, 0 }, + { 256, 256, 24, + "\x23\x20\x9b\x7c\x5a\xad\xcb\xd1\x3f\x72\x79\xaf\x1a\x86\xd3\xc7\xae" + "\x8f\x17\x9d\x1b\xca\xaa\xd0\xdf\xf9\xa1\x53\x02\xe7\x8d\xbf", + "\x84\xbd\xac\x37\xe1\xaf\x35\xd9\x35\x64\x04\xe2\x78\x7d\x47\xec\xe5" + "\x83\x48\xde\xa7\x6a\x4a\x46\xe8\xaa\xde\x34\x63\xd4\xdb\x8c\x94\xa0" + "\x51\xbe\x37\x33\xb3\x8d\x75\x69\x84\x86\x5d\x56\xc6\x0e\x80\x25\xf1" + "\x5e\x3f\x96\x8f\x09\x3e\x7f\xb7\xeb\xc7\xe3\x11\x89\xc5\x69\x2d\x15" + "\xed\x42\x56\x73\x7b\x9b\x18\x94\xe5\x80\x95\x03\xaa\xa1\xc9\x98\x3f" + "\xb0\x96\xaa\x21\x91\x63\x61\xee\xb6\xef\x45\x5b\x12\x97\x23\xa1\xa1" + "\xdd\xf9\xde\xdd\xea\x20\x85\x29\xa6\x48", + "\x4a\x85\xc4\x79\xd1\x65\x0d\xbd\x73\xbc\x52\x48\x07\x4a\x55\xff\x50" + "\x21\x8b\xdd\xaa\x8d\x1f\xdd\xaa\xf4\x49\x46\xdc\x19\xae\xfb", + 1, 896, NULL, 0 }, + { 256, 256, 25, + "\x7c\x9c\xc6\x67\xca\xe1\x75\xf4\x48\xfa\xa9\x66\x47\x31\x96\x33\xb2" + "\xd4\x85\x31\x37\x3a\xe7\xd3\x16\xc4\x4d\xdd\x8b\x9f\x69\xcf", + "\x92\x33\xc1\xd7\x3b\x49\x8c\x51\x06\xff\x88\x95\x1e\x07\xb9\x65\x2c" + "\xb0\xdd\xae\x74\x07\x37\xec\x20\x5c\x98\x76\xd0\x94\x97\x8b\xfc\x94" + "\x7f\x7d\xc9\x37\x11\x9f\xd6\xa9\x39\x15\xb1\x9b\x62\x59\x58\xa7\xa2" + "\x23\x63\xaa\x2a\xc3\x3f\xb8\x69\xed\x16\xb3\x03\x33\x6a\xb7\x40\xa0" + "\x49\x8a\x2d\xf6\x6a\x65\x99\xda\x71\x00\x94\x48\x1a\x7b\x54\x4b\xd9" + "\x55\xb6\xf9\x71\x35\xba\x46\x73\x40\x1d\xb2\xdb\x14\x4a\x6e\x28\x70" + "\x41\xe4\x7a\x51\xed\x9b\x6b\xa9\x56\xc1\x35\x08\xc1\xc0\xc2\x53\x10" + "\x10\x52\x39\xab\x73\x62\x9e\x30", + "\xca\x1b\x80\x44\x1d\x33\x39\x09\xc2\xbb\x30\x76\x96\x50\x05\x50\x51" + "\xed\x20\xf1\x7d\xe8\xee\x95\x3c\xb9\x07\x0a\xf5\x6c\x70\x4f", + 1, 1016, NULL, 0 }, + { 256, 256, 26, + "\x82\x31\x45\x40\x56\x4e\xa3\xce\x30\x59\x1e\x97\xf6\x8b\x26\x02\xde" + "\x40\xfa\x29\xf7\x73\xc2\x50\x83\x27\x47\x1b\x83\x48\xe8\xc4", + "\x6a\x6d\x2f\x45\xce\xbf\x27\x57\xae\x16\xea\x33\xc6\x86\x17\x67\x1d" + "\x77\xf8\xfd\xf8\x0b\xed\x8f\xc5\xcd\xc5\xc8\xb7\x08\x6b\xd2\x8e\x7e" + "\xb3\xee\xcc\x71\x63\x49\x11\x04\xe5\x30\x94\x55\xe6\x7f\x83\x65\x79" + "\xb8\x2a\x1d\xa3\xbf\x59\x91\xa8\xe2\xb2\xf1\x89\xa4\x9e\x05\x70\x0e" + "\x46\xc4\x09\xed\x5d\xe7\x77\x80\xa5\xf3\x89\xe3\xf1\x3d\xad\x40\x6c" + "\x9d\x55\x67\x53\x29\xc5\xc9\x21\xf0\x70\x34\x18\x09\x37\xc0\xf6\xef" + "\x34\xa2\x30\x8b\x6f\xf3\xe1\xa0\xe9\xdc\x1e\xa6\x5f\x56\x32\x73\x0e" + "\x87\x44\xd1\xdb\x2c\x40\xa6\x59\x5b", + "\x09\x00\xb3\xe6\x53\x5d\x34\xf9\x0e\x2c\x33\x57\x75\xe8\x6b\xf3\x8e" + "\xe7\xe3\xd2\x6f\xb6\x0c\xd9\xcd\xf6\x39\xeb\x34\x96\xb9\x4c", + 1, 1024, NULL, 0 }, + { 256, 256, 27, + "\xd1\x15\xac\xc9\xa6\x36\x91\x52\x41\x79\x5f\x48\x85\x20\x52\xe0\x7b" + "\x51\x27\x3a\xe2\x44\x82\x51\xec\x1d\x0d\x0f\x98\x07\xf3\xdb", + "\x69\x6d\x24\x56\xde\x85\x3f\xa0\x28\xf4\x86\xfe\xf4\x37\xb6\xb6\xd1" + "\xb5\x30\xa8\x47\x5e\x29\x9d\xb3\xa9\x00\x5a\xe9\xce\xf8\x40\x19\x85" + "\xb7\xd3\x1e\x17\x2e\x8f\x43\x9c\xcd\x1a\xd1\xec\x44\xc9\xb8\x6b\x78" + "\xf3\xf2\x43\xc1\x30\x5b\x53\xbc\x21\xab\xad\x7a\x8f\xc5\x25\x63\x11" + "\xbf\xd3\x4c\x98\xe3\x7d\xfd\xc6\x49\xe7\xae\x4b\xda\x08\xcf\x29\x94" + "\xb0\x63\xc0\xc7\x10\x6e\xd0\xb0\x2a\x1f\x48\xaf\x91\x91\xcb\xfb\x0d" + "\x6a\x95\x3b\x7e\x04\x32\x7d\xfe\x8c\x93\x77\x9c\xb5\x74\xba\x9c\xba" + "\x57\x5d\x01\x67\x4e\x83\x62\x1a\xa0\xc5\xf4\x00\xd6\xe6\xcd\x24\xb3" + "\x01\xe3\x3c\x9f\x33\x03\xe7\x3b\xf3\x57\x40\x8c\x1b\xe8\x6c\x24\x89" + "\xc0\x9d\xe9\x98\xff\x2e\xf3\x2d\xf5\x54\xf1\x24\x7d\x93\x13\xce\x1a" + "\x71\x60\x11\x5d\x06\xf4\xc1\x8d\x65\x56\xff\x79\x86\xef\x8a\x55\xe2" + "\xad\xcf\xa2\x7e\x4c\x69\xc7\x1c\xc2\xff\x01\x63\x9e\x9d\x49\xbd\x9e" + "\xd0\x68\x7f\x53\x0f\xfe\xb0\x89\x01\x32\x45\x7d\xf2\x08\x80\x81\xbc" + "\x4a\x2f\x7f\x0a\x9f\x4d\xce\xa2\xc8\x0d\x99\x1d\xb7\xf3\x74\x7a\x18" + "\x03\xd7\x61\x9a\xaf\x3d\xd3\x82\xc6\x95\x36\xa0\xbc\xdb\x93\x1c" + "\xbe", + "\x82\xf9\x29\x77\xf0\xb6\x05\xea\xad\xa5\x10\xff\xce\xb5\x3a\xd7\x5f" + "\xde\x16\xa8\x02\x9f\x1b\x75\xb4\x06\xa8\x42\x70\xdb\xb8\xb7", + 1, 2040, NULL, 0 }, + { 256, 256, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd1\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xda\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x53\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8a\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb8\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x89\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\xa7\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6c\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x08\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6f\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x0b\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x83\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x16\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x02\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x97\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x4b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x0c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x45\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbf\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd4\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x83\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd7\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x80\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\x55\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x02\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xca", + 0, 0, NULL, 0 }, + { 256, 256, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4f", + 0, 128, NULL, 0 }, + { 256, 256, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xc9", + 0, 0, NULL, 0 }, + { 256, 256, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4c", + 0, 128, NULL, 0 }, + { 256, 256, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\x8b", + 0, 0, NULL, 0 }, + { 256, 256, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x0e", + 0, 128, NULL, 0 }, + { 256, 256, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\x4b", + 0, 0, NULL, 0 }, + { 256, 256, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\xce", + 0, 128, NULL, 0 }, + { 256, 256, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8b\x42\x09\x6d\x80\xf4\x5f\x83\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb9\x9f\x27\x09\xa3\xca\x74\x16\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x89\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\xe7\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\xa7\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\x29\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\x67\x24" + "\x96\xa4\x15\xd3\xf4\xa1\xa8\xc8\x8e\x3b\xb9\xda\x8d\xc1\xcb", + 0, 0, NULL, 0 }, + { 256, 256, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\xa9\xb2" + "\x3a\x0c\x1e\x9c\x21\xbd\x85\x1f\xf2\xd2\xc3\x9d\xbe\xf1\x4e", + 0, 128, NULL, 0 }, + { 256, 256, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x2c\x74\xbd\xf6\x92\x7f\x0b\xa0\x7d\x94\xbb\x56\x2a\x9f\x82\x18\xdb" + "\x69\x5b\xea\x2c\x0b\x5e\x57\x37\x71\xc4\x46\x25\x72\x3e\x34", + 0, 0, NULL, 0 }, + { 256, 256, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x27\x46\x60\xd8\xf6\x5c\x35\x8b\xe8\xd3\x41\x6c\x7d\xb3\xe0\xd6\x4d" + "\xc5\xf3\xe1\x63\xde\x42\x7a\xe0\x0d\x2d\x3c\x62\x41\x0e\xb1", + 0, 128, NULL, 0 }, + { 256, 256, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 256, 256, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 256, 256, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 256, 256, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 256, 256, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x53\x0b\xc2\x89\xed\x00\x74\xdf\x02\xeb\xc4\x29\x55\xe0\xfd\x67\xa4" + "\x16\x24\x95\x53\x74\x21\x28\x48\x0e\xbb\x39\x5a\x0d\x41\x4b", + 0, 0, NULL, 0 }, + { 256, 256, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\x39\x1f\xa7\x89\x23\x4a\xf4\x97\xac\x3e\x13\x02\xcc\x9f\xa9\x32" + "\xba\x8c\x9e\x1c\xa1\x3d\x05\x9f\x72\x52\x43\x1d\x3e\x71\xce", + 0, 128, NULL, 0 }, + { 256, 256, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8a\x43\x08\x6c\x81\xf5\x5e\x83\x6a\x45\xa8\xd4\x61\x7c\xe6\x25" + "\x97\xa5\x14\xd2\xf5\xa0\xa9\xc9\x8f\x3a\xb8\xdb\x8c\xc0\xca", + 0, 0, NULL, 0 }, + { 256, 256, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb8\x9e\x26\x08\xa2\xcb\x75\x16\x2d\xbf\x92\x83\x4d\x1e\x28\xb3" + "\x3b\x0d\x1f\x9d\x20\xbc\x84\x1e\xf3\xd3\xc2\x9c\xbf\xf0\x4f", + 0, 128, NULL, 0 }, + { 256, 128, 82, + "\x7b\xf9\xe5\x36\xb6\x6a\x21\x5c\x22\x23\x3f\xe2\xda\xaa\x74\x3a\x89" + "\x8b\x9a\xcb\x9f\x78\x02\xde\x70\xb4\x0e\x3d\x6e\x43\xef\x97", + "", + "\xf4\x60\x55\x85\x94\x97\x47\xde\x26\xf3\xee\x98\xa7\x38\xb1\x72", 1, + 0, NULL, 0 }, + { 256, 128, 83, + "\xe7\x54\x07\x6c\xea\xb3\xfd\xaf\x4f\x9b\xca\xb7\xd4\xf0\xdf\x0c\xbb" + "\xaf\xbc\x87\x73\x1b\x8f\x9b\x7c\xd2\x16\x64\x72\xe8\xee\xbc", + "\x40", + "\x0d\xc0\x0d\x72\x17\xbb\xaf\xe8\xd7\x8b\xf9\x61\x18\x9b\x8f\xd2", 1, + 8, NULL, 0 }, + { 256, 128, 84, + "\xea\x3b\x01\x6b\xdd\x38\x7d\xd6\x4d\x83\x7c\x71\x68\x38\x08\xf3\x35" + "\xdb\xdc\x53\x59\x8a\x4e\xa8\xc5\xf9\x52\x47\x3f\xaf\xaf\x5f", + "\x66\x01", + "\xff\x29\x6b\x36\x8d\x3b\xf0\x59\xcc\x48\x68\x2f\x69\x49\xcc\xaa", 1, + 16, NULL, 0 }, + { 256, 128, 85, + "\x73\xd4\x70\x96\x37\x85\x7d\xaf\xab\x6a\xd8\xb2\xb0\xa5\x1b\x06\x52" + "\x47\x17\xfe\xdf\x10\x02\x96\x64\x4f\x7c\xfd\xaa\xe1\x80\x5b", + "\xf1\xd3\x00", + "\x2d\x02\xbd\x1c\x25\xb1\xfe\x52\xb1\xea\xd0\x73\x74\xd6\xe8\x83", 1, + 24, NULL, 0 }, + { 256, 128, 86, + "\xd5\xc8\x1b\x39\x9d\x4c\x0d\x15\x83\xa1\x3d\xa5\x6d\xe6\xd2\xdc\x45" + "\xa6\x6e\x7b\x47\xc2\x4a\xb1\x19\x2e\x24\x6d\xc9\x61\xdd\x77", + "\x2a\xe6\x3c\xbf", + "\x4d\x9e\x8b\xdd\xf9\xb7\xa1\x21\x83\x09\xd5\x98\x8a\xa1\xb0\xd9", 1, + 32, NULL, 0 }, + { 256, 128, 87, + "\x25\x21\x20\x3f\xa0\xdd\xdf\x59\xd8\x37\xb2\x83\x0f\x87\xb1\xaa\x61" + "\xf9\x58\x15\x5d\xf3\xca\x4d\x1d\xf2\x45\x7c\xb4\x28\x4d\xc8", + "\xaf\x3a\x01\x5e\xa1", + "\xcb\x8a\x4b\x41\x33\x50\xb4\x2f\x4a\xc3\x53\x3c\xc7\xf4\x78\x64", 1, + 40, NULL, 0 }, + { 256, 128, 88, + "\x66\x5a\x02\xbc\x26\x5a\x66\xd0\x17\x75\x09\x1d\xa5\x67\x26\xb6\x66" + "\x8b\xfd\x90\x3c\xb7\xaf\x66\xfb\x1b\x78\xa8\xa0\x62\xe4\x3c", + "\x3f\x56\x93\x5d\xef\x3f", + "\x1c\xfc\xe7\x45\xdb\x1c\xa7\xde\x9a\x1d\x44\x20\xe6\x12\xca\x55", 1, + 48, NULL, 0 }, + { 256, 128, 89, + "\xfa\xcd\x75\xb2\x22\x21\x38\x00\x47\x30\x5b\xc9\x81\xf5\x70\xe2\xa1" + "\xaf\x38\x92\x8e\xa7\xe2\x05\x9e\x3a\xf5\xfc\x6b\x82\xb4\x93", + "\x57\xbb\x86\xbe\xed\x15\x6f", + "\x0b\xde\x0d\x0c\x75\x6d\xf0\x9d\x4f\x6d\xa8\x1b\x29\x9a\x3a\xdf", 1, + 56, NULL, 0 }, + { 256, 128, 90, + "\x50\x5a\xa9\x88\x19\x80\x9e\xf6\x3b\x9a\x36\x8a\x1e\x8b\xc2\xe9\x22" + "\xda\x45\xb0\x3c\xe0\x2d\x9a\x79\x66\xb1\x50\x06\xdb\xa2\xd5", + "\x2e\x4e\x7e\xf7\x28\xfe\x11\xaf", + "\x40\x6a\x5c\x2b\xd3\xe6\xa9\x59\x5f\x9b\x7d\xff\x60\x8d\x59\xa7", 1, + 64, NULL, 0 }, + { 256, 128, 91, + "\xf9\x42\x09\x38\x42\x80\x8b\xa4\x7f\x64\xe4\x27\xf7\x35\x1d\xde\x6b" + "\x95\x46\xe6\x6d\xe4\xe7\xd6\x0a\xa6\xf3\x28\x18\x27\x12\xcf", + "\x85\x2a\x21\xd9\x28\x48\xe6\x27\xc7", + "\x0b\x1b\xf9\xe9\x8d\x0a\x79\x4f\xa5\x5c\x09\xb6\x3e\x25\x79\x9f", 1, + 72, NULL, 0 }, + { 256, 128, 92, + "\x64\xbe\x16\x2b\x39\xc6\xe5\xf1\xfe\xd9\xc3\x2d\x9f\x67\x4d\x9a\x8c" + "\xde\x6e\xaa\x24\x43\x21\x4d\x86\xbd\x4a\x1f\xb5\x3b\x81\xb4", + "\x19\x5a\x3b\x29\x2f\x93\xba\xff\x0a\x2c", + "\x71\xf3\x3f\x60\x21\xd9\x08\x58\xca\xdb\x13\x53\xd7\xfb\xe8\xd7", 1, + 80, NULL, 0 }, + { 256, 128, 93, + "\xb2\x59\xa5\x55\xd4\x4b\x8a\x20\xc5\x48\x9e\x2f\x38\x39\x2d\xda\xa6" + "\xbe\x9e\x35\xb9\x83\x3b\x67\xe1\xb5\xfd\xf6\xcb\x3e\x4c\x6c", + "\xaf\xd7\x31\x17\x33\x0c\x6e\x85\x28\xa6\xe4", + "\x4b\x8d\x76\x37\x2e\xbe\x5e\x5c\xaa\x56\xca\x4e\x5c\x59\xcd\xd3", 1, + 88, NULL, 0 }, + { 256, 128, 94, + "\x2c\x6f\xc6\x2d\xaa\x77\xba\x8c\x68\x81\xb3\xdd\x69\x89\x89\x8f\xef" + "\x64\x66\x63\xcc\x7b\x0a\x3d\xb8\x22\x8a\x70\x7b\x85\xf2\xdc", + "\x0f\xf5\x4d\x6b\x67\x59\x12\x0c\x2e\x8a\x51\xe3", + "\xc5\x80\xc5\x42\x84\x6a\x96\xe8\x4e\xa7\x77\x01\x77\x84\x55\xbf", 1, + 96, NULL, 0 }, + { 256, 128, 95, + "\xab\xab\x81\x5d\x51\xdf\x29\xf7\x40\xe4\xe2\x07\x9f\xb7\x98\xe0\x15" + "\x28\x36\xe6\xab\x57\xd1\x53\x6a\xe8\x92\x9e\x52\xc0\x6e\xb8", + "\xf0\x05\x8d\x41\x2a\x10\x4e\x53\xd8\x20\xb9\x5a\x7f", + "\x13\xcd\xb0\x05\x05\x93\x38\xf0\xf2\x8e\x2d\x8c\xe1\xaf\x5d\x0a", 1, + 104, NULL, 0 }, + { 256, 128, 96, + "\x3d\x5d\xa1\xaf\x83\xf7\x28\x74\x58\xbf\xf7\xa7\x65\x1e\xa5\xd8\xdb" + "\x72\x25\x94\x01\x33\x3f\x6b\x82\x09\x69\x96\xdd\x7e\xaf\x19", + "\xaa\xcc\x36\x97\x2f\x18\x30\x57\x91\x9f\xf5\x7b\x49\xe1", + "\xbd\x99\x3e\x44\x28\xcb\xc0\xe2\x75\xe4\xd8\x0b\x6f\x52\x03\x63", 1, + 112, NULL, 0 }, + { 256, 128, 97, + "\xc1\x9b\xdf\x31\x4c\x6c\xf6\x43\x81\x42\x54\x67\xf4\x2a\xef\xa1\x7c" + "\x1c\xc9\x35\x8b\xe1\x6c\xe3\x1b\x1d\x21\x48\x59\xce\x86\xaa", + "\x5d\x06\x6a\x92\xc3\x00\xe9\xb6\xdd\xd6\x3a\x7c\x13\xae\x33", + "\x86\xc9\xf4\xdd\xe0\xb2\x57\xa7\x05\x3a\x7b\x03\xc7\x50\x44\x09", 1, + 120, NULL, 0 }, + { 256, 128, 98, + "\x61\x2e\x83\x78\x43\xce\xae\x7f\x61\xd4\x96\x25\xfa\xa7\xe7\x49\x4f" + "\x92\x53\xe2\x0c\xb3\xad\xce\xa6\x86\x51\x2b\x04\x39\x36\xcd", + "\xcc\x37\xfa\xe1\x5f\x74\x5a\x2f\x40\xe2\xc8\xb1\x92\xf2\xb3\x8d", + "\xb9\x6b\xca\xca\xfa\xc3\x00\x94\xf1\x8a\xc5\x03\x9e\x7b\x36\x56", 1, + 128, NULL, 0 }, + { 256, 128, 99, + "\x73\x21\x6f\xaf\xd0\x02\x2d\x0d\x6e\xe2\x71\x98\xb2\x27\x25\x78\xfa" + "\x8f\x04\xdd\x9f\x44\x46\x7f\xbb\x64\x37\xaa\x45\x64\x1b\xf7", + "\xd5\x24\x7b\x8f\x6c\x3e\xdc\xbf\xb1\xd5\x91\xd1\x3e\xce\x23\xd2" + "\xf5", + "\x6e\x59\x7c\x4c\x38\x61\xa3\x80\xc0\x68\x54\xb4\x46\xfc\x2a\x87", 1, + 136, NULL, 0 }, + { 256, 128, 100, + "\x04\x27\xa7\x0e\x25\x75\x28\xf3\xab\x70\x64\x0b\xba\x1a\x5d\xe1\x2c" + "\xf3\x88\x5d\xd4\xc8\xe2\x84\xfb\xbb\x55\xfe\xb3\x52\x94\xa5", + "\x13\x93\x7f\x85\x44\xf4\x42\x70\xd0\x11\x75\xa0\x11\xf7\x67\x0e\x93" + "\xfa\x6b\xa7\xef\x02\x33\x6e", + "\xf7\x31\xaa\xf2\xf0\x40\x23\xd6\x21\xf1\x04\x95\x34\x46\x79\xa0", 1, + 192, NULL, 0 }, + { 256, 128, 101, + "\x96\xe1\xe4\x89\x6f\xb2\xcd\x05\xf1\x33\xa6\xa1\x00\xbc\x56\x09\xa7" + "\xac\x3c\xa6\xd8\x17\x21\xe9\x22\xda\xdd\x69\xad\x07\xa8\x92", + "\x91\xa1\x7e\x4d\xfc\xc3\x16\x6a\x1a\xdd\x26\xff\x0e\x7c\x12\x05\x6e" + "\x8a\x65\x4f\x28\xa6\xde\x24\xf4\xba\x73\x9c\xeb\x5b\x5b\x18", + "\x95\x24\x3e\xb1\xa9\xd4\x48\x17\x4a\xe4\xfc\xcf\x4a\x53\xeb\xfe", 1, + 256, NULL, 0 }, + { 256, 128, 102, + "\x41\x20\x15\x67\xbe\x4e\x6e\xa0\x6d\xe2\x29\x5f\xd0\xe6\xe8\xa7\xd8" + "\x62\xbb\x57\x31\x18\x94\xf5\x25\xd8\xad\xea\xbb\xa4\xa3\xe4", + "\x58\xc8\xc7\x3b\xdd\x3f\x35\x0c\x97\x47\x78\x16\xea\xe4\xd0\x78\x9c" + "\x93\x69\xc0\xe9\x9c\x24\x89\x02\xc7\x00\xbc\x29\xed\x98\x64\x25\x98" + "\x5e\xb3\xfa\x55\x70\x9b\x73\xbf\x62\x0c\xd9\xb1\xcb", + "\x34\x33\x67\x20\x7f\x71\x42\x5d\x8f\x81\xf3\x11\x0b\x04\x05\xf6", 1, + 376, NULL, 0 }, + { 256, 128, 103, + "\x64\x9e\x37\x3e\x68\x1e\xf5\x2e\x3c\x10\xac\x26\x54\x84\x75\x09\x32" + "\xa9\x91\x8f\x28\xfb\x82\x4f\x7c\xb5\x0a\xda\xb3\x97\x81\xfe", + "\x39\xb4\x47\xbd\x3a\x01\x98\x3c\x1c\xb7\x61\xb4\x56\xd6\x90\x00\x94" + "\x8c\xeb\x87\x05\x62\xa5\x36\x12\x6a\x0d\x18\xa8\xe7\xe4\x9b\x16\xde" + "\x8f\xe6\x72\xf1\x3d\x08\x08\xd8\xb7\xd9\x57\x89\x99\x17", + "\x15\x16\x18\xee\xc4\xf5\x03\xf3\xb6\x3b\x53\x9d\xe0\xa5\x89\x66", 1, + 384, NULL, 0 }, + { 256, 128, 104, + "\x7b\x0d\x23\x7f\x7b\x53\x6e\x2c\x69\x50\x99\x0e\x61\xb3\x61\xb3\x84" + "\x33\x3d\xda\x69\x00\x45\xc5\x91\x32\x1a\x4e\x3f\x79\x74\x7f", + "\x3d\x62\x83\xd1\x1c\x02\x19\xb5\x25\x62\x0e\x9b\xf5\xb9\xfd\x88\x7d" + "\x3f\x0f\x70\x7a\xcb\x1f\xbd\xff\xab\x0d\x97\xa5\xc6\xd0\x7f\xc5\x47" + "\x76\x2e\x0e\x7d\xd7\xc4\x3a\xd3\x5f\xab\x1c\x79\x0f\x80\x47", + "\xce\x20\x1c\x0d\xcf\xdc\x3f\x2b\xef\x36\x06\x09\xa3\x1f\xb1\x9e", 1, + 392, NULL, 0 }, + { 256, 128, 105, + "\x17\xc9\x26\x63\x74\x1f\x01\x2e\x5b\xb6\x71\x4e\x61\x4c\x2d\x15\x59" + "\x48\x61\x7f\x10\x93\x62\x69\xd9\x54\xc5\x8a\xba\x2a\xe6\x2d", + "\x7f\xdd\x6a\x15\xc8\x61\xd0\x31\x3f\x66\x35\xd7\x7d\xc5\x5e\x11\x5f" + "\xf1\x8c\x8a\xb0\x63\xb5\xd0\x3e\xab\x47\x2e\xec\xa8\x7a\x37\x81\x88" + "\xf2\x58\x13\x51\x5c\xf9\x0b\x6c\xff\xa9\x4a\x8f\xf3\x6b\x29\xd6\x56" + "\x03\xea\xb3\xfb\xd2\xaa\x95\x00\xb2\x61\xe1\x84\x04\x98\x93\xdc\x6c" + "\xa2\x01\x0b\xec\xac\x16\x30\x53\xf2\x11\x07\x0b\xdd\xa6\x21\xb8\xbd" + "\x8a\xf7\x7e\x45\x02\x68\x60\x3b\x52\xdb\x34\xc9\x0b\xe8\x36\xdf\xeb" + "\xdd\xef\x42\x30\x3f\x72\x4e\x63\xbf\x0f", + "\x76\xe8\xdf\xd9\x4d\xb4\xaf\x9d\x79\xd9\x71\x8e\xec\x46\xcb\x2d", 1, + 896, NULL, 0 }, + { 256, 128, 106, + "\x42\x4c\x6b\x22\x60\x6f\xcc\x09\x4a\xe8\x2f\xc5\xd3\xcb\xe4\x84\x17" + "\x4c\x22\x11\xb3\xec\x77\x80\x91\xca\xc3\x4a\x8e\x38\xa1\x52", + "\xd9\x6f\xf0\x62\xe2\x49\x0e\x8e\x0c\x54\xc5\xa8\xb8\x9e\x85\xb2\x5a" + "\x66\xd9\x3d\x7c\x2b\x93\xbd\xfe\xf8\x46\xb7\x0d\x38\x67\x27\x46\xa4" + "\xb9\x88\xd0\x8f\x15\xa5\xc5\x27\xca\x4f\x2c\x80\xe5\x3f\x7c\x6a\xc0" + "\x52\x1b\xc5\x7e\xbe\x38\x20\x91\x80\xcb\xf9\x34\xe0\xbb\xeb\x58\xcf" + "\xb6\x3d\x75\xda\x64\xaf\x41\xd0\x9c\xe1\x74\xaf\x18\x96\xf4\x25\x22" + "\x91\x0f\xce\xd3\x5e\xa0\x00\x40\x2e\x95\xfd\x3a\xc7\xaa\x6d\x5e\x0a" + "\x6b\x53\x3b\x08\x79\xbc\x46\x60\x19\xb3\xa5\xe6\xb1\x6e\x4b\xd1\xea" + "\x6c\xdf\xc9\xcc\xc1\xd6\xf0\xf0", + "\xed\xa7\x09\xc7\x00\x97\x14\xc3\x72\xd0\xd6\xa6\x3d\xfd\xe4\x69", 1, + 1016, NULL, 0 }, + { 256, 128, 107, + "\x15\xd5\x53\xc8\xda\x43\x3d\x53\xcd\xc7\xf1\x50\x87\xa7\x03\x49\xca" + "\xab\x57\xb3\x79\xa4\x07\x89\x28\xce\x9b\x99\x30\x2e\x31\xa6", + "\xd6\xc0\xc5\x3b\x73\xf7\x4f\xb4\x26\xad\xfd\xc1\x43\xd7\x0d\xb7\xf7" + "\xa8\xf8\xed\x32\xa2\xfa\xef\x26\x3c\xf9\xab\x11\x75\x37\xb6\xb9\xd1" + "\x72\x8b\xd1\x00\x0c\x1f\x28\x90\x6c\x6c\xe6\xad\x21\x86\x2b\xfa\x4d" + "\x68\x9c\x1a\x8e\xbe\x38\x68\xb9\x92\x09\x8b\x7f\x98\x1b\x2a\xf5\x18" + "\x9a\x6a\xde\xdf\xf5\x3a\x6c\x70\xc8\x36\x93\xf5\xc8\xd6\x38\x5a\x9a" + "\x8a\x4d\xca\x01\x7c\x57\x16\xac\x4d\x5b\x97\x65\xc5\xca\x2a\xb5\xf9" + "\x86\x7e\x02\x79\x51\x98\xc0\xb9\x52\x7e\x07\xd0\x8a\xf5\x2d\xbc\xb9" + "\x1c\xeb\x3d\x8b\x41\x2a\x2b\x24\x02", + "\x8c\xa1\x40\x2b\xf8\xfc\x23\x44\x2a\xc2\x06\x7b\xe9\x25\xb8\x28", 1, + 1024, NULL, 0 }, + { 256, 128, 108, + "\xff\xe5\x59\x46\x8a\x10\x31\xdf\xb3\xce\xd2\xe3\x81\xe7\x4b\x58\x21" + "\xa3\x6d\x9a\xbf\x5f\x2e\x59\x89\x5a\x7f\xdc\xa0\xfa\x56\xa0", + "\x23\x88\x99\xa8\x4a\x3c\xf1\x52\x02\xa1\xfb\xef\x47\x41\xe1\x33\xfb" + "\x24\xc0\x09\xa0\xcd\x83\x85\x4c\x6d\x1d\x7c\x92\x66\xd4\xc3\xea\xfe" + "\x6d\x1d\xfc\x18\xf1\x38\x45\xcc\xda\xd7\xfe\x27\x76\x27\xb5\xfd\x5f" + "\xf2\x55\x5c\xe6\xdf\xde\x1e\xe0\x78\x54\x0a\x0a\x35\x90\xc6\xd9\xbf" + "\x2f\xb6\x3b\xa9\xaf\xbe\x93\x80\xe7\x97\xbe\x7c\xd0\x17\x64\x5c\x5a" + "\x36\x13\xee\xf3\x8e\xf8\x9e\x3b\x74\x61\xe6\xe7\x00\xff\x2b\x4d\xee" + "\xf5\x63\x6c\x9d\x21\x98\xb1\x43\xf7\x97\xca\x18\x20\xa3\xdc\xc5\xd4" + "\x62\xeb\xf4\xa8\xc4\xc0\x9e\xb2\x02\xa2\x35\x92\xeb\x95\x24\x08\x2c" + "\x79\xad\xda\x8f\xcd\x56\xd2\x56\x04\x1a\x26\xbf\x8f\x52\x39\x62\xba" + "\x91\x1c\xe5\xa5\x78\x65\x70\xd6\x5b\xe3\xc4\xdf\x72\x2e\xd8\x83\x03" + "\x02\x06\x5f\xeb\xdf\x94\x47\x15\x29\x8a\x1f\xbb\x7d\x10\xb6\x8d\x7d" + "\xa2\xbf\x88\x93\x24\x31\x4c\xe5\x1e\x81\x5c\x7f\xbf\x03\xaa\x0a\x83" + "\x58\xaf\xf3\xa8\x6e\xb7\xa3\x3f\x9a\x49\x23\x66\x0d\xb3\x04\x7e\x79" + "\x3b\xeb\xb0\xc6\x91\x8f\x43\x95\xd4\x00\x38\x17\x23\xfd\xae\x28\x32" + "\xc3\x6e\xfc\x8e\x36\x8a\x68\xf3\x0f\x63\x51\xc3\xbc\x94\x2c\xd5" + "\x60", + "\xa8\x30\xb3\x13\xf4\x93\x6d\xea\x56\xa3\xae\xfd\x6a\x3e\xbe\x7d", 1, + 2040, NULL, 0 }, + { 256, 128, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd1\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xda\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x53\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8a\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb8\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x89\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\xa7\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6c\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x08\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6f\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x0b\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x83\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x16\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x02\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x97\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x4b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x0c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x45\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbf\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd4\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x83\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd7\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x80\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\x55\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x02\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe6", 0, + 0, NULL, 0 }, + { 256, 128, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x28", 0, + 128, NULL, 0 }, + { 256, 128, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xe5", 0, + 0, NULL, 0 }, + { 256, 128, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x2b", 0, + 128, NULL, 0 }, + { 256, 128, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\xa7", 0, + 0, NULL, 0 }, + { 256, 128, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\x69", 0, + 128, NULL, 0 }, + { 256, 128, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\x5f\x82\x6b\x44\xa9\xd5\x60\x7d\x67", 0, + 0, NULL, 0 }, + { 256, 128, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\x74\x17\x2c\xbe\x93\x82\x4c\x1f\xa9", 0, + 128, NULL, 0 }, + { 256, 128, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8b\x42\x09\x6d\x80\xf4\x5f\x83\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb9\x9f\x27\x09\xa3\xca\x74\x16\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x89\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\xe7", 0, + 0, NULL, 0 }, + { 256, 128, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\xa7\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\x29", 0, + 128, NULL, 0 }, + { 256, 128, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd3\x8b\x42\x09\x6d\x80\xf4\xdf\x82\x6b\x44\xa9\xd5\x60\x7d\x67", 0, + 0, NULL, 0 }, + { 256, 128, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd8\xb9\x9f\x27\x09\xa3\xca\xf4\x17\x2c\xbe\x93\x82\x4c\x1f\xa9", 0, + 128, NULL, 0 }, + { 256, 128, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x2c\x74\xbd\xf6\x92\x7f\x0b\xa0\x7d\x94\xbb\x56\x2a\x9f\x82\x18", 0, + 0, NULL, 0 }, + { 256, 128, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x27\x46\x60\xd8\xf6\x5c\x35\x8b\xe8\xd3\x41\x6c\x7d\xb3\xe0\xd6", 0, + 128, NULL, 0 }, + { 256, 128, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 0, NULL, 0 }, + { 256, 128, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 0, + 128, NULL, 0 }, + { 256, 128, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 0, NULL, 0 }, + { 256, 128, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 0, + 128, NULL, 0 }, + { 256, 128, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\x53\x0b\xc2\x89\xed\x00\x74\xdf\x02\xeb\xc4\x29\x55\xe0\xfd\x67", 0, + 0, NULL, 0 }, + { 256, 128, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x58\x39\x1f\xa7\x89\x23\x4a\xf4\x97\xac\x3e\x13\x02\xcc\x9f\xa9", 0, + 128, NULL, 0 }, + { 256, 128, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "", + "\xd2\x8a\x43\x08\x6c\x81\xf5\x5e\x83\x6a\x45\xa8\xd4\x61\x7c\xe6", 0, + 0, NULL, 0 }, + { 256, 128, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xd9\xb8\x9e\x26\x08\xa2\xcb\x75\x16\x2d\xbf\x92\x83\x4d\x1e\x28", 0, + 128, NULL, 0 }, + { 128, 256, 163, + "\xa3\x49\xac\x0a\x9f\x9f\x74\xe4\x8e\x09\x9c\xc3\xdb\xf9\xa9\xc9", + "", + "\x3a\x84\x37\xb8\x77\xb7\x5c\xc0\x8a\x4d\x8d\x75\x59\xa8\xfc\x68\x69" + "\xa5\x8c\x71\x3d\xa6\x3d\x1d\x4b\x35\x0d\x59\xb5\x97\xe3\x0c", + 1, 0, NULL, 0 }, + { 128, 256, 164, + "\xac\x68\x6b\xa0\xf1\xa5\x1b\x4e\xc4\xf0\xb3\x04\x92\xb7\xf5\x56", + "\x2f\xa4\x3a\x14\xae\x50\x05\x07\xde\xb9\x5a\xb5\xbd\x32\xb0\xfe", + "\x00\x85\x32\xa5\x3d\x0c\x0a\xb2\x20\x27\xae\x24\x90\x23\x37\x53\x74" + "\xe2\x23\x9b\x95\x96\x09\xe8\x33\x9b\x05\xa1\x57\x42\xa6\x75", + 1, 128, NULL, 0 }, + { 128, 256, 165, + "\x73\xef\x9e\xf1\xa4\x22\x5e\x51\xe3\xc1\xdb\x3a\xce\x1f\xa2\x4f", + "\xff\xad\x38\x0d\x9a\xab\xb0\xac\xed\xe5\xc1\xbf\x11\x29\x25\xcd\xfc" + "\x3d\x37\x9f\xc2\x37\x6a\x4f\xe2\x64\x44\x90\xd0\x43\x0a\xc3", + "\x9c\x7c\xb9\xf7\xc2\x07\xec\x46\xd1\xe3\xc5\x57\x64\x73\x1c\x4a\xb5" + "\xdd\xba\xe4\xe1\x40\x1e\x52\xa8\x95\xdf\x0c\xff\x47\x87\xc9", + 1, 256, NULL, 0 }, + { 128, 128, 166, + "\xe3\x4f\x15\xc7\xbd\x81\x99\x30\xfe\x9d\x66\xe0\xc1\x66\xe6\x1c", + "", + "\x1d\x76\x5a\xb9\xe2\x98\x92\xf7\xbf\xec\x29\x75\xad\x4b\xc2\xdc", 1, + 0, NULL, 0 }, + { 128, 128, 167, + "\xe0\x9e\xaa\x5a\x3f\x5e\x56\xd2\x79\xd5\xe7\xa0\x33\x73\xf6\xea", + "\xef\x4e\xab\x37\x18\x1f\x98\x42\x3e\x53\xe9\x47\xe7\x05\x0f\xd0", + "\xcf\xc1\x9e\xc0\x79\x02\xec\x8b\xe4\x89\x60\x6d\x8f\x40\xd1\x72", 1, + 128, NULL, 0 }, + { 128, 128, 168, + "\x9b\xd3\x90\x2e\xd0\x99\x6c\x86\x9b\x57\x22\x72\xe7\x6f\x38\x89", + "\xa7\xba\x19\xd4\x9e\xe1\xea\x02\xf0\x98\xaa\x8e\x30\xc7\x40\xd8\x93" + "\xa4\x45\x6c\xcc\x29\x40\x40\x48\x4e\xd8\xa0\x0a\x55\xf9\x3e", + "\xac\x50\xad\xad\x97\x85\xa8\x9c\x72\x82\xd8\xab\x88\x1d\xc6\x15", 1, + 256, NULL, 0 }, + { 520, 256, 169, + "\x8a\x0c\x46\xeb\x8a\x29\x59\xe3\x98\x65\x33\x00\x79\x76\x33\x41\xe7" + "\x43\x9d\xab\x14\x96\x94\xee\x57\xe0\xd6\x1e\xc7\x3d\x94\x7e\x1d\x53" + "\x01\xcd\x97\x4e\x18\xa5\xe0\xd1\xcf\x0d\x2c\x37\xe8\xaa\xdd\x9f\xd5" + "\x89\xd5\x7e\xf3\x2e\x47\x02\x4a\x99\xbc\x3f\x70\xc0\x77", + "", + "\xf5\xbf\xb9\x40\x56\x1f\xb4\xdb\x73\xeb\xba\x49\xbf\x2e\x48\x93\xbb" + "\x0c\xca\x61\x8a\x71\xb7\xec\xf6\xac\xa3\x82\x31\xe1\x67\xea", + 1, 0, NULL, 0 }, + { 520, 256, 170, + "\x28\x77\xeb\xb8\x1f\x80\x33\x4f\xd0\x05\x16\x33\x74\x46\xc5\xcf\x5a" + "\xd4\xa3\xa2\xe1\x97\x26\x9e\x5b\x0a\xd1\x88\x9d\xfe\x2b\x4b\x0a\xaa" + "\x67\x6f\xac\x55\xb3\x6c\xe3\xaf\xfc\x7f\x10\x92\xab\x89\xc5\x32\x73" + "\xa8\x37\xbd\x5b\xc9\x4d\x1a\x9d\x9e\x5b\x02\xe9\x85\x6f", + "\xba\x44\x8d\xb8\x8f\x15\x4f\x77\x50\x28\xfd\xec\xf9\xe6\x75\x2d", + "\x16\x90\xed\x41\x80\x64\x28\x99\xe0\xde\xb9\xec\x22\x70\x37\x4e\x8b" + "\x0a\x48\x42\x17\xf5\xa6\x82\xc5\x24\x31\x6e\xca\x21\x9b\x64", + 1, 128, NULL, 0 }, + { 520, 256, 171, + "\x21\x17\x8e\x26\xbc\x28\xff\xc2\x7c\x06\xf7\x62\xba\x19\x0a\x62\x70" + "\x75\x85\x6d\x7c\xa6\xfe\xab\x79\xac\x63\x14\x9b\x17\x12\x6e\x34\xfd" + "\x9e\x55\x90\xe0\xe9\x0a\xac\x80\x1d\xf0\x95\x05\xd8\xaf\x2d\xd0\xa2" + "\x70\x3b\x35\x2c\x57\x3a\xc9\xd2\xcb\x06\x39\x27\xf2\xaf", + "\x7d\x5f\x1d\x6b\x99\x34\x52\xb1\xb5\x3a\x43\x75\x76\x0d\x10\xa2\x0d" + "\x46\xa0\xab\x9e\xc3\x94\x3f\xc4\xb0\x7a\x2c\xe7\x35\xe7\x31", + "\xe5\x42\xac\x8a\xc8\xf3\x64\xba\xe4\xb7\xda\x8b\x7a\x07\x77\xdf\x35" + "\x0f\x00\x1d\xe4\xe8\xcf\xa2\xd9\xef\x0b\x15\x01\x94\x96\xec", + 1, 256, NULL, 0 }, + { 520, 128, 172, + "\x81\x3e\x0c\x07\x8c\x22\x13\x75\xe8\x05\x90\xac\xe6\x77\x4e\xaf\xd2" + "\xd2\xc2\x42\x35\x09\x88\xd0\x2e\xfa\x55\x0e\x05\xae\xcb\xe1\x00\xc1" + "\xb8\xbf\x15\x4c\x93\x2c\xf9\xe5\x71\x77\x01\x5c\x81\x6c\x42\xbc\x7f" + "\xbc\x71\xce\xaa\x53\x28\xc7\x31\x6b\x7f\x0f\x30\x33\x0f", + "", + "\xbb\x6a\xb6\x6f\x51\xe5\x3f\xa0\x86\xc9\xc6\x1a\x26\xca\x27\xe0", 1, + 0, NULL, 0 }, + { 520, 128, 173, + "\x57\x13\x34\x30\x96\xb0\xaa\xf0\x56\x2a\x6b\x92\xc1\xa1\x55\x35\x92" + "\x41\x60\x47\x5a\x4e\x42\x33\x58\x91\x59\x72\x8c\x56\x2e\x3b\x2a\xd9" + "\x6f\x74\x0c\x6a\x4d\xa2\xbc\x3f\x76\x8c\xe9\x8c\x9b\xd6\x6b\xac\x28" + "\xd1\x64\x6f\xf5\x92\x02\x8c\x94\x0d\x45\x5f\x35\xee\xb4", + "\x71\x71\x2d\xe2\xfa\xc1\xfb\x85\x56\x73\xbf\xf7\x2a\xf6\x42\x57", + "\xc1\x81\x65\xb8\xb9\x7d\xb1\xca\x5e\x24\x86\xa3\x2b\x39\x73\x1e", 1, + 128, NULL, 0 }, + { 520, 128, 174, + "\x72\x08\xaf\xbe\xcf\x5f\x1f\x34\x82\x8f\x98\xb7\x19\x41\x4e\x28\x07" + "\x16\xde\x64\xf5\xed\xd1\xae\x1c\x77\x41\x53\xcd\x20\x22\x33\x7b\xb2" + "\x0f\xad\xe1\xb7\x85\x6f\x1d\xbf\xd4\x0e\x2b\x43\x07\xf1\x29\x3c\xef" + "\xf1\x69\x2e\xe9\x0d\x8c\x90\xb5\xfd\xf9\x53\xab\x01\xa5", + "\x43\xb5\x33\x02\xb6\x04\xd6\x13\xe6\x2d\xb0\x02\x04\x4a\x47\x82\xd5" + "\x72\xac\x8f\xbd\x3c\xd0\xec\xe9\x1b\x43\xbc\x52\xe1\x8e\x98", + "\x2f\xec\xfe\x45\xd7\x93\x39\xc5\x7d\xdd\xba\x68\xab\x34\xf5\xf1", 1, + 256, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/hmac_sha384_test.json.c b/test/wycheproof/hmac_sha384_test.json.c new file mode 100644 index 00000000..09bd57fc --- /dev/null +++ b/test/wycheproof/hmac_sha384_test.json.c @@ -0,0 +1,1604 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* HMACSHA384, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha384_test_json[] = { + { 384, 384, 1, + "\xee\x8d\xf0\x67\x85\x7d\xf2\x30\x0f\xa7\x1a\x10\xc3\x09\x97\x17\x8b" + "\xb3\x79\x61\x27\xb5\xec\xe5\xf2\xcc\xc1\x70\x93\x2b\xe0\xe7\x8e\xa9" + "\xb0\xa5\x93\x6c\x09\x15\x7e\x67\x1c\xe7\xec\x9f\xc5\x10", + "", + "\xa6\x55\x18\x4d\xaf\x33\x46\xff\xc6\x62\x9d\x49\x3c\x84\x42\x64\x4e" + "\x49\x96\xa2\x79\x9e\x42\xe3\x30\x6f\xa6\xf5\xb0\x96\x7b\x6c\xf3\xa6" + "\xf8\x19\xba\xb8\x9b\xce\x29\x7d\x1d\x1a\x59\x07\xb2\xd0", + 1, 0, NULL, 0 }, + { 384, 384, 2, + "\x97\x66\x96\xc0\xdc\x97\x18\x2c\xa7\x71\x97\x5c\x39\x28\xff\x91\x68" + "\xef\x89\xcd\x74\x0c\xd2\x29\x28\x58\xfd\x91\x60\x68\xa7\x02\xbc\x1d" + "\xf7\xc6\xcd\x8e\xe1\xf0\xd2\x5e\x61\xd4\xc5\x14\xcc\x5d", + "\x2b", + "\x36\x3e\x89\x73\xfe\xdc\xf7\x89\x20\x13\xdf\xae\x0b\x70\x65\xd6\x1d" + "\x80\xb9\x8c\x63\x5b\xc0\x9e\xd8\x60\xa0\x14\x73\xb9\xbc\xd0\xdc\x55" + "\x0d\xbf\x66\xcf\x0d\x60\x1f\xe9\xcb\xf3\xae\x59\x62\x0d", + 1, 8, NULL, 0 }, + { 384, 384, 3, + "\xc5\x5e\xa4\xc6\x4a\x0a\x63\xe2\xd1\x4a\xd4\x25\x59\xba\x7c\x81\x6b" + "\x88\x24\xd2\x63\xc2\xcc\x6a\x01\x57\x61\xb5\x3f\x68\x1e\x51\x43\x69" + "\xf0\xdf\xba\x5c\xde\x16\x53\x20\xee\x10\xa9\x6e\xb1\xfc", + "\x5a\xbd", + "\xcc\xc2\x92\x5f\x16\x4a\x7d\x96\x62\xf1\xe7\x6b\xca\xf6\x34\x54\x92" + "\xbb\x09\x1d\x4d\x2d\x77\x5a\xf2\x17\x8a\x4b\xcc\x1c\xa2\x1d\xcf\x8b" + "\x3b\xf8\xf0\x56\x82\x37\x70\x78\x2f\x25\xa4\x19\xbb\x3e", + 1, 16, NULL, 0 }, + { 384, 384, 4, + "\x29\x28\xd4\x65\xd9\x2f\xa4\x00\x72\xca\x9d\x67\x76\x1b\xe6\x6e\x49" + "\x17\x55\xe4\x34\x99\x00\x3c\x10\x57\xd3\xbe\xc8\x70\xf2\x55\x12\x6c" + "\x36\x58\xd0\xd8\xa0\xc7\xd2\x07\xdf\x87\x10\x03\x7c\xa7", + "\xc4\x05\xae", + "\xd9\xe1\x9c\x67\x2a\x46\x6e\x4c\x83\xa8\x49\x90\x57\x28\xc4\xbe\x1d" + "\xb9\x9b\xdd\x26\x09\x46\xd9\xff\x52\x93\x97\x79\x00\x2d\xcc\x46\x0c" + "\x57\x6f\x02\xb4\x0d\xda\x07\x17\x18\x2b\xe9\x6b\x54\x11", + 1, 24, NULL, 0 }, + { 384, 384, 5, + "\x68\x6a\x37\x30\x08\x5c\xc9\x44\xfc\xeb\x14\x16\x28\x41\x98\x18\xe6" + "\x62\xfe\x21\xe5\x2b\xea\x27\x48\xf3\xb7\x04\xf8\x0c\xe8\x01\x08\x6d" + "\xb1\xe3\x06\x89\x17\xb2\x42\xe6\x2b\x4d\x6e\x6e\xd6\x85", + "\x66\x01\xc6\x83", + "\x10\xdc\x39\x10\x39\x83\xb3\xa6\xbe\x37\x6a\x8e\xda\x7b\x6f\x36\x3c" + "\xb9\x1e\xfe\x11\xb0\x27\xa6\x24\x40\xae\x13\x6b\xd6\x6f\x98\xb0\xa1" + "\xd8\xb8\xf2\x39\x90\x99\x49\x20\x21\x07\x6a\xfa\x14\xa0", + 1, 32, NULL, 0 }, + { 384, 384, 6, + "\xf2\x2d\x86\x7b\x97\x2b\x23\x2e\x3f\x44\x4a\x48\x8d\xd7\x94\xd1\x70" + "\x80\x7c\x70\xeb\x65\x0f\x95\x2b\x61\x77\x59\x6f\x76\xc5\x58\xa5\xd8" + "\x60\xd6\xf7\xbe\x0b\xe9\xe6\x66\xf9\xbd\x53\x73\x2f\x8d", + "\x15\xb2\x93\x77\xe0", + "\xe0\x2e\x4e\x20\xb5\xf1\xe5\xf0\x69\x13\xbc\x97\x45\xc9\x06\x9c\x09" + "\xec\x13\x69\xf1\xa2\x96\xad\x1d\x07\xc0\x4c\xc4\xf9\xcb\x47\x41\x24" + "\x8d\x7b\xa0\x97\xcd\x3b\xa0\xe7\x5d\x24\x09\xd6\xa0\x1b", + 1, 40, NULL, 0 }, + { 384, 384, 7, + "\x3a\xc9\xab\xd5\x3d\xbd\x0f\xbb\x89\x1f\x9b\x5e\x16\xdd\x45\xdf\x99" + "\x4e\x52\x83\x52\x78\x32\x70\x71\x38\xfc\x27\x12\xba\xd9\xe3\x47\x61" + "\xe7\xd9\xc6\xd0\x5d\x46\xf2\xc8\x32\x3d\xdb\x0e\xfe\x99", + "\x5a\x34\x15\x5b\x11\x15", + "\x78\xc5\x3d\xd1\xa2\x43\x11\x74\x62\x8f\x5f\x48\x67\xfa\x77\x7a\xfa" + "\x6d\xf1\xb3\x62\x69\xbb\xa1\x14\xd0\x16\xd1\x06\x5f\xcb\x02\x11\x70" + "\xba\xad\x09\xb4\xa5\x28\xf4\x05\x73\x90\x3a\x65\xf5\x40", + 1, 48, NULL, 0 }, + { 384, 384, 8, + "\xae\x3a\xa9\x4f\xdd\x35\xe2\xbe\xf4\x04\x72\xd2\x9b\xda\xd3\xa4\x09" + "\x84\x0e\xa4\x41\xc3\xd7\x02\x5c\xd7\x2f\x3e\x81\xff\x56\xda\x60\x21" + "\x61\xd8\x4b\x23\xd1\x63\x40\x61\x38\x5b\xe3\x0c\x5b\xbd", + "\x8a\x14\x0d\x78\x1e\x71\x91", + "\xfd\x22\xba\x89\x6c\xb1\x14\x7b\xb8\x6f\x8a\xd5\x1c\x25\x3b\x79\x26" + "\x57\xc0\xbe\xcc\x91\x3e\x90\x10\x4d\xa0\xf1\x39\xf9\xb0\x8c\x91\x69" + "\x70\x6f\x15\x31\xa2\xc6\xc0\x3d\x6b\xd7\x2a\x77\xef\xf2", + 1, 56, NULL, 0 }, + { 384, 384, 9, + "\x44\xb7\x98\x52\xca\xbc\xf3\xfe\x93\xd2\xff\xf5\x5d\x2a\xfe\x6a\x46" + "\xc3\x5b\x7a\xd1\x95\x4c\xe0\x88\x8d\xe7\xb4\x59\xb9\x82\x72\x2f\xaf" + "\x8b\x49\x0e\x6b\x00\xe7\xbc\xab\xbd\x36\xf1\x84\x43\xf5", + "\x93\x98\xcd\x25\x1d\xea\xfe\x8b", + "\x56\x12\x8f\xb4\x38\xa9\x3f\x6f\x48\xf4\x7c\x0f\x4c\x75\x49\xf8\x00" + "\x8a\x8e\x69\xbb\xdb\xf0\x88\x6e\xc4\x0f\x86\xe7\x87\x00\x34\xef\x90" + "\x90\xd2\xb0\x40\x57\x39\x1f\x1d\xef\x5b\x25\xe8\xf0\xad", + 1, 64, NULL, 0 }, + { 384, 384, 10, + "\x03\xfe\xd2\xf5\x79\xa3\xeb\xde\xce\xcf\xb1\x84\xeb\xe2\x98\x48\x76" + "\x11\x33\x99\xc4\xa5\x93\xd9\x8b\x5f\x5e\x60\x6d\xd3\x30\xfb\x39\x4c" + "\x28\x5d\x9e\xad\x60\x17\x48\x25\x9b\x49\x33\x35\xf8\xe5", + "\x18\xd8\x79\xb1\xf6\x3d\xf3\xac\x7a", + "\xa0\xe3\xb5\x66\x0e\xeb\x5f\xc4\xa5\xdd\x48\xe7\x25\xb0\x9a\x0e\x28" + "\x2b\x22\xbb\xe2\x69\x3d\x8b\x89\x3d\xdf\x0f\x21\x16\x45\x0e\x08\x75" + "\x92\x54\x07\xe9\x09\xfd\xe0\xf1\xf7\x28\xf6\x08\xfb\xa9", + 1, 72, NULL, 0 }, + { 384, 384, 11, + "\xf4\xef\x48\xbf\x40\x56\xd3\x9d\xbb\xa4\x15\x40\x18\xc6\x3b\xdf\x29" + "\x42\x0b\x99\x91\xea\x59\x4f\xf0\x5e\x3c\xc1\xcb\x02\xe1\x76\xd5\x4b" + "\xa0\x38\xa6\xb7\x86\x92\x51\x9d\x67\x88\xe4\x95\xbb\xab", + "\x0a\x5d\xe1\x3c\xd9\xba\x31\xc9\x44\x86", + "\xe9\xa1\x21\x9e\x86\x98\x3d\x69\xe3\x36\x06\x8b\x28\x03\x09\xf9\x74" + "\xab\x61\xf2\x59\x68\xfc\x63\x52\x32\x4b\xa4\x9c\x36\xce\x42\xc5\x78" + "\x67\x6a\x3a\x31\xef\x11\xe9\x60\xd6\x77\x13\x86\x65\x0e", + 1, 80, NULL, 0 }, + { 384, 384, 12, + "\xfc\x77\x1f\x7c\xcd\x49\x9a\x1e\xd6\x33\xd8\x68\x76\xd7\x07\xb5\xf1" + "\xd5\x3c\x6b\xcd\xf2\x1a\xa2\x90\x77\x66\xab\x3c\xa7\xfa\x6c\xdd\x6a" + "\x9b\x98\x1b\x1a\x84\xa5\x28\xe8\x14\x44\x30\x3f\x10\x57", + "\x03\xba\x11\xf3\xf3\x17\x3b\x85\x22\x6b\x25", + "\xcf\xb4\x97\x1d\x54\x49\xdb\x36\x4e\x2c\x8d\x0d\x42\x9a\x07\x67\x05" + "\x0d\x48\x0a\x53\x97\xf0\xdc\xc7\x42\x94\xf5\x2e\xa9\x62\x60\xa5\x7f" + "\xe6\xca\xd1\x44\x09\xad\x67\xda\x6f\xbe\xbf\x2d\xa0\xd8", + 1, 88, NULL, 0 }, + { 384, 384, 13, + "\xb3\x99\x9d\xe6\x80\xb1\x15\x50\xe1\x86\x31\xc8\x19\x9f\x7e\xb8\xa7" + "\x4e\x21\xbd\xc9\xd9\x7f\x78\x12\x45\xc2\xaf\x19\xf8\x54\x97\xd9\xf3" + "\x8b\x25\x0a\x56\x4e\x48\x65\x0f\xd0\x0b\xe3\x65\xf1\x55", + "\x9c\x65\x8c\xb5\xe6\x01\xd8\x5d\xc3\x85\x78\x63", + "\xd5\x47\xe4\xcb\xd5\x6e\x82\xb4\x7d\x2e\xc9\x3e\xeb\x6b\x34\x92\x4e" + "\xbd\xa4\x61\xfb\x60\xe4\x75\xbf\x32\x8d\x23\x68\x61\x8f\x55\xfb\xf7" + "\xb0\xe2\xeb\x1f\xf5\x42\xc4\xeb\x7e\xef\xbf\xc8\xbd\x2b", + 1, 96, NULL, 0 }, + { 384, 384, 14, + "\x88\x00\x5a\x62\x86\x4e\xa6\x99\xe1\x50\x96\x16\xec\x48\x03\x3e\x84" + "\xd2\xe2\xa1\x3b\x8b\xc2\xe8\xa7\x6f\x2e\xcc\xbd\xb2\x07\xa9\x5a\xc8" + "\xe2\xf5\xb5\xa7\x03\xb2\x2a\x0b\x57\x1e\x8a\xcc\x59\x9a", + "\x5a\x94\xf8\x45\x41\xa7\x94\xbf\x23\xd7\x2d\xb1\x6d", + "\xd6\xb7\x3e\xe6\x7e\x88\xa2\x0f\xce\xb5\x52\x0b\xe9\x25\x94\xda\xf1" + "\xb3\x78\x6c\x71\x87\x53\x5c\xcb\x1f\x0b\x92\x6d\xae\x11\xad\xde\x6e" + "\x86\x97\xba\x80\x3b\x15\x90\x19\x84\x9d\xf3\xc9\xd2\xc7", + 1, 104, NULL, 0 }, + { 384, 384, 15, + "\xb1\xcb\xda\x2c\x9a\x12\xf9\x23\x15\xa5\x10\x1a\xef\x31\x1e\x99\xd6" + "\xdb\x00\x2b\x0e\x04\xfb\x53\xc5\x01\x06\xaa\x4d\x28\xe9\xa3\x46\x69" + "\x7b\xa9\x70\x84\x57\x2e\xea\x56\xcc\xfc\x4a\xd7\xe5\x72", + "\xce\x12\xc0\xc7\x8e\x3f\x6b\x27\x6a\xc5\x6e\xd7\x43\x5e", + "\x5c\x08\x02\xcd\x0e\xd8\x23\x80\xe4\xc2\xa6\x1d\x14\x6e\xd7\x27\x62" + "\x61\x3d\xe8\x9e\xb4\xab\x9f\xe7\x1d\xa9\xad\x3d\x79\xe1\xd2\x32\x1c" + "\xae\x18\x62\x92\xf7\xc5\x2a\xb6\x39\xd3\xba\x6a\xa8\x5a", + 1, 112, NULL, 0 }, + { 384, 384, 16, + "\x08\x51\x7e\x80\x14\xe0\x0d\xb5\xc3\x7f\x2a\x20\xf9\x87\xea\x2e\xc5" + "\x2e\x79\x38\xde\x01\x8a\xd6\xbe\x25\x6b\xa2\x23\x68\x04\x14\x4a\xd2" + "\xa1\xbc\xc2\x42\x73\x88\x62\xb4\x06\x47\x00\x7e\x0a\x2c", + "\x21\xe2\xa0\xa1\x67\x78\x9a\x6b\x72\x2d\x17\x37\xd9\x2f\x8b", + "\x22\x64\xd3\xc9\xb8\x35\xae\xdf\x69\x9d\x5f\xbf\xc0\x5d\x46\xf0\x85" + "\x59\x14\x41\xdf\x75\xaa\x2b\x28\x73\xf6\xc8\xa1\x1a\x08\x56\xa2\xb7" + "\x9a\xe1\x1e\xa0\xa9\x16\x09\xdb\xd5\x64\xa0\xbe\xd4\x56", + 1, 120, NULL, 0 }, + { 384, 384, 17, + "\x50\x3d\x74\x78\xa7\x73\xb6\x94\xd6\xe5\x52\xc9\x70\x3c\xc8\xbc\x56" + "\xfd\x49\xfa\xfc\x9a\x17\xca\xb8\xb0\x33\x2d\xca\x8d\x49\x33\x6f\xa7" + "\xe9\xec\x2b\xcb\x56\x25\x3f\xe5\xbb\x50\x4e\x3e\x7f\x7f", + "\xd9\x6e\x6f\xed\x89\x3a\xdd\xfd\x92\x37\xc8\x1c\x4f\x4e\x34\x1b", + "\x19\x38\x97\x66\x78\x99\x12\x26\x0f\x3f\x97\x57\xdf\x36\x51\x66\x38" + "\x29\xc3\x58\xbb\x48\xb2\x2c\x1c\x63\x13\x20\x70\xdf\x31\x89\x05\xbe" + "\xff\xd4\x5f\x51\xe4\xdf\xcb\x3e\x78\x5f\x44\xcf\x91\x06", + 1, 128, NULL, 0 }, + { 384, 384, 18, + "\x41\x34\x1b\xab\x90\x2e\x76\x7d\x4d\x19\x64\xc0\xac\xfe\xcf\x46\xef" + "\xf1\xb0\x2b\x64\x55\xbc\xb2\x09\x7d\xe9\xc1\x54\xbe\x1f\x66\x7f\x21" + "\xbe\x07\x6d\xe1\x8c\xd2\xc1\x5c\x00\x58\x96\xfc\xa8\x7f", + "\x4c\x43\xac\x7d\xe3\x63\x1c\xc8\x6f\x4d\xa7\x2f\xe6\xb6\xa5\x52" + "\xf1", + "\x3c\x31\x04\xf2\x4b\x70\x70\xcc\x32\x77\xd9\xae\x64\x0d\x41\x62\x98" + "\xfc\x91\x7a\x0c\x1c\xdc\x3c\x2e\x7b\x6d\xa7\x57\x06\xfd\x2a\xe2\x34" + "\xef\xd5\x51\xaf\x12\xae\x29\x14\x47\x04\x79\x3e\x2f\x6a", + 1, 136, NULL, 0 }, + { 384, 384, 19, + "\xc2\xf8\x3b\xe1\xac\xce\x7b\x89\xa5\xf9\xe9\xea\x7e\x4c\x4f\x8b\x0f" + "\x43\x19\x98\x6f\xbe\x47\x9f\xa3\xb4\xa3\xc2\x98\x16\x83\x62\x39\x3b" + "\x56\xea\x03\xb5\xce\xf7\x7f\x48\xe5\xa7\x2a\xbe\x6d\x08", + "\x8d\xd0\xcd\x78\x6c\xd8\x00\xff\xeb\xec\x09\x87\x28\x92\x3d\x69\x24" + "\x9d\x32\x23\xc4\xc5\x95\xcb", + "\x75\x1c\x6c\x7d\x00\xfe\xf5\xe4\xed\xc9\x93\x91\x5f\xba\x69\x49\x43" + "\xa7\xee\x3a\x2c\x8e\x5b\x70\x0d\x0e\xe5\x36\xbf\x85\xfb\x11\x7a\x9c" + "\xd6\xc4\x56\x48\x5c\xd6\x70\xf7\xa0\xb4\x90\xc8\x3e\x61", + 1, 192, NULL, 0 }, + { 384, 384, 20, + "\x6b\xd2\xae\xe9\xdd\x98\xd6\xb6\x60\x9f\xce\x82\x18\x1b\x10\xc2\x0b" + "\xba\x86\x1d\xa6\x8a\x15\x90\x58\x6f\xab\x08\xc5\xe9\xe9\x0f\xf5\x84" + "\x04\x7d\xb4\x76\x08\x28\x64\x3f\xea\x38\x08\x71\x60\xe4", + "\x33\x23\x6a\x9d\xe6\x03\xc1\xe4\xf5\xe1\x11\x64\x22\x47\x40\x62\x7d" + "\x10\xf6\x00\x8e\xb7\x3e\xc2\x64\x23\x21\xbf\x0b\x82\xd5\x79", + "\xe4\xcd\x8b\x88\x68\xbb\x07\x8e\xd5\xd6\x93\x8e\x40\xd9\xff\x4b\xf6" + "\x1a\x49\x94\xbe\x40\xa5\xf2\xb5\x44\x64\x63\xe5\xdb\x90\x51\x6b\xcc" + "\xdd\x19\xf1\x6c\x92\xe3\xf8\x39\xb9\xd6\xde\x68\xb2\xa9", + 1, 256, NULL, 0 }, + { 384, 384, 21, + "\x2f\x98\xba\x2c\xea\xad\xc5\xba\x08\x88\x0a\x35\xcb\x00\x80\xdc\x87" + "\x0a\x57\x34\xa7\x82\xeb\xe3\x1c\x4b\xab\x10\x0f\xf8\x78\x6d\xcc\x3b" + "\xe6\xde\x18\x48\x2e\xa5\xd1\xb3\xbf\x14\xae\xab\xb4\x70", + "\x2d\x74\xa6\x6d\xac\xf1\x2e\xdb\x85\xef\x30\x73\xfe\xaf\xd1\x22\x88" + "\x9c\xb6\x34\xad\xd0\x0f\xf0\x39\x5d\x22\x4b\x4f\xf8\xb5\xd5\xd6\x7c" + "\xa6\x41\x9b\x68\x26\xab\xff\xdb\x41\xba\xb4\x27\xd5", + "\xa8\xea\x72\x10\x08\x59\xf4\xb7\xb6\xf2\xfe\x59\x62\x48\xf1\x72\x9b" + "\xcd\xf0\x60\x6c\x90\x0a\xb5\x2e\x51\xea\xb5\x48\xd2\x6e\x1e\xb6\x34" + "\xa4\x2e\x5f\xc7\xcc\xc1\x83\x56\xc0\xd2\x83\x59\x7e\xe2", + 1, 376, NULL, 0 }, + { 384, 384, 22, + "\x5e\x5f\x60\xe4\x0d\x84\xc7\xca\x26\x08\xaf\x3b\xcc\x6e\x04\xab\xc5" + "\xf8\xb7\xca\x73\x0a\x78\xaf\x7f\x6f\x03\x2e\x5a\x15\x01\x69\x5b\xd9" + "\x1f\x3b\xeb\xb2\x85\x90\xaf\x1d\xb9\x0d\x83\x90\xca\x58", + "\x2e\xfe\x6a\x14\xea\x8d\x67\x9e\x62\xdb\xce\xdf\x35\xe6\x18\x52\x27" + "\x8c\x83\xc5\x4a\xdb\xe1\xf1\xc7\x2c\xb1\xa7\x46\xb1\x1c\xff\x8c\xb4" + "\xfc\x3a\x2c\x3a\xcd\x44\x25\x5d\x51\xc0\x20\xca\x6d\x47", + "\x6e\x8c\x95\xa4\x09\x7e\xa1\x3d\x06\x4e\xd1\x08\x09\xa3\x3b\x56\x9a" + "\x6a\x84\x20\x51\x58\xbd\x69\x2f\xf8\x2b\xc4\xb7\x0b\x47\xa6\x0e\xd3" + "\x32\xf2\xf5\xbc\xa5\x21\x1a\x1c\xc8\x9c\x06\xf9\xc5\x95", + 1, 384, NULL, 0 }, + { 384, 384, 23, + "\xbc\x31\x0b\xc3\x91\x3d\x9f\xe5\x9e\x20\x12\xa0\x58\xc9\xe1\x50\x53" + "\x4d\x25\x61\x1e\x36\x20\x6c\xf0\x7c\xca\xef\xe1\x53\xf3\x8e\xb0\xea" + "\xad\x99\x41\xb6\x88\x3d\xfb\xce\x01\xbc\xb5\x19\x60\x41", + "\x9f\x07\x47\xd7\x39\x6b\xfb\xe0\x1c\xf3\xe8\x53\x61\xe5\x00\x85\xe0" + "\xa9\x1a\x74\x90\xb9\x94\x03\x1d\x81\x85\x1b\x72\x50\x65\x99\x3f\x45" + "\xda\xd0\xd6\x0d\x79\x4a\xed\xec\x7b\xa5\xd9\xd6\xdb\xbe\xe4", + "\x3a\x86\x49\x8f\x78\xc3\xfb\x7e\xb3\xb7\xb3\xd8\x2f\x67\x7d\x2d\xfe" + "\x01\x16\x6f\xe7\x6e\x23\x20\x83\x33\x4d\x74\xf1\x15\x88\xfd\x08\x96" + "\x37\xc9\x47\x61\xe9\xcf\xe8\x36\x43\x60\x05\xde\xae\xf7", + 1, 392, NULL, 0 }, + { 384, 384, 24, + "\xdc\x77\x0c\x64\xd0\x0d\x15\x6e\x43\xcb\x74\x97\x0e\x3a\x1a\x2a\xd2" + "\x8b\x6d\x9e\xc6\xb2\xb6\xe5\xac\x3e\x35\x6a\x99\xf8\x79\xcb\x62\x0f" + "\x00\x34\x0c\x04\x4c\xc1\xf3\x1b\xdc\xcf\xa0\xdb\xd1\x77", + "\x40\x3f\xd8\xe3\xef\x51\xb6\x53\x9d\xb6\x58\xa8\x94\xbe\x85\xb5\x8f" + "\xbc\x84\x88\x1e\x61\xc5\xe0\xcb\x13\xae\x42\x1a\x09\xd3\x1d\x78\x06" + "\x03\x25\x6d\x39\x0e\xdd\x05\x6d\x19\x08\x56\xbe\x00\xad\x20\xa7\x04" + "\x8f\x0c\x67\x41\x6f\xe8\xe0\x28\x84\x08\x61\x55\xf4\x26\x32\x62\xe8" + "\xc1\x27\x55\x04\xd4\xf9\x1f\x27\x51\xd3\xc3\xdc\xcd\x44\x09\xff\x2b" + "\x45\xe4\x1d\xe9\x3f\x7b\x10\x4d\x58\xf6\xe1\x5b\xac\xb6\x2a\xce\x97" + "\x00\x61\x5e\xcc\x1b\x30\xa0\xcc\x1b\x35", + "\x1c\x4f\x64\x74\xf3\x9e\x6e\xab\xbe\x7a\x99\xfa\xa2\x34\xf4\x98\x33" + "\x44\x41\x30\xac\xf0\x1d\xae\x68\xd6\x82\x51\xa9\x30\x41\x99\x60\xb0" + "\xfb\x5f\x48\x36\x01\x49\xe0\x5d\x12\x09\x94\x1c\xc9\xec", + 1, 896, NULL, 0 }, + { 384, 384, 25, + "\xcc\xa9\x29\x9c\x7b\xdc\x26\xa4\xb5\x95\x05\x5c\x99\xca\x23\xbe\xc8" + "\xed\x11\xb5\xde\xed\xa9\x1f\x83\xe2\x36\x5e\x73\x40\x39\x5c\xee\xf4" + "\xe8\x6e\x5c\xd9\x1f\x25\x93\xbc\xfe\xc4\x98\xa6\x7f\xc9", + "\xa0\x5b\x40\xb8\xd3\xa7\xbc\x7b\x75\xb0\xe9\x73\x09\xc9\xbd\x1c\x9d" + "\x87\x55\xc1\xff\x52\x45\xef\x63\x08\xa6\xa5\xca\xd3\xec\xfb\xcb\x63" + "\x64\xb4\x1c\xa6\xf3\xd2\x4b\xbe\xe8\x44\xd6\x20\x4d\x10\x26\xab\xe3" + "\x45\xaf\x7b\xde\xc1\x14\xa3\x73\xb1\x09\xaa\x57\x24\xb7\x38\xd5\x0a" + "\xb7\xa8\x26\xc2\x68\xe8\x73\x70\x9f\x8b\x35\x13\x5a\x87\x00\x45\xd5" + "\xfb\x9d\xaa\x82\xd3\xc2\x45\xb5\x33\x89\x17\x35\x4e\x72\xb3\x05\x8c" + "\x9a\x4b\x80\x71\x17\x46\x52\x17\xd7\xd1\x4f\x36\xf8\xa8\xd4\xe9\x7b" + "\xc3\xb9\x35\x87\xc9\x26\x41\xe7", + "\x1b\x6b\x5b\xa8\x48\xbc\x13\xdd\x46\xc3\x51\x77\xae\x9f\xf9\xbd\x2d" + "\x6c\xa5\xf4\xc9\x37\x39\x64\xd3\x18\x24\x83\xd9\x80\xb4\x65\x45\x27" + "\xf3\x6d\x7c\xc5\x1b\x9e\x2e\xfe\x7e\xd9\x7a\x82\xe3\xbe", + 1, 1016, NULL, 0 }, + { 384, 384, 26, + "\xc7\x28\xe6\x5e\x08\xd9\x29\x6f\xe3\xcd\xf2\xde\xdb\x49\xc8\x1a\x30" + "\xb6\x03\xa6\x25\x69\xee\xce\x4e\xe5\xd0\x1e\x9a\x32\xae\x3b\xcb\x4e" + "\xc1\x63\xe4\x55\xe4\x52\x58\x24\x54\xce\xef\xef\xc0\x46", + "\xe6\xc6\xba\xc8\x7c\x17\xe2\x69\xa4\x71\x43\x4c\xa9\x56\x84\x01\x45" + "\x1d\x78\xc2\x44\x4a\x9d\x6e\xdc\xda\x3c\xda\xb5\x1c\x5b\xed\x1c\x19" + "\xea\xf3\x43\x26\x58\x0f\xd8\x5a\xe5\x23\x6a\xd5\x1b\xc5\xda\xe3\x86" + "\xb3\x61\x01\xf5\x46\x95\xc5\x95\xee\xed\xcd\xd0\x18\x2a\x4a\x11\x7f" + "\x80\x93\xf4\xf4\x81\x2e\x03\xdb\x39\x6e\xde\x98\x49\xd1\x93\xe7\x72" + "\x20\x81\xae\xec\x4b\xe6\xc4\xca\xf6\xc9\x79\xd3\x6e\xad\x56\x63\x4a" + "\x21\xbe\x21\x16\x2e\xa2\x32\xde\xc9\xcf\xfd\xbd\x24\x74\x24\x58\x78" + "\xdc\xa3\x69\xe8\x14\xfd\x02\x83\x03", + "\x53\x39\x20\xa0\x13\xcf\x00\x6a\xa2\x9b\x26\xf7\x4b\x6d\xd2\x93\x63" + "\x42\x93\x08\x99\x86\xaa\x24\x92\x71\xc4\x26\xb9\x42\xdc\x6b\xae\x32" + "\xb2\x64\x16\x16\x67\x2f\x3d\x75\x96\x88\x66\xe1\x82\xe5", + 1, 1024, NULL, 0 }, + { 384, 384, 27, + "\x90\xc4\x21\x5d\xc3\xf2\x37\x43\x50\x47\xfe\xfd\xd8\x63\x8d\x33\x9a" + "\x3f\xc6\x6f\xca\x06\xc5\x06\x3e\xac\xbd\xa0\x02\xab\x33\x5e\x62\x16" + "\x05\xf6\x72\xf3\xda\x9f\x64\x1f\xae\x11\x0a\xfc\x3e\x7b", + "\x1e\xbc\x22\xc3\x03\x1b\x64\x61\x5e\xb6\xf1\xa0\x69\x6e\x33\xb7\xdf" + "\x13\x9a\x4b\x89\x1d\x3e\x67\x21\xcc\x26\xc0\x5d\x55\xde\x79\x0d\xca" + "\x62\x36\x68\xc1\x03\x08\x48\x5d\x38\xe9\x5e\xc4\x76\x9f\xa4\x43\x0c" + "\xa3\xeb\xc2\x5d\xa9\xf5\xd3\x1c\x97\x26\x74\x51\x7d\x9a\x22\x22\xe6" + "\xb9\x7d\x8d\xef\x65\x12\xaf\x09\x6c\x6d\x14\x80\xd8\x3a\x22\x9c\x84" + "\xb7\xf2\x8c\x80\x18\x4b\x6b\xeb\xf3\xf4\xef\xf5\xfc\x4e\x5c\x6c\xfe" + "\xa4\xf8\xeb\xa9\xa9\x57\xf7\x91\x3b\x20\xa8\x8a\xd1\x73\x4f\x7c\x38" + "\x54\x7e\x93\x4d\x1d\xbf\x2d\x73\xdb\xd6\x1e\x31\xfb\x15\x83\xc7\xb6" + "\x57\x7a\x17\x1e\x7d\x02\xf1\x90\x45\x12\x6a\xc2\x97\x3d\x85\x5b\xc1" + "\x8d\x34\xd3\x23\x26\xd1\xe2\x16\xda\x58\x36\x6a\x60\x03\x34\x50\x09" + "\x11\x28\xae\x26\xa4\x79\x06\x9b\xba\x7b\x91\xb2\xab\x7f\x3c\x5f\xbc" + "\xde\x39\x1d\xe3\xca\x11\x4b\x95\x1d\x68\x52\xf9\x27\x95\xf8\x02\x3d" + "\x7a\x29\xa7\xf4\xce\x61\xe9\x24\x1b\x4f\x23\x5d\x21\xe8\x99\x08\x71" + "\x67\xab\x3f\x3a\x0e\x93\x21\xc7\x94\x2b\x16\x51\x78\x78\x8d\xf4\x8d" + "\x3b\x10\x6b\x20\x3e\xc1\xe0\x1d\x29\xbd\xa4\x1a\x99\xac\x0d\x2c" + "\x00", + "\xc5\x2b\x91\xda\xed\x6e\xe4\x64\x16\xf2\xdb\x78\x97\x82\x51\xcb\x33" + "\x4e\x5d\x8e\x00\xb3\x2a\xe0\x6e\x36\x5f\x45\x5d\x28\xde\x40\x6a\x9c" + "\xce\x2f\x9f\x29\x37\x8f\x22\x98\x22\xdb\xf2\x6b\xfd\xad", + 1, 2040, NULL, 0 }, + { 384, 384, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x46\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xab\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xc4\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x29\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbf\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4c\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\x44\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\x7f\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x14\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x91\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x17\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x92\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7b\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x65\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\xfa\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\xe4\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x42\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\x90\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa5\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x37\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x89\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd6\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x8a\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd5\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x08\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\x57\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x67", + 0, 0, NULL, 0 }, + { 384, 384, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x17", + 0, 128, NULL, 0 }, + { 384, 384, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x64", + 0, 0, NULL, 0 }, + { 384, 384, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x14", + 0, 128, NULL, 0 }, + { 384, 384, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x26", + 0, 0, NULL, 0 }, + { 384, 384, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x56", + 0, 128, NULL, 0 }, + { 384, 384, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\xe6", + 0, 0, NULL, 0 }, + { 384, 384, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x96", + 0, 128, NULL, 0 }, + { 384, 384, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbe\x81\xc4\x15\xd2\x83\xab\x7b\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4d\x07\xff\x90\xb3\x38\xe0\x65\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\x44\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\x7f\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\x7b\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c\x92\xc1\xfc\x36\xf1\x98\xc0\xb3\xa7\x14" + "\x92\x18\x48\xd5\xe0\x3d\xf1\xc4\x84\x9b\xb8\x31\x0c\x66", + 0, 0, NULL, 0 }, + { 384, 384, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x82\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15\xdd\xe7\x66\x74\x89\x64\x00\xf9\x7b\x84" + "\x08\xbf\xef\xa6\xee\x86\xc7\x16\xbf\xa4\xa4\x60\xd2\x16", + 0, 128, NULL, 0 }, + { 384, 384, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xbb\x41\x7e\x3b\xea\x2d\x7c\x54\x85\x9d\x5b\xae\x77\x1a\x25\x04\x34" + "\x68\x25\x9f\x94\x2a\x4e\x93\x6d\x3e\x03\xc9\x0e\x67\x3f\x4c\x58\xeb" + "\x6d\xe7\xb7\x2a\x1f\xc2\x0e\x3b\x7b\x64\x47\xce\xf3\x99", + 0, 0, NULL, 0 }, + { 384, 384, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x56\xb2\xf8\x00\x6f\x4c\xc7\x1f\x9b\x4f\xc9\xfc\x28\x94\x30\xfd\xeb" + "\x4e\x04\x77\x39\x94\x6b\xea\x22\x18\x99\x8b\x76\x9b\xff\x06\x84\x7b" + "\xf7\x40\x10\x59\x11\x79\x38\xe9\x40\x5b\x5b\x9f\x2d\xe9", + 0, 128, NULL, 0 }, + { 384, 384, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 384, 384, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 384, 384, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 384, 384, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 384, 384, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xc4\x3e\x01\x44\x95\x52\x03\x2b\xfa\xe2\x24\xd1\x08\x65\x5a\x7b\x4b" + "\x17\x5a\xe0\xeb\x55\x31\xec\x12\x41\x7c\xb6\x71\x18\x40\x33\x27\x94" + "\x12\x98\xc8\x55\x60\xbd\x71\x44\x04\x1b\x38\xb1\x8c\xe6", + 0, 0, NULL, 0 }, + { 384, 384, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x29\xcd\x87\x7f\x10\x33\xb8\x60\xe4\x30\xb6\x83\x57\xeb\x4f\x82\x94" + "\x31\x7b\x08\x46\xeb\x14\x95\x5d\x67\xe6\xf4\x09\xe4\x80\x79\xfb\x04" + "\x88\x3f\x6f\x26\x6e\x06\x47\x96\x3f\x24\x24\xe0\x52\x96", + 0, 128, NULL, 0 }, + { 384, 384, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbf\x80\xc5\x14\xd3\x82\xaa\x7b\x63\xa5\x50\x89\xe4\xdb\xfa\xca" + "\x96\xdb\x61\x6a\xd4\xb0\x6d\x93\xc0\xfd\x37\xf0\x99\xc1\xb2\xa6\x15" + "\x93\x19\x49\xd4\xe1\x3c\xf0\xc5\x85\x9a\xb9\x30\x0d\x67", + 0, 0, NULL, 0 }, + { 384, 384, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4c\x06\xfe\x91\xb2\x39\xe1\x65\xb1\x37\x02\xd6\x6a\xce\x03\x15" + "\xb0\xfa\x89\xc7\x6a\x95\x14\xdc\xe6\x67\x75\x88\x65\x01\xf8\x7a\x85" + "\x09\xbe\xee\xa7\xef\x87\xc6\x17\xbe\xa5\xa5\x61\xd3\x17", + 0, 128, NULL, 0 }, + { 384, 192, 82, + "\x1c\x67\x82\x67\xbe\x13\xac\xb4\x64\x93\x9c\x28\x96\xc9\xe9\xce\x1d" + "\xeb\x5b\x30\x83\x3b\xdd\x9c\xa0\x03\x70\x88\x9b\x84\x41\x07\x82\xad" + "\x52\xaf\xe2\x5d\xc1\x0a\xb7\xec\x5c\xf5\xf3\x47\x93\xb7", + "", + "\x6d\xd5\x66\xbe\x67\x8c\x1e\x63\x59\xab\x31\xb6\x35\xcc\x16\x01\x60" + "\xa0\xc5\xa9\xc4\x9a\x0a\xc5", + 1, 0, NULL, 0 }, + { 384, 192, 83, + "\x00\xb1\x84\xc2\xc0\xa4\x91\xd7\x64\xa2\x6f\x8b\x2e\x56\xa9\x65\x22" + "\x2b\x36\x21\x3b\xdd\x10\x6a\xe7\x82\x30\x5c\x50\xf8\x92\x69\x90\x24" + "\x76\xe5\xdf\x3f\xa5\x8e\x0e\xcf\xae\x82\xa9\x60\x7c\x8e", + "\x9f", + "\x5a\xff\xf4\xb0\x09\xca\x9c\x9e\x5d\xcd\x84\xf0\x56\x07\xe7\xa7\xd4" + "\x3e\xe4\x3b\x42\x49\x89\x89", + 1, 8, NULL, 0 }, + { 384, 192, 84, + "\x05\x5b\x67\xed\xb6\x59\xe2\x9c\x10\xe3\xe9\xcd\x25\xaa\x1c\xd5\xab" + "\xf0\x88\x0e\x20\x26\xed\x84\x36\xe3\x9b\x06\x4b\x73\x15\x76\x0c\xd7" + "\xa9\x29\x4e\xe2\x3d\x47\x50\x96\x9c\xc8\xb5\xdb\xae\xd7", + "\x40\x47", + "\x4d\x08\xba\xef\x96\x9e\xed\x23\xb8\x14\x47\x2a\xcf\xf0\x8d\x08\xfd" + "\x34\x91\xa7\x28\x77\x8a\x1c", + 1, 16, NULL, 0 }, + { 384, 192, 85, + "\x9e\x3c\x19\x8e\x93\x93\x0f\x07\x6b\x03\x5c\x5f\xa8\xf1\x0d\x9a\x65" + "\xe9\x8c\x66\xcf\xb3\x66\x33\xe3\xcb\x33\x27\x9c\xdf\x57\x68\x8f\x10" + "\xb7\x47\x2d\x1f\xc9\xd9\x62\xce\x69\x54\x51\x9b\xfb\xf6", + "\x88\xcf\xab", + "\x1c\xde\x37\x65\xba\x5a\x15\xb1\xd0\x18\x21\x36\xa7\x2c\x60\x3a\xcd" + "\x3b\x90\x4c\xea\xc8\xf7\xad", + 1, 24, NULL, 0 }, + { 384, 192, 86, + "\xf5\xf5\x96\x2b\xda\x25\x7b\x38\xb2\xa2\x31\x89\x29\x12\x1b\x2e\xae" + "\xf7\x92\xd5\xc6\xa9\x58\x5e\x48\xb8\x0c\xf5\x35\x7b\x29\xc3\x95\x1b" + "\x78\x7e\xd3\xe0\x3e\x38\x5b\x05\xb8\xff\xe6\x86\x1d\xc3", + "\xd9\x39\x77\x53", + "\x46\x38\xe4\x42\x7e\x60\x84\xb7\x6c\x53\xed\x9d\x6e\x91\x61\x62\xfc" + "\xb8\xb9\x62\xc3\xd6\x16\xf1", + 1, 32, NULL, 0 }, + { 384, 192, 87, + "\xf6\x28\x20\xed\x5f\x98\x33\xfd\x22\xde\xe7\xbd\x49\xe2\xc9\xb1\x9f" + "\xc9\x66\x88\x97\xc2\xc3\x3e\x6c\x7c\x1f\xa5\xc2\x77\xc3\xb9\xf5\x81" + "\xfa\xef\x3d\xdc\x66\x4b\xa5\x37\x97\x5d\x8a\xfa\xa7\x07", + "\x9b\x6c\xc7\xca\xa4", + "\xf6\xe2\x72\xa7\xa6\x23\x5f\x60\xb7\x2b\x4c\x74\x24\xcf\x32\xa0\x7f" + "\x98\xea\x59\x26\x65\xba\xd8", + 1, 40, NULL, 0 }, + { 384, 192, 88, + "\xf2\x22\xa1\xda\xbf\x32\x2a\xff\x84\x63\xac\xee\x64\x44\x93\x93\x31" + "\x21\x2b\xe3\xe1\x9d\x31\xf4\xb7\x3f\xdc\xc9\x7e\x29\x25\x36\x5e\xa3" + "\x3c\x98\x52\x82\x80\x5c\x83\xdc\xd8\xfb\x42\xa0\xe2\x14", + "\xc8\x5a\xd7\x87\x2b\x76", + "\x93\x3f\x0f\xa6\x1d\x44\x66\xb5\xba\xf5\xa6\x01\xf6\xb9\x6d\x81\xa9" + "\x7e\x81\xc5\x12\xd8\x22\xe6", + 1, 48, NULL, 0 }, + { 384, 192, 89, + "\x56\xe8\x0f\x38\x99\xe9\x45\x31\x0a\x9d\x9b\xef\x3d\x32\x09\x1f\x29" + "\xc1\x57\xdd\x46\xb2\xd4\x39\xad\x89\xd6\x3e\x14\xb2\xc2\x43\x90\xf7" + "\x4d\xb4\xd9\x05\xf6\xbd\x03\xf7\x5c\x32\xe9\x12\x25\xfe", + "\x80\xba\x25\xf1\xc2\x76\x50", + "\xa1\xa6\xe2\x48\xb4\x08\x64\xdd\xf8\x3b\x00\xc5\x2a\xe2\xc3\x03\xb7" + "\xe7\x6f\xba\x05\x48\xd4\xd4", + 1, 56, NULL, 0 }, + { 384, 192, 90, + "\x6c\xb6\x26\x1a\x56\xa2\x1b\x2c\x3c\x13\x45\x3c\x15\x83\x64\xaa\xfa" + "\x78\xf5\x81\x72\xa9\xae\x3e\xeb\x32\x8a\xc3\x88\x08\xb5\xc6\x8c\x11" + "\x11\x97\xa3\x03\xec\x36\x84\x7c\x9a\x31\x5a\xc5\xeb\x5b", + "\x79\x43\x0d\xe5\x1d\x68\xcf\x34", + "\x33\x59\x3a\x80\xda\x45\x5e\x58\x0c\xcc\x5e\xe9\xb6\x0e\xdc\xd1\x46" + "\x84\x60\x53\x97\x88\xfc\x41", + 1, 64, NULL, 0 }, + { 384, 192, 91, + "\x44\xca\x1e\xcb\x49\x04\x70\xa8\x4c\x7e\x13\xe1\xf1\xc6\x9d\xa2\x1f" + "\x48\xc3\x3b\x6f\x05\x0f\x48\xf7\xf2\x44\xf0\xfd\xa8\xb3\xc8\x55\x90" + "\x4e\xd0\x61\x2e\x2d\xaf\xa5\x10\x5c\xbd\x7f\x64\x49\xeb", + "\x87\x0b\x98\x1c\x8a\xfd\x9f\xae\x1b", + "\x93\x0f\x2e\x40\x1e\x3a\xaf\xb4\x6a\x0c\x40\x29\x00\x2f\x4e\xf1\xab" + "\x9f\xe8\x38\xbc\x00\xc7\x9e", + 1, 72, NULL, 0 }, + { 384, 192, 92, + "\xaa\xca\x68\x88\x2c\xfa\x72\x50\x98\x8a\x24\x7b\x96\xcf\xb3\x23\x2d" + "\x65\x67\x37\x8f\x8f\xa7\xe7\xaa\xac\xa1\xc3\x86\xe1\xae\x15\xe5\x49" + "\x57\xd2\x2b\xff\xf1\xe5\x0a\xe7\xf2\x1b\xee\xa1\x97\xa5", + "\xa6\xf3\x1b\x82\x2e\xc2\x4d\xa1\xb1\xe9", + "\xa9\xc2\xd6\x8f\x0a\xd1\xba\x50\x08\x9b\x16\x9c\x86\xd9\x65\xf9\x7f" + "\x52\x38\x8a\x48\xac\xe7\x44", + 1, 80, NULL, 0 }, + { 384, 192, 93, + "\x1b\x32\xf9\xb6\x37\x89\x34\xa5\x02\xdd\x74\xd8\xb7\x4a\x46\x06\xd5" + "\xb2\xc9\xa8\x58\x7f\xab\x1c\xfa\x90\xd7\x50\x07\x73\x4d\x2b\x8b\xdf" + "\xe6\x34\x81\x52\x43\x52\x6e\xbc\x0f\x33\xc0\x4d\x0d\x05", + "\x55\x36\x7c\x65\x7c\x79\x26\x10\xef\xdc\xc0", + "\x93\x40\x83\xc8\x59\x45\x91\xda\x78\x3f\x0d\xa2\x8f\x4b\x58\xad\xb6" + "\x04\xe9\xcc\x76\xb9\x9e\xfe", + 1, 88, NULL, 0 }, + { 384, 192, 94, + "\x09\xd9\x1b\x2f\xa2\x2e\x68\xb5\x33\x5d\x47\x82\x35\xaa\x4e\x15\x74" + "\x35\xc9\xac\xfe\xd7\x72\x21\x9a\xdf\xa1\xe9\xdd\x72\xf3\x3e\x1a\x21" + "\x83\xa0\x20\x3a\x10\x4f\x80\xe6\x43\xcd\xf2\x9e\x5a\xff", + "\xb3\x1e\x25\x49\x57\xdb\x6b\x1b\x70\xa0\x6c\xe2", + "\x7d\x45\xf3\x89\x94\x55\x78\x7e\x71\x16\xb5\x70\xdf\x8f\x77\x87\xf6" + "\x72\xd5\x82\x1d\x6f\x75\xfe", + 1, 96, NULL, 0 }, + { 384, 192, 95, + "\xd3\x11\xa8\x0a\xc8\x01\xe3\x63\x9b\x91\x85\x60\x8a\xf4\xa8\x5e\x41" + "\x22\xe2\x9b\x5c\x23\xf0\x52\x34\xc3\x0d\x92\xd5\x9a\xd1\x3c\xb8\x03" + "\x90\xe5\xfa\x0e\xa4\xa5\x48\x53\x22\x8b\x35\x66\x89\xf5", + "\xe6\xb4\x43\xdb\xa0\xda\xb3\x5d\x43\xca\x5d\x6c\xe6", + "\x27\x29\x70\x96\xf5\x8f\x59\x83\x91\xc5\x77\x78\x12\x99\x49\xb9\x46" + "\x28\xbf\x17\xbb\x24\x22\xd1", + 1, 104, NULL, 0 }, + { 384, 192, 96, + "\x06\x29\x7e\x6c\x46\x55\x8b\x9b\x0f\xc3\x6c\x27\x2b\x4a\xe7\xe6\x5d" + "\xd5\x36\xcc\x1d\x13\xac\xbf\xa8\x31\xfa\x55\x74\xb3\x4f\x99\xe0\x9a" + "\xdf\xb7\xf2\x03\x21\xf2\x03\x07\x5f\xd2\x6e\xd2\xe2\x9d", + "\x30\x9b\x95\xe5\xf1\xec\x26\xf7\x07\x86\xe7\x4d\x80\x6d", + "\xaa\xbf\xf2\x6f\xc4\x4a\x40\xf0\xb8\x7a\x40\xc1\x75\xc1\x7e\xa7\x14" + "\x0f\x84\x67\xdc\xdb\x95\xcd", + 1, 112, NULL, 0 }, + { 384, 192, 97, + "\xe8\xb6\x3a\x25\xcd\x85\xad\x4f\x39\xe3\xc0\xe9\x58\x4e\xac\xb9\x4d" + "\x6a\xe3\x3f\x98\x4d\xa2\x59\xaa\x53\x3d\x4d\x28\xae\xb3\x41\xcf\x3f" + "\xfe\x49\xc0\x29\xe4\xaf\x6a\x48\x05\xf7\x60\xf3\x5f\x2c", + "\xd2\x25\xc2\x77\x95\xf8\x09\x45\x4b\xb2\xc5\x1d\x21\xf3\xac", + "\x0e\x12\xb7\x58\x01\x5a\xc8\x97\x97\xd5\x54\x70\xf3\x98\x2c\x13\xa5" + "\xff\x14\x83\x27\x60\x83\xd2", + 1, 120, NULL, 0 }, + { 384, 192, 98, + "\xd8\x3a\x68\x5a\xce\x9f\xa0\xc0\xaa\x47\xf0\xc7\xb4\xf0\xf0\x07\x17" + "\x61\x9a\x82\xe2\xee\xff\x87\xf5\x1f\x67\xd8\x14\xd5\x1d\xd9\xe4\xca" + "\xd7\x57\x8a\x4e\x49\xb6\x72\xb5\xaf\x83\x94\x3c\x25\x83", + "\xab\xfa\x7f\x59\x78\xf7\x51\xe8\x7e\x8b\x5a\x15\xa6\xe8\x9f\x4f", + "\xe4\xe6\xba\x04\x1b\xbb\x7a\x47\xec\x84\x82\xb2\x04\x34\x55\xc1\x19" + "\xfb\xdb\x38\x9a\x39\x45\xa0", + 1, 128, NULL, 0 }, + { 384, 192, 99, + "\x5b\xea\xf4\x06\xa6\x62\x7e\xaa\xfc\xad\xb6\xde\xa4\xe2\x7b\xa4\xfd" + "\x87\x9f\xd3\xe5\xbf\xd8\x7e\xa3\xc8\xd5\xe0\xac\xfb\xbd\xa2\xc6\xbf" + "\x00\x6b\xea\xf5\xa3\x03\x12\xe6\x90\x72\x4c\x47\x44\xa3", + "\xbc\x57\xd4\x67\xa9\xa2\xaf\x64\xad\x5e\x14\xb7\xbc\x08\x98\xdc" + "\x63", + "\x3f\xab\x1a\x7a\x19\x23\x59\xb6\x33\x3a\x96\x99\xb7\x56\x12\x21\x1a" + "\x38\xb6\xdc\xca\xb4\x57\x2d", + 1, 136, NULL, 0 }, + { 384, 192, 100, + "\x76\xb3\x6c\xc3\xb8\xca\x97\x57\x08\xee\x4b\x32\xbd\xbe\x40\xca\x13" + "\xf9\xce\x38\x4c\x52\xc4\xb6\x60\x2b\x7f\xd9\x21\x64\xf1\xfd\x84\x32" + "\x70\x6c\x19\x66\xf6\x48\xbf\x48\x30\xf4\xde\xb3\x47\x95", + "\xb1\xd0\x22\xc6\x53\x6f\x40\x1d\x14\x7d\xfc\x0d\x7d\x4e\x60\x0b\xb7" + "\x53\xef\x0e\x9f\x24\x3b\xc3", + "\xc9\x1e\xb3\xf3\x62\x04\x9c\x53\x36\xc5\x07\x4c\xb8\x87\xed\xcb\x27" + "\xaa\xc1\xef\x65\x75\xa9\x2d", + 1, 192, NULL, 0 }, + { 384, 192, 101, + "\x20\x56\x9a\x16\xf4\x53\xdd\x3c\x34\xdf\x98\x15\x52\x86\xb1\xca\x8a" + "\x39\x2e\xa1\x64\xc9\x19\x31\x1f\x0d\xf9\xd3\x9d\x97\x60\x62\xf4\xf9" + "\x92\xb9\x6d\xef\x38\x51\x88\x6e\x62\x95\xf2\x61\x50\x64", + "\x54\x02\xc4\xe6\x83\xd1\xa4\x31\x86\x8a\xd5\x28\xaf\xbf\x41\x28\xb0" + "\xb1\x0c\xef\x94\x7d\x06\x3b\x34\xd3\x76\xd3\x44\xb7\x93\xb2", + "\x27\x72\x80\x59\x69\x6a\xed\x5b\xb0\x0a\x13\xc1\xdb\x10\x06\x91\xd4" + "\xa2\x1e\xbe\xa0\xa8\xe4\xc3", + 1, 256, NULL, 0 }, + { 384, 192, 102, + "\x9e\xf6\xa5\x5f\x8a\x9b\x6b\x9e\xf1\xf8\x29\x61\x67\x31\x90\x78\x16" + "\x37\x06\xae\x5b\x60\x89\x7c\x2d\xd6\xe3\x40\xb6\x7e\xd5\xd5\x77\xfb" + "\x54\xc5\x54\x7c\xd5\xf2\x48\xf0\x6e\x70\x82\xff\xb8\x26", + "\x6a\x0d\x16\x27\x69\x41\xd8\xf0\x4e\xac\x2e\xc7\x23\xfa\x53\xb9\xd6" + "\xb1\x6d\xa7\xe3\x0e\x7f\x2d\x9a\xd8\x98\xe7\xcb\xb7\x1b\xd3\xdd\x23" + "\x4e\xe2\x28\x36\xff\x4a\xc6\x01\x1b\x6f\x12\xbd\x3a", + "\xce\xf5\xd9\x00\xee\xf0\xab\xef\xc6\x25\xc1\xd2\x86\x2a\x3f\x42\x99" + "\x8c\xe8\xb1\xe0\x07\xd2\xb8", + 1, 376, NULL, 0 }, + { 384, 192, 103, + "\xfb\x56\xbb\xbc\x6d\x75\x1b\x74\x4d\x8c\x1b\x57\xcc\x27\xa1\xd2\xc2" + "\xf4\xe3\x8e\x34\x91\xf5\x44\x48\xcf\xcf\xb9\x38\x9b\x7f\x63\xfd\x0d" + "\x41\x92\x09\x68\xef\x61\x25\x10\x62\x5f\x26\x37\xd2\x8d", + "\xcf\x17\x91\x51\x7e\xf5\xa6\x1c\x0d\xb6\x5a\x66\x8b\xee\x26\xfd\xbc" + "\x97\x5d\x79\x9b\x26\x23\xcc\x0f\x3e\x45\x60\xe8\x0c\x70\x14\xfa\x9c" + "\x02\xd5\x68\xc9\x8c\x86\x38\x5e\x00\x0f\xe6\x77\x6b\xb7", + "\x88\xe9\x9a\xcc\xc9\xc2\x3c\x9c\x8c\x11\x10\xe7\x47\x0c\xad\xe0\x31" + "\x78\x17\x91\x6d\x85\x05\xf5", + 1, 384, NULL, 0 }, + { 384, 192, 104, + "\xd0\x41\xe2\x4e\x59\xb3\x4d\x7a\x18\x12\x8a\x42\xd8\xa7\xa5\x2d\xcb" + "\xa5\xd7\x9e\x5e\xd5\x85\xb5\x5c\x7c\x9e\x49\x46\xe5\xcc\xaf\x7e\x59" + "\xdf\x0f\x3d\xa9\x8c\x7d\x05\x23\xe4\xcc\x8f\x9d\x7d\xa4", + "\x52\x79\x61\x8f\x1b\x41\x53\x49\x10\x39\x5a\x78\xde\xd9\x68\xae\xe3" + "\x43\x10\x85\xb5\x99\xc4\xf5\x5e\xb5\xff\x8a\x2e\x87\x9b\xc4\x42\x91" + "\xd9\x23\xde\x31\x00\x9d\xb1\xb9\xf7\xf8\x10\x95\xaf\xb3\xea", + "\x85\x00\xf6\x03\xce\x85\xc0\x30\xcf\xa0\x57\x31\x75\x8b\x6b\xe3\x31" + "\x7b\x6f\xe8\xe9\x9b\x7d\x48", + 1, 392, NULL, 0 }, + { 384, 192, 105, + "\xe1\xce\x48\x84\xfd\x74\xa0\xe1\x97\xc6\x8a\xce\x3b\x29\xb5\x52\x31" + "\x3a\xf8\xe4\x51\xe9\x8d\x9a\xb8\xd0\xe8\xf8\xee\x74\x14\x3e\x8f\xcb" + "\x64\x46\x21\x7c\x0f\x31\x23\xa4\x26\xb8\xab\x6f\x62\xcb", + "\x71\x15\x4b\x9a\x65\x7b\x90\x5f\x88\x4b\xa5\x14\x0d\x5e\x7b\x92\x43" + "\xfe\xc3\xe0\x3f\xbb\xdb\xb3\x60\xc8\x19\x49\x63\xae\x43\x17\x7b\x55" + "\x02\xcd\x20\xf5\x59\xee\xef\xf8\x63\x8d\x02\x8c\x50\x19\x26\xeb\xc7" + "\xed\xdd\x13\x2c\xce\xa2\x9e\xad\x7a\xd0\xc9\x5a\x30\xb9\xd3\x25\x95" + "\x2c\xaf\xb0\xea\x5e\xc9\xd9\xd6\xfd\xeb\x63\x95\x0d\x5d\x69\xc8\xbb" + "\xbe\xa7\x02\xae\xd1\xd4\x44\xda\x28\x68\x07\xff\xd6\xb3\x6c\xb4\x99" + "\x02\xcb\xa7\xab\xf9\xbd\xa1\xb5\x77\xc6", + "\xc7\xe9\xae\x2a\x81\xde\x32\x28\x0b\x51\x8d\x05\x5c\x2c\x9d\x7f\x0f" + "\x5d\xb6\xd0\x6a\xd0\xe4\xae", + 1, 896, NULL, 0 }, + { 384, 192, 106, + "\x8a\x24\x2c\x22\xd1\xb5\x4c\xe2\x16\xca\x03\xc8\x84\x55\xbe\xb1\x28" + "\x21\x1a\x9f\x35\xaf\x23\x43\x70\x9a\xf7\xc5\xf4\x3a\x68\x14\x51\xea" + "\x53\xa3\x6d\xe2\xe5\x04\x8e\xb4\x4a\x51\x68\x1c\x61\x20", + "\xab\x5e\xee\x6b\x83\x86\x91\x19\xf0\x0d\xd3\xcc\x66\xdd\xe7\x5c\xb5" + "\x70\x05\x35\xa9\x0e\x9b\x3e\x32\xb3\x14\x34\xc2\x97\xef\x53\xf9\x46" + "\x59\xd7\xd9\xb1\x13\x23\x16\x1b\x2e\x66\xc6\xb9\xc9\xad\x20\xe3\x13" + "\x30\x3f\x81\xe8\x8e\x47\x17\x86\xc8\xe9\x36\x01\x1f\x78\x12\x1e\x39" + "\x63\x0b\x2e\x08\x04\xfc\x97\xce\x5c\xb3\xa3\x4f\x26\x94\x94\x39\xfe" + "\x53\x0a\xdc\xea\x6e\x97\xc7\x8b\x04\x2e\x08\x17\x25\x3b\xf7\x5d\xd5" + "\x43\x35\x58\x41\x22\xf5\xed\xd2\x10\x34\x1b\x6d\x93\xf5\x8a\xa1\xb4" + "\xde\x2a\xad\x76\xfe\xce\xc4\x4f", + "\x77\x39\x2b\x18\x57\x7b\xa8\x81\x9f\xbd\x76\xfc\x73\xd4\x50\x29\xe5" + "\x5e\x7e\xbe\xcd\x58\xa3\x20", + 1, 1016, NULL, 0 }, + { 384, 192, 107, + "\x31\x1c\x4b\xee\x7c\xf2\x57\xb7\x80\x13\x5a\x2e\x4a\x64\x13\xe6\x8a" + "\x81\x6f\x5d\x84\x62\x51\x5d\xcb\x1c\x72\x49\x4b\x63\x35\x58\x1a\x9b" + "\x60\xa2\x17\xb9\xff\x1c\x75\xe7\x76\x81\x48\xf8\xdf\x46", + "\x63\xcc\xc3\x84\x9c\x4c\x32\x3c\xb6\xce\x92\x68\x77\x96\x90\x48\xb8" + "\x49\xee\x4a\xf1\x8e\x71\xee\xf5\x2f\xe9\xf2\x74\xa8\x67\x85\x60\xf9" + "\xa5\xd4\x75\x10\xc3\xc9\x8c\x8a\x08\xed\x4c\x01\xa0\x1e\x0a\x36\x63" + "\xef\x0c\xc6\xc3\xcd\xca\x62\x76\xd9\x1e\x99\xb0\xd4\x14\x26\x34\x98" + "\xfb\x64\xad\x74\xb8\x20\xab\x52\xb3\x7a\xde\xaf\x27\xcb\x44\x54\x5e" + "\xdb\x8f\x09\x09\x49\x92\x83\x7b\x8d\x3a\x0b\xaa\x2a\x10\x1a\x49\x59" + "\x2e\xb8\x89\xdc\x8b\xac\xe4\xc7\x1e\x3e\xfc\xb9\xd4\x14\x9b\xd6\x70" + "\xce\x2f\x77\x4d\x73\xc1\x2f\x2a\x45", + "\x94\x67\x4a\xae\xfc\x06\xee\xad\x22\xd1\x53\x17\x90\x0f\xa2\x6c\x8d" + "\xf8\xcd\xfb\x25\x2b\xca\xae", + 1, 1024, NULL, 0 }, + { 384, 192, 108, + "\xfb\x79\x28\x67\xc8\x92\x8f\x05\x03\xaa\x24\x47\x7c\xeb\xf4\x2e\x0b" + "\x01\x83\x46\xe3\x61\x97\x70\xb9\xe8\xf5\x09\x79\x45\xe2\xe2\x75\xad" + "\x06\xf0\xc1\x21\x52\x36\x6a\xc0\x6e\x27\x8c\x94\x09\x0a", + "\x0a\x63\xe6\xd9\x1d\x7a\x6a\x18\xdb\xad\x87\x9f\xb8\xe2\x3a\xe3\x51" + "\x92\x03\x91\xeb\x40\xfe\xad\x6c\xba\x84\x67\x68\xa2\xc6\x79\x7f\xf3" + "\x47\xb4\x30\x13\x27\xb0\x9a\xfc\x41\xf7\xb8\x03\xaf\x6b\x61\xf6\xd9" + "\xb8\x18\xe0\xdd\xcc\x02\x53\x6d\x05\x43\xdb\xf1\xa8\x7f\x2c\x5e\x02" + "\x0f\x64\x59\x09\x43\x44\xb7\x25\x96\xd5\x48\x43\x5c\x31\x35\x44\xe9" + "\x2c\x25\x4d\x54\xa7\x0a\x1d\x6f\x6e\xdd\x2f\x82\x54\x0a\x1e\xa2\xe8" + "\x21\x25\xb0\x71\x5f\xa0\xf8\x90\xbb\x2b\xe4\xba\x00\x65\xd2\xba\x01" + "\x44\x85\x46\x82\xae\xd0\x41\xc1\x03\x59\x96\x64\x8e\x2e\xd6\x71\xb7" + "\x25\x3b\xa5\x67\xff\xb9\x99\xd9\x1f\xd8\xe7\xff\xce\x5c\x6d\xc4\x79" + "\x07\x32\xad\xae\x44\x34\x35\xa4\x54\xfe\x6c\x2a\x7c\x67\x08\xd9\xd5" + "\xb2\xeb\x92\x92\xd6\xfb\xe5\xe0\x26\xd6\x53\x32\xb3\x8c\x79\x25\xef" + "\xf9\xbe\xb8\x90\x63\xca\xb6\x3f\xbe\xcb\x2a\xc0\xe1\xbb\x61\xa5\xb1" + "\xe5\x11\xf9\x49\xc4\x3a\x34\xee\x26\xf1\x15\x6e\x97\x79\x3d\xa9\x7b" + "\xcf\x5b\x5c\x67\x64\x13\x84\xf2\x68\x13\x1b\x29\x78\x57\xd7\x19\xee" + "\xb6\xca\xfa\x3d\xbe\x9b\x8d\x0d\xa5\x5c\x98\x65\x6f\x20\xe5\xb3" + "\x9b", + "\x1a\xaa\xff\x96\x6c\x0a\x84\xba\xc7\x91\xab\x9e\x0b\x9b\x50\x5d\x39" + "\x30\x73\x66\x57\x32\xa7\x4a", + 1, 2040, NULL, 0 }, + { 384, 192, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x46\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xab\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xc4\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x29\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbf\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4c\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\x44\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\x7f\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x14\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x91\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x17\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x92\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7b\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x65\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\xfa\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\xe4\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x42\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\x90\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa5\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x37\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x89\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd6\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x8a\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd5\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x08\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\x57\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6d", + 0, 0, NULL, 0 }, + { 384, 192, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x14", + 0, 128, NULL, 0 }, + { 384, 192, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6e", + 0, 0, NULL, 0 }, + { 384, 192, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x17", + 0, 128, NULL, 0 }, + { 384, 192, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x2c", + 0, 0, NULL, 0 }, + { 384, 192, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x55", + 0, 128, NULL, 0 }, + { 384, 192, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\xab\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\xec", + 0, 0, NULL, 0 }, + { 384, 192, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\xe0\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x95", + 0, 128, NULL, 0 }, + { 384, 192, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbe\x81\xc4\x15\xd2\x83\xab\x7b\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4d\x07\xff\x90\xb3\x38\xe0\x65\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\x44\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\xfb\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\x7f\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x02\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x44\xbe\x81\xc4\x15\xd2\x83\x2b\x7a\x62\xa4\x51\x88\xe5\xda\x7b\xcb" + "\x97\xda\x60\x6b\xd5\xb1\x6c", + 0, 0, NULL, 0 }, + { 384, 192, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa9\x4d\x07\xff\x90\xb3\x38\x60\x64\xb0\x36\x03\xd7\x6b\xcf\x82\x14" + "\xb1\xfb\x88\xc6\x6b\x94\x15", + 0, 128, NULL, 0 }, + { 384, 192, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xbb\x41\x7e\x3b\xea\x2d\x7c\x54\x85\x9d\x5b\xae\x77\x1a\x25\x04\x34" + "\x68\x25\x9f\x94\x2a\x4e\x93", + 0, 0, NULL, 0 }, + { 384, 192, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x56\xb2\xf8\x00\x6f\x4c\xc7\x1f\x9b\x4f\xc9\xfc\x28\x94\x30\xfd\xeb" + "\x4e\x04\x77\x39\x94\x6b\xea", + 0, 128, NULL, 0 }, + { 384, 192, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 384, 192, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 384, 192, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 384, 192, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 384, 192, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\xc4\x3e\x01\x44\x95\x52\x03\x2b\xfa\xe2\x24\xd1\x08\x65\x5a\x7b\x4b" + "\x17\x5a\xe0\xeb\x55\x31\xec", + 0, 0, NULL, 0 }, + { 384, 192, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x29\xcd\x87\x7f\x10\x33\xb8\x60\xe4\x30\xb6\x83\x57\xeb\x4f\x82\x94" + "\x31\x7b\x08\x46\xeb\x14\x95", + 0, 128, NULL, 0 }, + { 384, 192, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "", + "\x45\xbf\x80\xc5\x14\xd3\x82\xaa\x7b\x63\xa5\x50\x89\xe4\xdb\xfa\xca" + "\x96\xdb\x61\x6a\xd4\xb0\x6d", + 0, 0, NULL, 0 }, + { 384, 192, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xa8\x4c\x06\xfe\x91\xb2\x39\xe1\x65\xb1\x37\x02\xd6\x6a\xce\x03\x15" + "\xb0\xfa\x89\xc7\x6a\x95\x14", + 0, 128, NULL, 0 }, + { 192, 384, 163, + "\x08\x47\x6e\x9d\x49\x49\x9c\x5f\x52\xe3\x7f\x80\xec\xe6\xf5\xa4\x54" + "\x59\x94\x88\x06\xb4\x82\x41", + "", + "\x1b\x6c\xfc\x87\x09\xaa\xb8\x07\x54\x65\xf3\x2e\x13\xb0\xb0\xf7\x96" + "\xcc\x34\xd9\x3d\x7b\xed\x09\x0f\x29\x7d\xcf\x9f\xb7\x5e\x0d\x8e\x28" + "\x5b\x15\x00\xb7\x32\xd5\x54\xac\x97\xba\x45\xf3\x3e\x47", + 1, 0, NULL, 0 }, + { 192, 384, 164, + "\x21\x3b\x44\xd8\xe1\xfa\xba\xff\x83\x7e\xf3\x0e\xe2\x54\x2f\x9a\xb8" + "\x2e\xd7\x04\x11\xda\xe7\x8f", + "\xee\x0b\xf4\x85\x85\xc1\x86\xff\x99\x1b\x4d\x86\x07\x81\x7c\x9c", + "\x54\xf4\x01\x0d\x50\xf8\x0b\xcd\xb4\xb8\x4d\x56\xbc\x4e\xf3\x0e\x4c" + "\x68\xf7\x51\x28\x21\x4c\xf4\x46\xb5\x14\x5f\x6f\xff\x13\x26\xa2\x09" + "\x94\x5f\xc2\x1a\xb5\xe1\xf5\xd9\x17\x55\x9e\xa9\xb8\x00", + 1, 128, NULL, 0 }, + { 192, 384, 165, + "\xb4\xaf\xa9\xda\xaa\x8c\x94\x4d\x73\xa3\x88\x1f\x32\x21\xe4\x2b\x34" + "\xef\x4e\x35\xf1\x84\xe8\x78", + "\xcf\x60\x7f\x6a\x0e\xb4\x4e\xcb\xca\x81\xb6\xd1\xfd\xb5\x95\xce\xe3" + "\x5f\x23\x53\xda\x02\xe8\x2e\x28\xe1\x33\xb9\xde\xcd\x8f\xbb", + "\xd0\x64\xa5\x1f\xb1\x09\xc3\xb1\xd4\x43\xf1\x3f\x41\xe9\x0e\x14\x19" + "\x8f\x84\x60\x80\x46\x45\x47\x80\x6d\x46\xa8\x15\x1c\x4e\x38\x55\xa8" + "\x1f\x4a\xf4\x09\x15\x60\x90\x95\xdd\x72\xf8\x69\xaa\x1b", + 1, 256, NULL, 0 }, + { 192, 192, 166, + "\x89\xe4\x6b\x66\x20\x95\x48\xc8\x0b\x0c\x83\x06\x62\x22\x3b\x49\xb0" + "\xe3\xb8\x95\xeb\x30\xe2\xfc", + "", + "\x4b\x01\x2c\x0c\x0d\xa4\x4e\xde\x2a\x42\x7e\x85\xac\xe8\xec\xc5\x4b" + "\x37\x9e\x9e\x24\xf0\x8d\x41", + 1, 0, NULL, 0 }, + { 192, 192, 167, + "\xf2\xc1\x0c\xe8\xcb\x1c\xf3\xb3\x63\x35\x44\x73\xb0\x27\xc1\xe5\x3d" + "\xec\xce\xf0\x32\x33\xbe\x0c", + "\xe1\xfa\x10\xb8\xe3\x01\xe0\x34\x84\x05\x77\x0b\xc3\xfa\xfc\xb1", + "\x2d\x08\x8a\xf2\x9c\xc7\x44\xe3\x47\x12\x4f\xbe\x41\x00\xcb\xcd\xeb" + "\xba\xe0\x37\xed\x9b\xf6\x9d", + 1, 128, NULL, 0 }, + { 192, 192, 168, + "\x92\xe0\x74\x44\x2c\xc4\xc5\x9e\x72\x26\x08\x08\xd8\x0d\x8e\x7b\x85" + "\xc6\x33\x50\x68\x91\x7b\x83", + "\x34\xea\xe2\x74\x25\xac\xe1\x77\x71\xe1\x64\xcb\xb6\x34\x30\x6f\x35" + "\x2e\xdc\x9c\x37\xbf\x60\x8b\xe8\xa7\x55\xfb\x94\x14\x81\x83", + "\xb7\xe6\xb7\xbb\x29\xc0\x2e\x46\x35\xdb\xdc\x50\xd8\xbe\x71\xe2\xdd" + "\xf0\xa5\x44\x47\x1d\xe2\x85", + 1, 256, NULL, 0 }, + { 520, 384, 169, + "\xdb\x6f\x99\x56\xc3\xf4\xca\x6e\x41\xf1\xf7\xf1\x46\x29\xd4\x4c\x79" + "\xe0\x35\x3e\xdb\xf3\xe3\x10\xe6\x85\x8b\xbc\x45\xa7\xcd\x57\x77\x8a" + "\x90\x53\xba\x22\xa1\x41\xbf\x58\xbf\xd4\x34\xad\x08\x64\x8c\x70\x41" + "\xa2\x24\xb9\x7a\x0d\x17\xe0\xed\xf9\x4f\xd4\x0b\x41\x0a", + "", + "\x0c\xb1\xb2\x96\x25\x5b\xb2\x59\xf3\xb6\x01\xb4\x9b\x35\x52\x4a\x5e" + "\xca\x6c\x52\x36\x07\x54\xd3\xd9\x6d\xd5\x21\xc9\x05\xb1\xc1\x82\x1d" + "\x74\x96\x59\x67\xd8\xe8\x6d\x50\xde\x95\x0f\xe4\xd6\x35", + 1, 0, NULL, 0 }, + { 520, 384, 170, + "\xf0\x34\x04\xbd\xb3\xe0\x8f\x53\x0d\x4c\x3a\x5f\x16\x5d\x23\x60\x12" + "\xa4\xc4\x5c\xd0\x63\xe3\xe4\x48\x3d\xa0\x88\xec\x0a\xfd\xb2\x4e\x96" + "\x39\xfc\xca\xbb\x91\xf9\x8a\x49\xdc\x29\x72\xe2\x98\x14\x26\x57\x3e" + "\xcf\xe6\x9c\x00\xc4\x3a\x2d\x99\xa3\x10\x7c\xef\x3a\x70", + "\x73\xed\x9f\xa2\xac\xf4\x9d\x6c\x98\xbf\xc7\xd6\xc5\xad\x9c\x56", + "\xb6\x13\x2e\x52\x16\xf7\x11\xee\xeb\x44\xda\x3d\x92\x98\x3f\xe5\xb6" + "\xde\x5c\xd9\x41\x0b\xe7\x1d\xb8\xd3\xb0\x72\x28\x34\x16\x86\xaa\x60" + "\xe7\x08\x1e\x95\xf2\xe4\xb6\x9b\xb7\xcd\x96\x48\xbc\x0b", + 1, 128, NULL, 0 }, + { 520, 384, 171, + "\xee\x79\x9e\x25\xed\xb1\xb1\x84\x52\xe5\xed\x17\x4b\xc6\xb2\x18\x5a" + "\x67\x54\x41\x7d\x6c\xc0\x5d\x73\x6d\x2b\xa9\xef\xc8\x36\x7e\x4b\x05" + "\xba\x0a\x2e\xe5\x25\xce\xea\xb7\x4f\x98\x04\xa8\x47\x91\x30\xc3\x28" + "\xd6\x71\xe3\x40\x70\xcf\x17\x4a\x00\x3a\x1d\xfb\x59\x94", + "\xac\x3e\x7d\xa7\xe5\x78\xb9\xb4\xdc\x24\x24\x03\x04\x46\xc7\xf6\xae" + "\xbc\xc4\x71\x44\x5a\x9e\x0e\x6e\x65\x09\x9c\xae\xec\x5b\x2f", + "\xc8\x60\x7f\xca\x18\x88\x41\x81\x66\xc5\x50\xdd\x58\xd7\xa3\x97\x6a" + "\x6e\xcd\x0e\x4c\xa9\x9b\x02\xfb\x18\x78\x00\xa9\xc9\xef\x90\x9a\x6c" + "\x14\x97\xc0\x65\x2d\x4d\xca\x82\x40\x5a\xb0\x7f\x5e\xed", + 1, 256, NULL, 0 }, + { 520, 192, 172, + "\x06\x3d\x6e\x12\xe6\x70\x09\x8a\xda\xbe\x68\x19\x20\x23\xb6\x37\xbb" + "\x6d\x8d\x71\x3f\xc8\x43\x61\x88\xc4\xec\x06\xfd\xd0\x84\xce\x6d\x19" + "\x3f\x26\xc8\x6a\x95\x60\xe1\xab\xc2\x7d\x81\x3f\xce\x2b\x3e\xac\x01" + "\x70\xfd\x1c\xb7\x2e\x19\x30\xa2\x77\x6b\xc8\x4d\x6c\x11", + "", + "\x9d\xc2\xac\xbf\xa2\x8a\x7a\xc5\xf2\xa5\xbd\xd4\xb1\xb2\xdb\xc8\x06" + "\xc4\x8f\x96\xce\x95\x0e\xb5", + 1, 0, NULL, 0 }, + { 520, 192, 173, + "\x35\x93\x18\xe6\xc6\x27\x9b\xa9\xeb\xcb\x16\x75\xf5\xa9\x81\x95\xbb" + "\xf5\xd8\x95\xda\x9c\x17\xb8\x32\x90\x38\xbe\x85\x7d\xc3\x95\xb1\x2a" + "\xe9\x1a\x55\x59\x88\x76\x59\x3c\x1c\x20\xbc\x01\x72\xcf\x15\x12\x6b" + "\x7a\x6b\xf0\xa2\x38\xed\xa3\x32\x5d\x6d\xd6\x06\x00\xef", + "\x7a\xd0\xc9\x09\x8e\xa1\x0e\x61\x5b\xb6\x72\xb5\x2c\x96\x54\x2d", + "\x41\x63\x73\x7c\x21\x9f\x7c\x5e\x74\x38\x43\xdc\x3d\x36\x01\x9c\x65" + "\x85\xea\x5d\x4e\x7c\xf2\x4f", + 1, 128, NULL, 0 }, + { 520, 192, 174, + "\xd0\x1c\xd8\x98\x08\x9d\x8a\x1e\xeb\x00\x35\xb0\xd3\x32\xda\x80\xfb" + "\xd3\x57\x1b\x91\x92\xdb\x10\xfa\x6f\x55\xf6\x65\xab\x19\x2d\x70\x50" + "\xca\xb6\x43\x99\x6e\x99\x25\x4d\x95\x73\xe0\xcf\x4e\xea\xa6\x3a\xfc" + "\xcd\xef\xd8\x16\x14\xfe\x7b\x83\xdf\xe3\x0e\x3b\xa1\x9f", + "\xd6\x7c\x77\xcd\xd0\xaf\x5d\x10\xe8\xca\xe8\x87\xe5\xa6\x09\xbb\x76" + "\xa9\xe5\x59\x76\x53\x77\x3c\x30\x3b\x82\xb9\x18\xfd\xc5\x9f", + "\xe7\xdf\x52\x7a\x98\x80\x80\x74\x9e\xe2\x15\xba\x0f\x82\x07\x83\x8d" + "\xf3\x8a\x37\x70\x7a\x63\x30", + 1, 256, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/hmac_sha512_test.json.c b/test/wycheproof/hmac_sha512_test.json.c new file mode 100644 index 00000000..4e7b79fb --- /dev/null +++ b/test/wycheproof/hmac_sha512_test.json.c @@ -0,0 +1,1853 @@ +/***************************************************************************** + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +/* Vectors from https://github.com/google/wycheproof */ +/* HMACSHA512, 0.8rc21 */ +#include "mac_test.h" +const struct mac_test hmac_sha512_test_json[] = { + { 512, 512, 1, + "\x53\x65\x24\x4b\xb4\x3f\x23\xf1\x8d\xfc\x86\xc0\x9d\x62\xdb\x47\x41" + "\x13\x8b\xec\x1f\xbd\xdc\x28\x2d\x29\x5e\x0a\x09\x8e\xb5\xc3\xe3\x7b" + "\xd6\xf4\xcc\x16\xd5\xce\x7d\x77\xb1\xd4\x74\xa1\xeb\x4d\xb3\x13\xcc" + "\x0c\x24\xe4\x89\x92\xac\x12\x51\x96\x54\x9d\xf9\xa8", + "", + "\xd0\xa5\x56\xbd\x1a\xfa\x8d\xf1\xeb\xf9\xe3\xee\x68\x3a\x8a\x24\x50" + "\xa7\xc8\x3e\xba\x2d\xaf\x2e\x2f\xf2\xf9\x53\xf0\xcd\x64\xda\x21\x6e" + "\x67\x13\x4c\xf5\x55\x78\xb2\x05\xc8\xa1\xe2\x41\xba\x13\x69\x51\x6a" + "\x5e\xf4\x29\x8b\x9c\x1d\x31\xe9\xd5\x9f\xc0\x4f\xe4", + 1, 0, NULL, 0 }, + { 512, 512, 2, + "\x00\x69\x89\x77\xf7\x10\x2c\x67\xb5\x94\x16\x69\x19\xaa\x99\xdc\x3e" + "\x58\xc7\xb6\x69\x7a\x64\x22\xe2\x38\xd0\x4d\x2f\x57\xb2\xc7\x4e\x4e" + "\x84\xf5\xc4\xc6\xb7\x92\x95\x2d\xf7\x2f\x1c\x09\x24\x48\x02\xf0\xbc" + "\xf8\x75\x2e\xfb\x90\xe8\x36\x11\x07\x03\xbf\xa2\x1c", + "\x01", + "\x4d\x16\x09\xcc\x2c\x2f\x1a\xb5\xdd\xc3\x58\x15\xae\x1b\x5d\xc0\x46" + "\xf2\x26\xbd\xe1\x7e\xc3\x7a\x4c\x89\xec\x46\xfb\xd3\x1a\xf2\xae\xb8" + "\x10\xb1\x96\xdf\xfd\xd1\x19\x24\xd3\x77\x2b\xef\x26\xa7\xa5\x42\xe0" + "\xa1\x67\x3b\x76\xb9\x15\xd4\x1c\xbd\x3d\xf0\xf6\xa6", + 1, 8, NULL, 0 }, + { 512, 512, 3, + "\xed\x6d\xc6\x5d\xbe\xaa\xdb\xda\xab\x53\x0a\x0d\x35\xf1\x9f\x78\xa7" + "\xbd\x93\xe6\x98\x54\x6c\x82\x75\x1b\xf6\x50\xc2\xa4\x4f\xc8\x52\x90" + "\x33\xd0\x88\xfe\xbe\xed\x28\x8f\xb4\xc8\x13\x2a\x59\xdf\x02\x07\x68" + "\x76\x40\xc7\x6d\xcd\xb2\x70\xac\x3a\xf5\xf0\x42\xf1", + "\xa7\x8f", + "\x07\x57\xb2\x7e\x12\x05\x59\xd6\x4c\xd3\xd6\xe3\xcb\x40\xd4\x97\x84" + "\x53\x75\x81\x51\x81\xbd\x9b\x4e\x74\xf2\x18\x9d\x09\xd0\x1a\x1b\x3e" + "\xad\x53\x70\x13\x80\xd9\x88\x95\x8e\xd2\x2b\xc3\x79\xac\xe9\xd4\x7c" + "\xbc\xac\x1d\x49\xbf\xa7\xe1\x4f\x1f\x44\x80\x4c\x30", + 1, 16, NULL, 0 }, + { 512, 512, 4, + "\x46\x3c\x5e\x69\x6d\xa0\xec\x0d\x78\x43\x88\xbe\x77\x5d\x1d\x91\xd9" + "\x47\x46\xaa\x8d\x3d\x2c\x20\x9f\x56\xac\x95\xea\x54\xe7\x28\x83\x29" + "\xf9\xfb\x40\xbe\x4e\xef\x35\x54\x7e\x64\xc6\x1d\xc5\x1a\x4a\x1f\x33" + "\x80\xa2\xb9\x64\x20\xf0\x88\x65\x5e\xa9\xd8\x5b\x97", + "\xe9\x56\xc1", + "\xac\x4b\x15\x09\x39\x18\x14\xae\x5c\xb5\xa1\x23\xe7\xa0\x60\x60\x15" + "\x75\xc1\x1d\x81\xb5\x63\xbd\xc5\x2f\xeb\xe6\xbb\x2c\x74\x7b\x85\xee" + "\xdd\xcb\x67\x48\xc9\x81\x47\xa4\x6a\x1c\xc9\xbe\x67\x76\xd1\xa8\xe8" + "\x2a\xe4\x89\x6b\x9c\x18\xda\x2f\xf3\x51\xc5\x67\x95", + 1, 24, NULL, 0 }, + { 512, 512, 5, + "\x4b\xc0\xd3\x2e\x94\x5c\xfd\xaf\xd2\x0d\x39\xbe\x38\x20\xf9\x64\x97" + "\x27\xcb\xda\x5a\xb5\x85\x99\x53\xa3\x22\xcb\xde\x1a\xb7\xa5\x14\xd7" + "\xdc\xd1\x4b\xa9\x09\x05\xe7\x09\x19\xbb\x86\xb8\x5c\xfe\xaa\x37\x5e" + "\xe2\xce\x27\x03\x71\x1b\x93\x8c\x8f\x4a\xb5\xf1\x78", + "\xb2\xaa\x48\xb3", + "\xc4\xec\xdb\xd2\xef\xb1\x76\x40\xce\x67\x07\xe2\xe9\xd0\xee\x5b\xfb" + "\x98\xb9\x15\x84\xbc\x86\xab\x38\x64\x37\xea\xa3\x7b\x0f\x2e\xb7\x05" + "\x00\x36\x11\x05\x41\x6c\x0d\xce\xcf\xf3\x89\xdc\x94\xc7\x23\xfc\xff" + "\x18\xcb\x80\x17\x40\x96\x23\x12\x00\x7a\x19\x5a\x23", + 1, 32, NULL, 0 }, + { 512, 512, 6, + "\xac\xa4\x7f\x63\x50\x94\x1a\x0e\xfd\x8c\x3b\xac\x90\x64\xa5\x54\xbe" + "\x33\x7c\xde\x7d\x19\x2f\x6f\xbf\x86\xd1\xb4\xdb\x09\xb3\x65\x31\x16" + "\x5c\xba\xe0\xa6\x34\x20\x6f\x71\xfa\x40\x0d\xf3\x33\x52\xff\xf6\x0e" + "\x1f\xba\x40\x09\xac\x66\x71\xcd\x37\x31\x2b\xdd\x98", + "\xbc\x99\x3b\x1d\xb0", + "\x89\xaf\x2f\x57\x46\xca\xb8\x9f\xda\x69\x93\xe0\x0f\x1b\xf0\xcc\x70" + "\xa7\x71\x88\x94\x5b\xb7\xb5\x40\x9b\x53\x6a\xec\x55\x33\xad\x50\x1d" + "\xb6\xec\xfa\x3e\x51\x6b\x58\x0b\x7d\xf9\xc8\xea\xdb\x3c\xf5\x56\xcc" + "\xc0\x16\x68\xbe\x98\x43\x35\xbd\x5a\x62\x55\xd5\x66", + 1, 40, NULL, 0 }, + { 512, 512, 7, + "\xb3\xec\xae\x6f\x25\xc2\xf6\x99\xf1\x58\xb3\xff\xcd\x0a\x7a\x57\x55" + "\x83\xe4\xc9\xcb\x56\xb5\xc2\x2e\xf4\x27\x3c\xde\x6c\x67\x34\xe8\x4d" + "\x74\x00\x74\x9c\x17\xe4\x7e\x8c\xfc\xca\xfa\xf8\xb5\x0c\x65\xeb\x47" + "\xdf\xeb\x27\x3d\x5d\x30\xa1\x18\x1e\x37\xb2\x7a\xd0", + "\xf0\x36\x1d\x58\x29\x1e", + "\x40\x37\xa5\x7a\xa2\x79\xb5\xa0\x7a\xbe\x93\x89\xdc\xf5\x08\xbe\x94" + "\x95\xa8\x25\x7d\xcb\x3f\xeb\xa3\xf0\x80\x1c\xd5\x75\x74\xc3\x0b\xfd" + "\xdc\x6d\xf5\xdf\x65\x67\xcd\x57\x2c\x4e\x82\x73\x5f\xd4\xe6\x7b\x65" + "\xe8\x5b\x03\x0f\x18\x3a\x7f\x44\x57\xfb\x7d\x2c\x3d", + 1, 48, NULL, 0 }, + { 512, 512, 8, + "\x70\xff\x24\xa2\x52\xd6\x51\x83\xbd\xc6\xb7\xc8\x87\x51\xf8\x50\x82" + "\x11\x41\xa6\x12\x46\x72\x7c\x32\x40\xb4\xf9\x60\x88\xae\x32\x78\x76" + "\x7a\x82\x2b\x65\x73\x5a\x28\xcc\xeb\xe4\xc8\x74\xbc\xb2\xc9\x42\x88" + "\x2c\xb2\x3f\x9d\xd8\x7f\xe0\x8f\xba\xad\x5a\xe7\x2f", + "\xe1\x8d\xa3\xeb\xf0\xff\xa4", + "\x87\x8d\x48\x87\x54\xbc\x79\x6c\x70\xe1\x1d\x5d\xb7\x7a\xcd\xa2\xe1" + "\x79\x6d\x86\x14\x6e\x27\xd8\x62\x58\x67\x40\xc4\xd4\x88\xed\x12\x23" + "\x9e\x6f\xb4\xab\x29\x25\xaf\xc8\x81\x68\x60\x9e\xdc\x04\x8f\x85\x72" + "\x53\x6f\xae\x96\xe1\x49\xd7\x3d\x23\x0b\x18\xdb\x66", + 1, 56, NULL, 0 }, + { 512, 512, 9, + "\xdd\x4e\x05\x93\x3d\x09\x71\x1e\xe8\x8c\xb4\xc1\xce\xb3\x60\x0b\x2b" + "\x33\x80\x8b\xc0\x8d\x49\x93\x87\xb3\x31\xd9\xc7\xaf\x49\xbc\x65\xb5" + "\x51\x72\xcf\x80\x83\x38\x5a\x94\x0e\x4b\x86\x4b\x7b\x4b\x73\xdd\xf3" + "\xbd\x51\x3a\x6c\xbc\xac\x73\x87\x8a\x87\x9b\x4d\x06", + "\x66\x94\x80\x29\x35\x14\x32\xc3", + "\x99\x68\xa1\x6e\xff\x2b\x4e\xee\xcb\x2f\x9d\x11\xfc\xb1\x05\xe8\xd8" + "\xca\x59\xed\x4e\x69\x13\x1c\x9d\xe5\x99\xcd\x81\x55\xfa\x4f\x33\xde" + "\xf1\x19\x5a\x6b\x45\x22\x63\xaa\xd9\x26\x5e\x16\xd4\x95\x18\x41\xd7" + "\xcd\x33\xc7\x4c\x47\x5d\xa0\x44\x97\xc0\x29\x22\xea", + 1, 64, NULL, 0 }, + { 512, 512, 10, + "\xfb\xd3\x2c\xaf\x89\x84\xfc\x43\x76\xd1\x0d\xaa\x72\x88\xdb\x8e\x6e" + "\x74\x46\x4b\xdd\x94\xb4\x48\xad\xab\x44\x97\xb3\x19\xe9\xa6\xdc\xce" + "\x54\x2f\x82\xa7\xff\x2e\x77\x5d\x12\x47\x7c\x88\x0e\x46\x0a\x9e\xab" + "\x8e\xfc\x49\xfc\xfc\x8c\x54\x76\xcb\x4b\x08\x95\x4a", + "\x38\xa2\x58\x6a\x28\x83\x95\x3c\xc4", + "\xe0\xc6\x9b\xd0\x34\xcd\xec\x5b\x48\x15\x0f\xdf\x3a\x43\x83\x45\x6a" + "\x76\x26\xd4\x40\x5d\xf5\x2d\xc6\xc2\xbc\x8f\xe9\x3b\xd8\x7e\x36\x9e" + "\x06\xa7\x81\xed\x80\xba\x8b\x1f\xe1\x14\x6c\x4d\xf8\x2b\x6a\x51\x44" + "\x12\x35\x8b\x31\xb7\x7b\x9b\x79\xc7\xa9\x1e\xc9\xe4", + 1, 72, NULL, 0 }, + { 512, 512, 11, + "\xfd\x4c\x3f\x6b\x21\x37\x51\x36\x16\xc2\x8e\xd4\xd8\x63\x8f\x86\x7a" + "\xd0\xb9\x71\x88\xb7\x3f\xc9\xb3\x6f\x3d\x52\xb8\x2d\x72\xa4\x9b\x9d" + "\xc1\xb8\xb2\x53\x97\xeb\x44\x80\x54\xa8\xd3\x8d\x83\x8e\x7a\x88\xb4" + "\xdf\x9c\x26\x3a\xea\x1b\x96\x87\x71\xd5\xac\x57\x56", + "\x86\xb4\xe6\x1b\x3b\x7d\x65\x00\x44\xad", + "\x29\x34\x5d\x7d\xa4\x4e\x2f\x22\x8e\x8d\x50\x2e\x29\xfb\x65\x5d\xa3" + "\x67\x6a\x48\x1f\x99\x47\xc8\x48\x25\x02\xce\x07\x0b\x3d\xa5\x06\x55" + "\x89\xd8\x4c\x02\xa0\x5c\xd7\x74\xb4\xbd\x5a\x15\xb6\x68\xc5\x9b\xaf" + "\xc1\x92\x69\x5a\xec\x43\xe5\xdf\x3a\x82\x30\x17\x45", + 1, 80, NULL, 0 }, + { 512, 512, 12, + "\xf9\x5b\xae\xa5\x35\xf4\x77\xd2\x2b\x40\x5c\x67\xd9\x27\xf5\x9a\x9e" + "\x04\x2c\x46\x29\x7a\x16\x81\xbc\xc1\x6f\xdb\xe1\xb2\xcd\x59\x67\x5a" + "\x22\x13\x51\xa7\x80\x75\x98\x1e\x7e\xb4\x99\x80\x66\x76\x88\x01\xcb" + "\xd7\xa8\x52\x31\x11\x4d\x7f\x27\xf9\xbd\xf2\x48\x99", + "\x5a\x34\xde\xe4\xe0\x98\x2d\x45\x8e\xff\xfb", + "\x63\x86\x7b\xb3\xe8\x2b\xd4\xa5\xf7\x15\xb3\xdd\x67\xba\x36\x25\x66" + "\x6e\x45\x8c\x5e\x3d\x75\x80\x47\x09\xf8\x0b\x6d\xde\x6f\x77\x4e\xa2" + "\x23\xba\x9e\x25\x36\xc6\x0a\xb6\x36\xdd\x12\xd0\x7b\x21\x72\x34\xa4" + "\x90\xea\x9c\xae\x4f\xe6\x73\x21\x5d\x33\xf8\xc5\x7a", + 1, 88, NULL, 0 }, + { 512, 512, 13, + "\x4d\x76\xae\x95\xa1\x23\x20\x7e\x01\xc6\xd2\x2d\x8b\x58\x7e\x63\xba" + "\x68\x29\x63\xe5\x09\x61\xaf\xff\x53\x11\x60\xa9\xb9\xaa\xc6\xc7\x72" + "\xc5\xe8\xbf\x91\x8d\xde\xcb\xeb\x56\x45\x5e\xa6\x47\x10\xe5\x1a\xc2" + "\x1e\x3b\xb9\xaf\x4b\x24\xea\xa8\x53\x5b\x3c\x29\x24", + "\x2c\x31\xf2\xd9\x86\xf6\x8a\x6d\x6a\x96\xc4\xb0", + "\x9d\x4f\x95\x49\xac\x13\x4a\x6f\x60\xf1\x7f\xd0\xfb\xc8\x0f\x55\x42" + "\x6a\xfa\x73\xcd\xaf\x84\xa8\x06\xd9\x8d\xff\xfc\x94\x26\x31\x78\x11" + "\x6f\x76\xaa\xdc\xa9\x5a\x92\x43\xa9\x12\x8f\x5f\x66\xd3\xe7\xf3\x3e" + "\x72\x60\x3d\x4b\x35\xab\x90\xab\x7d\x1e\x87\x0a\xd7", + 1, 96, NULL, 0 }, + { 512, 512, 14, + "\x0d\xa7\xfa\x1f\x5d\x21\x79\x51\xe3\xe3\x43\xcd\xa8\x1f\x23\x2d\xeb" + "\x71\x76\x4e\xb4\x9e\x85\x10\xbc\x28\xdb\xa8\xeb\x62\xaf\xa2\xa9\x8b" + "\x6f\x05\x36\xad\xb1\x02\x50\xc7\x48\x78\xfe\x64\x9f\x47\xbb\xaf\xdf" + "\x3f\x72\x2f\xa1\x50\xf6\x6e\x83\xf6\x5f\x60\x6a\xb0", + "\x83\x51\x1d\xe1\x90\x66\x3c\x9c\x42\x29\xac\xe9\x01", + "\x11\xbd\x76\xba\x2f\xd5\x68\x4e\x3f\xaa\xdd\x44\xab\xc0\x5d\x32\x66" + "\x14\x72\xae\x4c\x75\xfd\x69\xe6\x2e\x47\xa2\xd4\x62\xe4\x83\xab\x5f" + "\xd3\x74\x07\x0e\x64\x80\x17\x25\x09\x34\xd4\x86\xfe\xd5\x5e\x68\xf4" + "\x33\x85\x47\xfb\x5d\xc5\x4d\x4b\xed\x89\x4c\x1c\x2f", + 1, 104, NULL, 0 }, + { 512, 512, 15, + "\xce\xc9\xe9\xf2\x5e\xd9\xa0\x17\x00\x4a\x78\x82\xb1\xe4\x4e\x8b\xd8" + "\xfa\x32\x03\xc5\x0c\xb6\x05\x84\x55\xed\x4f\x2a\x03\x67\x88\xd4\x6f" + "\xcd\x32\x83\x27\xd0\xd8\x6b\x1a\xba\xe6\x9f\x7b\xbb\x96\xe3\xd6\x63" + "\x73\xec\x8b\xd4\x50\x75\x89\x08\x79\xa8\x3f\x4d\x33", + "\x80\xdc\xd8\xba\x66\xf9\x8b\x51\x09\x41\x44\xe9\xb8\xbd", + "\xc6\x9f\x17\x87\xbf\x78\x04\xbf\xff\xd9\xda\x7e\x62\xf5\x8c\x1c\x9f" + "\x59\x9c\xca\xe2\xed\x4f\xc6\xab\xda\x1b\xe4\x86\x20\xaf\xc7\x97\xd5" + "\x9d\x4a\xdb\x39\x6e\x1f\xa5\xd1\x8b\x8c\x1a\xa1\xc7\xc1\x52\x18\xa9" + "\xf9\xe3\xaa\xb2\x26\x11\x9a\xda\xd7\x42\x64\x10\x89", + 1, 112, NULL, 0 }, + { 512, 512, 16, + "\xbb\xe2\x56\x49\xec\xdf\x54\xae\x00\x28\xfb\x92\x3c\xc8\xc2\x8e\xc0" + "\x0e\x10\xe2\xd4\x42\x14\x59\x07\x81\x23\x8a\x14\x3b\x75\xd5\x4e\xfb" + "\x03\x7e\xb9\xf5\x30\x82\xa8\xab\x3d\x88\x76\xda\xf4\xdb\xdc\x24\x83" + "\xc4\xba\x22\x27\x97\xfe\x20\xda\x3b\x77\x30\x36\x8b", + "\x33\xf6\x30\x08\x8c\x0d\x24\xcd\xa9\x8c\xaf\xf1\xa3\xaf\xc7", + "\xc8\x03\xca\x83\x3e\x85\x14\x18\xa3\xd9\xed\x76\x4f\x8c\x83\xf4\x81" + "\x06\x01\x41\xeb\x1b\x2b\xf6\x4d\x7e\xe7\x99\x1b\x04\x1c\x48\xbf\xc7" + "\x47\xbc\xe1\x3d\x69\x72\x2f\x63\x94\x40\x85\xce\xf8\xe7\xa1\x66\x27" + "\x05\x30\xfe\x31\xa2\xa5\x25\xa9\x9b\x8a\x75\xf1\xb1", + 1, 120, NULL, 0 }, + { 512, 512, 17, + "\xf5\xe2\xb9\xe2\x31\x3f\x4f\x80\x7c\xb3\xa9\x24\xa7\xd4\x94\x3f\xc3" + "\xfb\x47\x5d\x8f\x1a\x1b\x40\xce\x09\xa3\x77\x70\xf6\x21\xaf\x89\x77" + "\x72\x9c\xad\xf9\x86\xc9\x8c\x75\xf0\x8a\x4f\xab\x42\x80\x53\x8e\x09" + "\xe7\xe5\x1e\x87\xa8\xd6\x2c\x03\x41\x1b\xdb\x8d\x24", + "\x74\xef\x62\x3c\x83\x27\x5a\xe9\x97\x45\xbf\xf7\xe6\x14\x2a\xfa", + "\x47\x10\x55\xf7\xa2\xd4\x47\x58\xe7\xd7\x83\x7d\xb8\x5c\x33\x62\x6b" + "\x83\x06\x76\x0e\xb4\x5e\x18\xd4\xba\x8d\xfb\xcd\x0d\x42\x79\xfc\xf8" + "\xb5\x39\xef\x7b\x16\x5e\xea\xbf\x54\x57\xee\x2c\x41\xe5\x2d\x07\xe9" + "\x12\x1d\xa0\x2c\x98\x8f\x08\x16\x2f\x86\xbd\xf2\x08", + 1, 128, NULL, 0 }, + { 512, 512, 18, + "\x8e\x32\x3d\x5f\xb4\x75\x2d\x92\xa6\xd9\x05\xc5\x12\xb2\x87\xd0\x7b" + "\x21\xae\x50\x00\x2d\x02\x6f\xf0\x38\x8e\x15\x93\xbd\xe9\x99\x8d\xd0" + "\x23\x21\xe2\x00\xd1\x48\xf5\xfa\x2e\x82\x4b\x37\xe9\xf5\xa7\x74\x41" + "\x79\x4b\x84\x0b\xed\xd5\x52\xd1\x05\x1c\x1d\xdd\x8c", + "\x4d\xaa\x22\x9b\x00\x9b\x89\x84\x35\x4c\x2e\xc3\xe7\x97\x3e\x00" + "\x42", + "\x93\xa2\x13\x7c\xc8\x4e\x2f\xa1\x43\x9d\x7c\x23\x97\x67\xb3\xce\x65" + "\x3d\x63\x4c\x58\xa4\x59\x0e\xb6\x1a\xf9\xd3\xef\x98\x64\x45\x22\x0a" + "\xff\x35\x54\xde\x45\xa1\xb0\x93\x3f\xa0\x6d\x3d\x64\x46\x04\x18\x91" + "\x09\x77\xd8\xd9\xdd\xb2\xeb\x04\x96\x3c\x81\x68\x41", + 1, 136, NULL, 0 }, + { 512, 512, 19, + "\x46\x5b\xc1\xab\x21\x25\xcc\xa2\x97\x29\xd0\x1d\xf0\x44\xe3\x93\xb0" + "\x67\x7d\xef\xdd\x93\x92\x80\xa3\xaa\x14\x12\x24\xef\xa0\x64\x57\xe6" + "\x23\x05\x6d\x02\xf6\xc3\x6e\xca\x3d\xfc\x4a\x74\x76\xdd\x36\xb9\x7d" + "\x0c\x2d\x60\xc7\x67\x21\x29\x18\x9e\x73\xb6\xaf\x8f", + "\xdd\x84\x59\x9b\x47\xba\x9a\xe9\xf2\xad\x0c\x8e\xac\x67\x84\x85\x43" + "\x3e\xb6\xb1\xdf\xb7\xc9\x98", + "\x9f\xff\x43\xa8\x3c\x71\x83\x32\x11\xf9\xd6\x0e\xee\xf4\x16\x69\x65" + "\xc4\x1a\x37\xc7\x66\x34\xb1\xbd\xf9\xc5\x29\x1d\xf7\x5d\xc8\x77\x66" + "\x8f\x22\x87\xbc\xf8\x10\x8e\xa9\xe0\x3d\x06\x1a\x70\x8d\xb2\xdb\x08" + "\x68\x7e\xda\x61\xfa\x97\xb1\xca\x92\xdc\xf2\x2b\x92", + 1, 192, NULL, 0 }, + { 512, 512, 20, + "\xb9\x02\x26\x79\x8d\xff\x2f\xfb\x91\xd1\xee\x41\x03\xf2\x63\x97\xd0" + "\xbf\x84\xc1\x3c\x1e\xc7\x17\x39\x2c\x5f\xe1\xd4\xd0\xf4\xdc\x79\x02" + "\x36\xd7\x59\xfa\x1b\xe8\x52\xe3\x05\xda\x58\x5a\x3d\xbd\xe0\xd3\x91" + "\x2b\xea\x60\xd6\xb1\x40\xc2\x56\x45\xeb\x00\x94\x3f", + "\xaa\x29\xc3\x72\xf1\x36\x99\x3c\x65\xac\xe5\xe1\xd6\x20\x78\x80\x6e" + "\xb7\x87\x91\x3b\xb3\x5a\xf3\x33\x71\x05\x63\x59\xd3\x54\xb2", + "\x49\x3a\x72\x75\x36\xb0\x7d\x43\x4a\x7f\xc8\xdf\x6b\x70\x98\x91\x48" + "\xa8\xd9\x4c\xad\xb9\x76\x1a\xd8\x45\xac\x5f\xde\x20\x68\xf9\x56\x5e" + "\x68\x60\x7b\x53\x1b\x0f\x30\x7d\x7c\x17\xce\x0a\x2b\xa6\x9f\xb1\xac" + "\x1b\x0c\x71\x6f\x93\x90\x4e\xec\x75\x66\x9e\x70\xb7", + 1, 256, NULL, 0 }, + { 512, 512, 21, + "\xaf\x1b\xb9\x17\x75\xcb\x40\xc7\x39\x83\xf1\x19\xc9\x27\xa2\xce\x8f" + "\x7b\x95\x4a\x62\x74\xec\xc1\xcd\x96\x01\x9e\x5c\x41\x7a\xf4\xb0\x94" + "\x37\x61\x94\xea\xe7\x1c\x7f\x68\xf3\x34\x56\x54\xd5\xd9\xf8\x19\x8a" + "\x69\x7b\x41\xae\x25\x1e\x82\x30\x8a\xcc\xd9\x35\xbd", + "\x75\xed\xed\xdf\xa7\xf1\xdf\x1d\xc1\x44\xfb\x19\x5b\x27\xe4\x54\x64" + "\x0e\x3f\x89\x7c\xb5\x64\x22\x2f\x05\xe8\xaa\xb0\xc6\x02\x4f\x90\x47" + "\x2a\xfe\xa6\xe7\x25\x4e\xd2\x51\x34\xea\x43\x45\x2a", + "\xb5\x3d\x56\x40\x86\xa7\x45\xb1\x0d\x88\xa4\x8b\x50\xed\x8b\x53\xf4" + "\xc8\x3f\xd1\x2b\xf5\x6a\x75\x10\x80\x74\xde\x9b\x34\x3c\xdf\x06\x68" + "\xce\x8b\x6a\x3d\x88\x4b\xa2\xda\x5f\x4c\x95\x7f\x13\x19\xe2\x6c\x08" + "\x13\xc9\x9a\x42\x69\xc1\x71\xad\x80\x98\x10\x13\xa2", + 1, 376, NULL, 0 }, + { 512, 512, 22, + "\x51\x3e\x0e\x76\x22\xea\xbc\xb6\xbf\xc8\x16\x69\xda\xc9\x03\xdf\x46" + "\xda\xea\x12\x40\xf3\x22\x48\xbb\xf4\xfc\x61\xf1\xf9\xb1\x3b\x2c\x3f" + "\xe1\xbc\xc9\x75\x40\xd3\x00\x65\xbe\x9e\xee\x41\xe5\x17\x48\xbc\x42" + "\xc1\x6a\x8c\x82\x69\xfb\xe2\xb6\xf6\x25\xc1\x92\x28", + "\x81\xd8\x65\x09\x37\xf5\x08\x71\xa6\x6a\xf7\x16\x05\xea\x4f\xa9\xd6" + "\xc5\xd7\xa3\x75\x77\x4c\x22\x80\xeb\x34\xae\xfc\xee\x8c\x0e\xf8\x33" + "\x45\xbc\x54\x7e\x4d\xe7\xcb\xea\x48\x23\x69\xb2\x5a\x93", + "\x9d\x94\x2e\x45\x85\x74\x2b\xa1\x18\xbd\xa6\xe1\x32\x51\x0a\xf3\xb9" + "\x29\x70\x47\xd3\x64\xf7\x6b\x2a\x0d\x1f\xc8\x03\x84\x9b\x06\xcc\xac" + "\x0e\xaa\x42\x79\x34\x05\x5c\x9d\x2e\x5a\x5d\xa1\x9c\xf1\x72\x99\xff" + "\xda\xb6\x50\x89\x58\x0d\x10\xff\x72\x07\xc9\xed\x03", + 1, 384, NULL, 0 }, + { 512, 512, 23, + "\x62\x7c\x9a\x72\x24\x7d\x07\xb0\xce\xc8\x34\x62\x77\x46\x83\x11\xc7" + "\x40\x1f\xc4\xce\xca\xea\x8e\x22\xe1\x3e\xce\x4b\x35\x2c\x8f\x7a\x7e" + "\xb1\xba\x81\xce\x34\x8a\x08\x67\x04\x38\xc9\x7b\x8d\x9e\x88\x36\x14" + "\xd5\x50\xf1\xff\x16\xd6\x36\x97\x5c\x59\x98\x8c\x2d", + "\x11\x8e\x04\x68\xcb\xb5\x2f\x93\xa3\x39\x6e\xbf\xaa\x11\x48\x81\xa9" + "\x8a\x41\x01\xf4\xff\x91\x2c\xed\x47\xec\xfc\x73\xb2\x7f\x52\x20\x5b" + "\x7a\x5d\x4f\x38\x99\x50\x6f\x9e\x34\xeb\xf9\x94\x60\xda\x7a", + "\xa1\x86\xe0\x8c\x77\x31\xd4\xbb\xb1\xd5\x34\x2a\x10\x5e\xf4\x8f\x53" + "\x53\xc5\xc5\x42\x27\x7d\xe6\x07\x83\x1f\xcb\xbc\x8d\x0b\x9f\xd5\x09" + "\xc7\x4b\xf9\xe3\x52\xee\x73\x97\x92\xee\x3c\xd6\x38\x2f\x96\xe7\x0a" + "\xdb\x58\x9f\xdf\x1f\xb0\x31\xd4\x3e\xef\x1a\x59\x5f", + 1, 392, NULL, 0 }, + { 512, 512, 24, + "\x1e\x98\x1d\x0c\xbb\xad\x5b\xea\x94\x80\xd8\x36\xb4\x70\x4b\xf3\x14" + "\x76\x63\xb6\xea\x59\xe1\xe0\xa2\x80\xfb\x45\xd9\xb8\x5d\x44\x5d\xc9" + "\x72\x15\x9d\xde\x30\x1c\x6f\x1e\x66\x68\x1f\x95\x64\x2d\xbb\x9a\x92" + "\x18\xc0\x0d\x0c\xd7\x24\xcb\x02\xf3\xbc\xae\xa2\xea", + "\x44\x0d\xff\x39\x06\x88\xc9\xfd\xe3\x1c\x17\xfd\xb6\x1c\x1d\x13\x89" + "\x9f\x95\x44\xa9\x86\x32\x4c\x34\xd5\xeb\x07\xbe\xf9\xa4\x43\x62\x97" + "\xf4\xa7\xfe\x16\xde\x5d\xd7\xb2\x4e\x0c\x7c\x12\x90\x51\xef\xe6\xf2" + "\xdd\x0a\x21\xae\xc0\x5c\x3e\x3c\x8f\x6f\xa3\x0d\x9c\x0c\xbd\x60\xd8" + "\x40\xd1\x4f\x0b\x2a\x92\x8b\xc7\x18\x9b\x9d\xe4\xa6\xa7\x31\x15\x1d" + "\x6b\x31\xe6\xa0\xec\xae\x75\x09\x54\x34\x73\x7b\xe8\xc3\xdb\x11\xa6" + "\xa6\x97\xd0\x61\x6c\x78\xb9\x70\x41\xde", + "\xc5\x2e\xb5\xd1\x8e\x90\x68\x72\x48\x34\x2a\x84\xdc\x02\x41\xc6\x80" + "\xe9\x92\xb8\x8b\x14\x09\x27\x5d\xf7\xe3\x47\xc9\x91\x69\xa5\x0c\xd7" + "\x80\xeb\x47\x26\xad\x75\x9e\x2a\x02\x7f\xb0\x91\x35\x4e\x3d\x7c\x7a" + "\xba\x8a\x21\xf8\xac\xd1\xd0\xe2\x12\x36\xaf\x5f\x98", + 1, 896, NULL, 0 }, + { 512, 512, 25, + "\xee\x8a\xea\x2a\x52\xeb\x7e\x0c\x11\x20\xab\x73\x6b\x1a\x82\x5b\x12" + "\x61\x00\x63\xde\x96\x42\xc5\x94\x76\x6c\x02\x0c\xb8\x73\x14\xd8\xac" + "\x94\xb1\x30\x72\xbf\xbf\x3c\x01\x9b\x4a\xac\xb1\xd2\x69\x5c\xdd\x75" + "\x63\xa2\x6f\x57\x4e\x12\x55\x99\x06\x78\x4d\x85\x3c", + "\xa3\x95\x1f\x1d\x18\x13\x56\x02\xfd\xad\xce\xee\xf5\x74\x1c\x24\xad" + "\x22\x75\x61\x60\xd0\xc5\x5e\x51\xb7\x88\xaf\x95\x2a\xda\xeb\x13\xe1" + "\x8c\x24\xc6\xb0\x96\x72\xf4\x05\xd7\xec\x3d\x49\xb0\xbd\x86\xc7\xf8" + "\x69\x1b\x6f\x69\xaf\x49\x17\x54\x23\x21\x5c\xf5\x7d\x7c\x08\xa5\x4a" + "\xb0\xb0\x29\x3e\x68\x5c\x9a\xa2\x50\xf1\x59\x9d\x78\x19\x3a\x00\xaf" + "\x82\x2d\xec\x4b\x56\xfd\xb4\x1f\x03\x43\xab\x2c\xf8\x5e\xa2\x7b\xb2" + "\xe6\x50\x93\x0f\x5e\x8c\xa8\x36\x83\x39\x03\xb0\x53\xb3\xe0\x68\x99" + "\xb4\x01\x2a\x65\x32\x97\x8d\x90", + "\xd3\x67\x8c\xa7\xc5\xc1\xaa\x21\xf1\x2e\xcc\xc2\x1a\x1a\xdd\x0b\x3e" + "\xb1\x2c\xcd\x13\x40\x33\x57\x04\x68\x19\x1e\x51\xb0\x58\xc6\x1f\x2a" + "\x7d\x88\xf2\xca\x6c\x65\x2c\x29\xc6\x5c\x49\x1b\xf1\xf0\x25\x2b\xc1" + "\x57\xbd\xd7\x74\x36\xff\x55\x20\x4e\xac\x6d\xfb\x0d", + 1, 1016, NULL, 0 }, + { 512, 512, 26, + "\xec\xd1\x86\x1a\x12\xea\xee\x48\xae\xf1\xd7\xed\x27\x82\x23\xb5\x0d" + "\x34\x16\xdb\xff\x81\xe9\x76\xc5\x6e\xcd\x4b\x1a\x1b\xc8\x89\x2b\x58" + "\x4c\xbc\xc7\x23\x70\xff\x5e\x97\x6a\x6a\xf1\x79\x0c\xaa\x32\xf9\xea" + "\x91\x28\x55\x91\x4c\x03\x15\x97\x95\x78\xfb\xf1\x65", + "\x57\x79\xc5\x63\x73\xa8\xe5\xdb\x43\xbd\x65\xc0\x45\x3c\xe2\x31\x44" + "\x23\x0d\x43\x66\x6d\x71\x7a\x3b\x59\xd2\xe9\x0f\x0e\x10\x73\x23\x76" + "\x83\x1d\x72\x81\xcb\x23\xdd\x55\x66\xe5\xf8\xc6\x27\xd0\x0d\x39\x65" + "\x01\x39\xce\xb8\x7c\xd4\x7e\x92\x1d\x65\xd6\xc1\xcc\x77\x12\xac\x4b" + "\xd7\x5b\xda\x88\x28\xe6\x8a\xbc\x96\x8f\x41\x60\xed\x91\xb2\x89\x46" + "\xc9\xd7\x06\xb0\x36\x0b\xbb\xdd\x65\xf4\x7e\xf9\x98\x3c\x50\xf2\xd0" + "\x9d\x05\xc3\x67\x4c\x09\x43\xea\x4a\xf5\x4c\x38\x10\x89\xf9\xb8\x46" + "\xdd\x69\xce\x90\x8e\x0f\x6e\xaa\xaf", + "\xd3\x77\xe4\xef\xc3\x9f\x25\xca\x75\x14\x52\xe7\x9d\xcb\x56\x61\xf8" + "\xad\xcc\x06\x57\x0b\xd3\xf7\x10\xe0\x38\x54\xe0\x32\x28\x6c\xa4\x77" + "\xe6\xa6\x20\x64\x79\x58\xfd\x31\x70\x64\x63\xb5\x42\xdd\xf6\x17\x75" + "\x78\x75\xf3\x49\xc6\x11\x09\x35\x8d\x04\xf6\xdc\x58", + 1, 1024, NULL, 0 }, + { 512, 512, 27, + "\x71\xaa\xdb\xf3\x30\xea\x13\x3b\x46\xc9\x39\xd1\x2e\x60\x38\x96\x90" + "\x2e\x8d\xf6\x38\x59\x7c\x98\x87\x2d\xfb\x5a\xec\xd5\x16\x1b\xc8\x40" + "\x95\x22\x1d\xe3\x22\x23\x67\x01\x2f\x45\xc6\xd7\x07\x01\xe8\x62\xab" + "\x00\x0e\x78\x2e\x91\xb5\x05\xb2\x1b\x4e\x21\x2c\x38", + "\xe6\xd7\xb0\x28\x0d\x2f\x7d\xf8\x3f\xd2\x65\x62\xfc\xde\xa2\x59\x7c" + "\xf6\x87\xa9\xc9\xfa\x19\x4f\x65\x5c\x44\xd3\x27\x1b\x88\x1f\x28\xad" + "\xc4\x36\xdb\x8e\x04\x37\xff\x4d\xc5\xd3\x83\x56\x27\x1c\x33\x88\x29" + "\xc3\xe2\xd9\xba\x4a\xc1\x77\x7c\x94\x88\x69\x83\xd4\xb7\x2c\x27\x5b" + "\xc0\x0e\x4f\x7b\x06\xc5\xce\x38\xa2\xfe\x54\x9f\xe5\x37\x61\x85\x7f" + "\x23\x6d\xa7\x05\xfd\x03\x79\x0b\x41\xcc\x6f\x75\x9f\x41\xaa\x20\x6f" + "\xec\xa7\xba\x54\x86\xf4\xfc\x9d\x09\xf3\x5c\x8e\x08\x87\x24\x12\x91" + "\x88\x20\x10\x41\x4a\xe4\x1b\x8b\x38\x4a\x71\x5a\x40\x9b\xe1\x3d\xa1" + "\x7b\xfd\x60\xd3\xfb\xd4\xb8\xcb\x3c\xc7\xc2\x60\x43\x80\x72\x64\xa2" + "\x0b\x9a\x5c\x02\x72\x5e\x74\x2f\xff\x03\xe1\x80\x6b\x38\xaf\x35\x7e" + "\xbf\x8c\x79\xfc\x4c\x38\xb0\x07\xbf\x06\x13\x28\x6c\xf0\x63\xe4\x54" + "\x82\x37\x54\x75\xe6\xc4\x26\xd4\xf7\x00\x57\xcd\x92\xef\xcb\x2d\xfe" + "\x86\xe4\x5b\xde\xa3\x99\x27\x3a\x5e\x0f\x14\x22\x21\xfa\xe2\x06\x80" + "\x05\x55\xc0\x1b\x18\x53\x32\x95\xf5\x77\xe2\x3a\x9a\x7a\x0a\xa0\x72" + "\x82\x30\x02\xb9\x09\x65\x01\x17\x4d\x3b\xc4\xaa\xc3\x3e\x0d\xc6" + "\x00", + "\x0c\x1c\xbb\x2f\x19\x6d\x3d\x1a\xf5\xf9\x82\xa3\x30\xbf\x1d\x9a\xcc" + "\xaa\xda\x72\xcf\x6c\x25\x46\x58\xcb\x32\xbf\xd8\x70\x54\x81\xab\xd2" + "\xe1\x63\xa7\x33\x38\x70\x0f\x0d\x96\x1c\xa0\x2a\x31\xb6\x00\xdf\x04" + "\xfa\xf3\x11\xcd\x06\x49\x85\x57\x83\x11\x02\xf8\x0f", + 1, 2040, NULL, 0 }, + { 512, 512, 28, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 29, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 30, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd0\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 31, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbd\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 32, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x52\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 33, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3f\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 34, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9a\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 35, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x31\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 36, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\xbf\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 37, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\x3f\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 38, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x86\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 39, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb4\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 40, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x85\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 41, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb7\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 42, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 43, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 44, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf2\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 45, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x22\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 46, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\x73\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 47, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\xa3\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 48, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x61\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 49, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xd1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 50, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x08\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 51, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x67\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 52, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc6\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 53, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x49\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 54, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc5\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 55, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x4a\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 56, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\x47\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 57, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\xc8\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 58, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8d", + 0, 0, NULL, 0 }, + { 512, 512, 59, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x53", + 0, 128, NULL, 0 }, + { 512, 512, 60, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8e", + 0, 0, NULL, 0 }, + { 512, 512, 61, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x50", + 0, 128, NULL, 0 }, + { 512, 512, 62, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\xcc", + 0, 0, NULL, 0 }, + { 512, 512, 63, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x12", + 0, 128, NULL, 0 }, + { 512, 512, 64, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x0c", + 0, 0, NULL, 0 }, + { 512, 512, 65, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\xd2", + 0, 128, NULL, 0 }, + { 512, 512, 66, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9b\x9e\x3f\x87\x80\x96\x86\xf2\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 67, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x30\x1c\xbf\xb5\x66\x72\x0e\x22\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 68, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\xbf\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 69, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\x3f\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 70, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\x2b\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62\x12\x2e" + "\x58\x27\x2a\x31\x67\x97\x20\xb2\x54\xcb\xd6\x3a\x7c\x6d\x69\x6b\xf9" + "\x28\x3f\x9c\x68\x97\xe7\xd7\x92\x48\x3b\xb0\x38\x8c", + 0, 0, NULL, 0 }, + { 512, 512, 71, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x16\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3\xe7\xe2" + "\xdc\xa7\xb0\x11\xbf\x4c\xec\x4c\x7e\x7d\x6c\xc4\x1b\xc1\x0c\x3b\xe3" + "\x6e\x83\x20\xc5\x0a\xaf\x6c\x35\xf0\x4a\xc8\xca\x52", + 0, 128, NULL, 0 }, + { 512, 512, 72, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x2d\x64\x61\xc0\x78\x7f\x69\x79\x0c\xbe\xf6\x04\x38\xe7\x29\x54\x44" + "\xf6\x3d\x87\x30\xfa\x5d\xf9\x52\x0d\xeb\x9c\x1e\xe8\xfc\x9d\xed\xd1" + "\xa7\xd8\xd5\xce\x98\x68\xdf\x4d\xab\x34\x29\xc5\x83\x92\x96\x94\x06" + "\xd7\xc0\x63\x97\x68\x18\x28\x6d\xb7\xc4\x4f\xc7\x73", + 0, 0, NULL, 0 }, + { 512, 512, 73, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\xcf\xe3\x40\x4a\x99\x8d\xf1\xdc\x0e\x99\x1d\xb7\x9a\x3c\x69\x0d" + "\xe9\xe6\x38\x3e\xaf\xcc\x33\x91\x71\x44\x33\x73\xa3\xa4\x5c\x18\x1d" + "\x23\x58\x4f\xee\x40\xb3\x13\xb3\x81\x82\x93\x3b\xe4\x3e\xf3\xc4\x1c" + "\x91\x7c\xdf\x3a\xf5\x50\x93\xca\x0f\xb5\x37\x35\xad", + 0, 128, NULL, 0 }, + { 512, 512, 74, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 512, 512, 75, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 512, 512, 76, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 512, 512, 77, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 512, 512, 78, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x52\x1b\x1e\xbf\x07\x00\x16\x06\x73\xc1\x89\x7b\x47\x98\x56\x2b\x3b" + "\x89\x42\xf8\x4f\x85\x22\x86\x2d\x72\x94\xe3\x61\x97\x83\xe2\x92\xae" + "\xd8\xa7\xaa\xb1\xe7\x17\xa0\x32\xd4\x4b\x56\xba\xfc\xed\xe9\xeb\x79" + "\xa8\xbf\x1c\xe8\x17\x67\x57\x12\xc8\xbb\x30\xb8\x0c", + 0, 0, NULL, 0 }, + { 512, 512, 79, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3f\xb0\x9c\x3f\x35\xe6\xf2\x8e\xa3\x71\xe6\x62\xc8\xe5\x43\x16\x72" + "\x96\x99\x47\x41\xd0\xb3\x4c\xee\x0e\x3b\x4c\x0c\xdc\xdb\x23\x67\x62" + "\x5c\x27\x30\x91\x3f\xcc\x6c\xcc\xfe\xfd\xec\x44\x9b\x41\x8c\xbb\x63" + "\xee\x03\xa0\x45\x8a\x2f\xec\xb5\x70\xca\x48\x4a\xd2", + 0, 128, NULL, 0 }, + { 512, 512, 80, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9a\x9f\x3e\x86\x81\x97\x87\xf2\x40\x08\xfa\xc6\x19\xd7\xaa\xba" + "\x08\xc3\x79\xce\x04\xa3\x07\xac\xf3\x15\x62\xe0\x16\x02\x63\x13\x2f" + "\x59\x26\x2b\x30\x66\x96\x21\xb3\x55\xca\xd7\x3b\x7d\x6c\x68\x6a\xf8" + "\x29\x3e\x9d\x69\x96\xe6\xd6\x93\x49\x3a\xb1\x39\x8d", + 0, 0, NULL, 0 }, + { 512, 512, 81, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x31\x1d\xbe\xb4\x67\x73\x0f\x22\xf0\x67\xe3\x49\x64\xc2\x97\xf3" + "\x17\x18\xc6\xc0\x51\x32\xcd\x6f\x8f\xba\xcd\x8d\x5d\x5a\xa2\xe6\xe3" + "\xdd\xa6\xb1\x10\xbe\x4d\xed\x4d\x7f\x7c\x6d\xc5\x1a\xc0\x0d\x3a\xe2" + "\x6f\x82\x21\xc4\x0b\xae\x6d\x34\xf1\x4b\xc9\xcb\x53", + 0, 128, NULL, 0 }, + { 512, 256, 82, + "\xee\xf6\xbc\xf1\x6e\xf7\xae\x17\x32\x6a\x33\xf2\x2d\x14\x06\xec\x1b" + "\xd3\xf8\x66\x50\x5f\x4b\x2e\x4f\xe8\xb4\x5b\xd6\x2c\xcb\xd8\x50\x32" + "\xa9\x89\x9f\xac\xf2\xdb\x0c\x93\xa2\x34\x5c\xb8\x89\x2a\xfb\x74\xdb" + "\x54\x97\x81\x21\x1d\xd8\x88\x1a\x8c\x8e\x25\xc1\x71", + "", + "\x75\xf6\x97\x5e\x35\x00\xbe\x4f\xbf\xee\x1b\xc9\x56\x44\x74\x5c\xe9" + "\xf8\xd4\x7b\x6f\x38\x18\xa4\x8f\xf3\x4e\x8c\x2b\x18\x6b\xa6", + 1, 0, NULL, 0 }, + { 512, 256, 83, + "\x83\x86\x96\xe6\x19\x0c\x87\x4c\x37\x17\xb8\xbe\x0c\xf0\x63\xca\x6d" + "\x60\x76\x09\x87\xd1\xa3\x37\x03\xe7\xe3\x5e\xb1\x73\xe5\xae\x95\x4e" + "\x74\x1a\x37\x93\x51\x39\xd6\x12\x14\x9e\x76\xf6\xab\x2a\x37\x06\x04" + "\xf5\xb4\xa6\x8b\xee\x87\xe3\x09\x24\x0a\x9b\xa3\xd6", + "\xd8", + "\x5b\xcf\x44\x53\x9d\x87\x83\xbb\x70\x8e\x7f\x5d\xaf\xc4\xd6\x83\xbc" + "\xaa\x0d\x24\x0c\x90\x26\x75\xbd\xde\x05\x9f\x94\x4d\xac\xde", + 1, 8, NULL, 0 }, + { 512, 256, 84, + "\xfa\x99\x9e\xd1\xcf\xbc\x5c\x43\xaf\xb1\x6f\x22\xd0\x24\xe3\xce\x64" + "\x5e\x00\xb0\x67\x12\xc9\x3b\x94\x61\x67\xc9\xc2\xc0\x37\xd1\x92\xf0" + "\xf3\x00\x3f\x87\xc4\x3a\x71\x16\x6f\xe1\xa3\xc5\x82\x4c\x34\x86\x73" + "\xa2\xf0\xf3\xc4\x75\x70\x69\x85\x94\x0f\x6b\x02\xa2", + "\xca\xd2", + "\x9a\x93\xf4\x72\x8a\xa5\x94\x1d\xa1\x60\xec\x70\x7f\x14\xb7\xe9\xee" + "\x1e\x76\x8c\x7f\x62\x72\x69\x54\x34\x30\xd2\xfc\x68\x1e\x90", + 1, 16, NULL, 0 }, + { 512, 256, 85, + "\xb5\x3f\xce\xd8\xb7\xb4\xaa\x59\xd3\xb5\x6d\x91\xe1\x38\x67\x63\xc3" + "\x9e\x35\x1c\x2f\x5a\xd6\xa4\x88\x5e\x44\x2a\xd8\x94\xd5\x18\x1c\x5b" + "\xfe\x5c\x05\x28\x0a\x84\xad\x19\xd7\x58\xe3\x59\xbf\x81\x71\xfe\x65" + "\x29\x88\xfc\xf9\xd1\x45\x8e\xa1\x73\x64\xca\x8f\xa9", + "\xd3\x39\x3c", + "\x02\xc4\x96\x8e\x86\xd1\xc6\x28\x37\xa4\x16\x50\xd3\x19\x9c\xa6\xb3" + "\xc5\x9b\x82\x27\xf5\x5e\x0b\xa4\x0e\x5e\x39\x04\xab\x51\x2a", + 1, 24, NULL, 0 }, + { 512, 256, 86, + "\x3f\x0c\xbe\xeb\xe3\x91\xc6\x49\x1e\x77\xc5\x7a\x05\xe8\x5a\x16\xf0" + "\xb5\x29\x4d\x19\xf9\xa7\xf3\x39\x0b\xaf\x7a\x20\x51\xfb\xf9\x80\xe0" + "\x41\xee\x45\xc9\x10\x4a\x91\x26\xa6\xa7\xec\x18\x2e\xae\xc2\x7a\x99" + "\xc1\xa7\xa3\xf5\xa1\xe8\xcd\x8f\xfd\xe6\x06\x41\xdc", + "\xbe\xbb\x03\x92", + "\x8d\x68\xdd\x7d\x67\x76\x3e\x8e\xaf\xee\x00\x29\xd0\x1e\x96\xe6\xa0" + "\x9c\x4b\xa0\x9e\x57\xa2\xe5\xbd\xe3\xba\xcc\xa2\x13\xc6\x95", + 1, 32, NULL, 0 }, + { 512, 256, 87, + "\x4a\x7f\x11\x0b\x92\x24\x1b\xad\xc9\x07\xba\x3c\x61\x47\x7b\xab\x09" + "\x53\xa8\x51\xbf\x32\x74\x25\xe8\x58\xfc\x72\x4b\xd3\x3e\xde\x2a\x4a" + "\x50\x18\xfe\x71\xaa\xb4\x34\xbd\xa8\xeb\x24\x64\xa4\x15\x77\xc8\xd5" + "\x70\x53\x0c\x46\x0f\x7c\x8b\xc0\x17\x2f\x1e\xe0\xf1", + "\x40\xa3\x33\xf4\xcf", + "\x5f\xed\xa3\x36\x6c\xe9\xf5\xac\x34\x02\xc9\x77\xa4\x06\x2d\x33\xc0" + "\x9e\x2c\x9a\x3d\x0c\x2d\xcd\xa0\xc8\xd6\x7b\x4b\xbd\x1a\x37", + 1, 40, NULL, 0 }, + { 512, 256, 88, + "\xc6\xc9\x5b\xf9\xfa\xca\xb2\x95\xce\xbf\xa4\xb9\x07\x85\x58\x74\xf5" + "\x7a\x5c\x15\x48\x57\x6a\xd8\xba\xe6\x94\xa3\x64\xf5\xe5\x8d\xbb\x8c" + "\x8d\xd4\x93\x30\xb2\xfd\xd1\xb7\x16\x57\xd2\x11\xdb\x2a\x65\x69\xa9" + "\xf3\xa3\x56\xc0\xc5\xb3\xc4\xef\xbd\x5b\x67\x77\xcb", + "\x4c\xf9\x26\xaf\x47\x5a", + "\x0a\x74\x8a\xa3\x37\x62\xd3\x74\xaa\x04\xb6\x17\xb5\x8d\x12\x9a\xd1" + "\xaa\xf2\x52\xa4\x04\x63\xfd\x62\xce\x92\x4e\x21\xe6\xd5\x2c", + 1, 48, NULL, 0 }, + { 512, 256, 89, + "\x0d\x33\x87\xfe\x59\xe8\xe7\xc4\x3c\x92\x8d\xac\x79\x13\x82\x6d\xec" + "\x4d\x63\xac\x3c\x8e\xe3\x46\xba\x7c\xae\xd9\x50\x5b\x9e\x63\xff\x89" + "\x42\xcd\xe9\x09\x97\xe8\xdb\xea\xf6\xe1\x7e\xe1\x87\xc0\xa8\x4a\x18" + "\x53\x95\x2d\x86\x6c\x15\xf9\xa3\x2f\xce\xe6\xa8\x2a", + "\x2b\xa3\xbc\x3c\xd6\x4b\xc6", + "\xcc\x7e\x97\x63\x08\x84\xbd\x8a\xd5\x6f\x6d\x96\xfd\x34\x69\x0e\xab" + "\x6c\x8a\xd5\x55\x6b\x51\x9d\xb3\xbc\x3c\x60\x83\xc8\x2d\x3c", + 1, 56, NULL, 0 }, + { 512, 256, 90, + "\x3c\xb9\xce\x56\x53\x88\xa6\xf0\x45\x4a\x80\xad\xd8\x6c\x7e\x10\x7e" + "\xa5\x37\xd7\xf4\x68\xa0\x64\x89\x30\xfc\x37\x17\x2c\xf7\xb4\xca\x90" + "\x58\x03\x30\x71\xc3\x54\xa2\x0a\x60\x8e\x2d\x46\xe9\x8a\xfe\x46\x43" + "\x5a\x34\x43\x62\x98\x9c\xba\xaf\xac\x18\x85\x9b\xba", + "\x33\xce\x49\x8e\x1f\x94\xf4\x12", + "\x76\x33\x5e\xe2\x3e\xe1\xdc\x25\x88\x12\xbe\x37\x3b\xb5\xf9\x59\x18" + "\xba\xbe\xd0\xb6\x9e\x56\x5e\x00\xee\x3a\xf7\x76\xc5\xa5\xfc", + 1, 64, NULL, 0 }, + { 512, 256, 91, + "\xc2\x5f\x45\xce\xb2\xa5\x59\x7f\x36\x14\x45\xfa\x41\xa9\x01\x9f\x41" + "\xa6\xe6\xd7\xf1\x44\x20\x3f\x29\xc0\xb9\xfc\xea\x36\x2d\x60\x89\x4c" + "\x3c\xad\xc1\xce\x25\xd5\x3d\xa3\x62\xe4\x64\xc1\x1f\xc6\xe1\x69\xe3" + "\xdb\x2e\xa1\xcf\x40\xfe\x08\xff\xfb\x42\x9b\x1a\x5b", + "\x81\x97\x8a\xf4\x79\x5c\x50\xf8\x9c", + "\xbf\x73\x21\x85\x44\xd8\x45\x8e\x6a\xd0\x07\x27\xb2\x36\xf8\x33\xd2" + "\x81\x72\x3d\x7d\xca\xe4\xd1\x01\x9b\x70\xb9\xd6\xe8\xbc\x4d", + 1, 72, NULL, 0 }, + { 512, 256, 92, + "\xb4\x58\x76\x4e\xe2\x73\xf3\x91\xcb\x71\x8f\x64\xa1\xbc\xa6\x4c\x96" + "\xa8\x70\xd9\x42\x6d\x62\x54\xee\x37\xe5\xc9\x38\x98\xd6\xa5\xef\x68" + "\xe9\xd3\xb0\xe0\x57\xa3\xc3\x96\xfa\xa8\x34\xa2\x99\x26\xa9\x68\x0c" + "\xfa\x90\x3d\x2a\x60\x5b\x85\x40\x7b\xb2\x4c\x8c\xee", + "\xd8\x04\xdc\x2a\x1e\x14\x6f\x62\xb6\x21", + "\x5b\x25\x84\x34\x16\x46\x7b\x9e\x0a\x24\xcf\xab\x67\xd8\xfc\x27\xe0" + "\x62\x3f\xf9\xe0\x1b\x22\x04\xb5\xaf\xdf\x3e\x9c\xc0\x5d\x1b", + 1, 80, NULL, 0 }, + { 512, 256, 93, + "\x7c\xbc\x57\x78\xf7\x05\x95\xfc\x21\x1c\xee\x3a\x93\xe1\x7f\xf7\xf2" + "\x5a\x1d\x9f\x37\x66\xf8\xeb\x70\xeb\x1e\x08\xc9\x42\x0a\x62\xbd\x89" + "\xe8\xb7\xd8\x34\xcc\x85\x4d\x05\x9a\xfd\x41\x3e\x4d\x9c\x06\x2a\x53" + "\x2e\x01\x59\x28\x52\x8c\x7f\x58\x12\xdf\xeb\xec\xac", + "\x40\x17\xe0\xff\xca\xac\x4c\x48\x5c\xe7\xec", + "\x66\xb4\x19\x1b\x37\xa6\xf7\x88\x09\xc4\x34\x73\x6e\xd6\xac\x22\x73" + "\xc0\x4a\x11\x21\x96\x36\xe9\x26\x71\xea\x05\xbf\x6d\xc2\x99", + 1, 88, NULL, 0 }, + { 512, 256, 94, + "\x93\x7b\x97\x11\xd6\x70\xda\xa2\x35\x99\x20\xe4\x7d\xd6\xd0\xfb\x75" + "\x27\x51\x05\xb3\xef\x07\xbb\x4a\x31\xd3\xc4\xb9\x9b\xaa\x8f\xf7\x9e" + "\xe4\xb4\xa1\xb4\xa5\xb2\x50\xd0\xfd\x7b\x47\x21\xe0\x4a\x7b\x06\x03" + "\x5b\x1d\x0c\x9d\x73\x95\x97\x70\x78\x39\x01\x8b\xce", + "\x69\x91\x81\x0e\x97\x88\xaf\x7a\xab\xff\x8e\xb4", + "\x23\x52\x2d\xe8\x0f\xf6\xa3\xa6\xd8\xfc\x1b\xf9\xb6\x32\xe1\x60\x0d" + "\xf5\x3c\x59\xf3\x85\x89\xa6\xf2\xae\x9b\x95\xd9\x40\xa3\x40", + 1, 96, NULL, 0 }, + { 512, 256, 95, + "\xad\x37\x63\x0a\x28\x0b\x1a\x75\xeb\xed\x19\x84\x21\x71\x50\xa4\x00" + "\xa5\x5d\xca\x23\x62\xa8\xea\xf3\xc9\x07\x85\x8d\x0b\x45\xdb\x3e\x20" + "\x8c\x31\x6d\x03\x38\x54\xeb\x42\x84\xf7\x11\x17\xd3\x38\x76\xe0\xe2" + "\x03\xca\x92\x2d\x26\xf9\xb7\x6d\x94\xca\xb0\xd4\xf3", + "\x7c\x6f\x2f\xd8\x3e\x56\x91\x82\x7b\xe3\x8e\x49\xa0", + "\xb6\x09\xc0\xb5\xd3\x59\x06\x1a\xc0\x66\xbd\x3b\xcc\xab\xc9\x84\x93" + "\xfd\x33\xba\xc8\xfe\x0e\x3f\x2e\x2b\x47\x58\xcb\x65\x78\xb0", + 1, 104, NULL, 0 }, + { 512, 256, 96, + "\x52\x6a\xa2\x64\x5a\xb7\x12\x29\xe2\xc6\x0f\x68\xbd\x5b\xbc\xf0\xce" + "\xa0\x97\x8a\x0a\x9c\x60\xcf\x69\x5e\x81\xfe\xca\xed\xec\xfa\x03\x62" + "\xc2\x17\x47\xea\xa9\x95\x97\x52\x08\xca\x35\xcf\xa6\xbc\x2b\x95\xd1" + "\xc2\xaf\xce\xe1\x12\x75\xf4\x3a\xdd\x6f\x02\x6d\x6c", + "\x68\x1b\xca\x55\x0f\xe3\x01\xf2\xdc\xc1\xe3\x8b\x53\xc1", + "\x1f\x1f\x50\xda\xcb\x3d\xc3\x5b\x90\x42\x9c\x0f\x9b\x31\xed\xd2\x39" + "\xa6\xaf\x4c\x09\xd5\x10\x95\xcb\x39\xff\x11\xc7\xc2\x65\x98", + 1, 112, NULL, 0 }, + { 512, 256, 97, + "\x60\x26\xa9\x04\x7a\x07\xcc\xf1\x97\xfd\xe0\x9c\x8b\x9f\x15\xf3\x4b" + "\xc5\x24\x72\xb7\xd1\xea\x46\x73\xb4\x91\x4a\x0e\x1c\x3a\xa4\xcf\xe8" + "\xd6\xef\x12\xd4\xd4\x01\x9f\x5f\xf3\xac\x0a\xdc\xad\x73\x58\x49\x08" + "\x74\x15\x5d\xaf\x41\xda\x2f\x74\xac\xa1\xcd\x89\x4a", + "\xce\x1d\x86\x9c\x53\x04\x1c\x5f\x6a\xcd\xda\x7a\x05\xaf\x15", + "\xbd\x2d\x67\x8c\xea\xdd\x71\x68\x0f\x99\x87\xc8\x8d\x24\xc4\x93\x35" + "\xcb\x98\x5a\xf0\xbc\xf8\xe2\x3f\xe8\x10\xe8\x3a\x92\x0f\x13", + 1, 120, NULL, 0 }, + { 512, 256, 98, + "\xa0\x3d\x2c\x54\x3c\x30\x29\x49\xc5\x1b\x66\x2f\x43\x11\x4c\x13\x05" + "\xa8\xf6\x96\x1a\xe4\x83\x42\x80\x3d\x36\x90\xdc\x18\x25\x5f\xab\x92" + "\x49\x65\x53\x6a\x79\xbc\x38\x56\x4c\x7c\x97\xcb\x8c\xc0\x20\x97\x86" + "\xe9\xf7\x63\x75\xbf\x18\x15\x29\xcf\x7f\x93\xd9\x54", + "\x69\x76\x17\xae\x31\xf1\x9b\x8a\x6a\xd4\xb8\x48\x9b\xfc\x3d\xb1", + "\x57\x7d\x62\xd7\x27\x9b\x39\xa0\xd7\x1e\x2f\x80\x83\x34\x25\xed\x43" + "\xe4\xa1\x62\x33\xee\xb5\xd2\x51\xf7\x66\xdb\x0b\xf7\xa5\x8c", + 1, 128, NULL, 0 }, + { 512, 256, 99, + "\x25\x84\x36\x65\xd3\x9c\x3a\xe9\xa7\xb3\xe4\x42\x7e\x2b\xf7\x78\x52" + "\x81\xfd\x25\x94\xbd\xde\x67\x86\x0c\xa9\xb8\xfa\x11\x64\x64\x69\xd1" + "\x64\x5a\xe8\xca\x38\x25\xb8\xc5\x51\xf9\xea\xc3\xda\x06\x60\xd8\xc2" + "\xe2\xe3\xbd\x23\xd3\x43\x95\xc6\x77\x5d\xcd\xfd\x2e", + "\x74\x08\x2c\xc5\xd1\x4d\xb1\x96\x74\x42\xd6\x6a\xac\x60\x92\xbd" + "\x23", + "\xb3\x73\xac\x5f\xb1\x98\x2b\x9d\x47\xd2\x88\x44\xe9\x69\xd5\x16\x80" + "\xdc\x81\xd2\x1d\x55\x6c\x26\x71\xc2\x9c\x11\xdf\xa6\xe3\x40", + 1, 136, NULL, 0 }, + { 512, 256, 100, + "\x48\x31\xab\x49\x62\xa2\xd2\x43\x60\x91\xcb\xbf\x38\x8d\x2b\xa0\x42" + "\xb4\x72\xa2\x62\xed\x83\x73\xc8\x5d\x04\x7c\x70\x2a\xdf\x73\xa8\x7e" + "\xb0\x97\xe7\x2d\x91\xd0\x89\xb7\xd1\x50\x4a\x7f\x7d\x8a\xbb\x3b\xc2" + "\xc4\x4c\x13\x40\xd6\xc1\x6c\x84\xea\x92\x69\xe6\x4b", + "\x28\x03\xc2\xf4\xe5\xb4\xbf\xcc\xd2\xb4\x07\x46\x9a\x6c\xb5\xef\x21" + "\xfd\x14\xa6\x82\x63\x63\x97", + "\x64\x1f\x29\x92\x5c\x06\xaa\x01\x08\x6c\x8b\xce\x89\xd9\x9e\x14\x56" + "\xdc\x2a\xd4\xb1\xd3\x36\x44\x42\x18\x7c\xe4\x39\x28\x38\xee", + 1, 192, NULL, 0 }, + { 512, 256, 101, + "\x44\x9b\xb5\x7d\x04\x65\x51\xe1\x81\x9b\x3c\x99\x41\x22\xc3\x60\x59" + "\x54\x31\x7d\x0d\x76\x21\x22\x84\xa3\x32\x8c\x22\x67\x32\xcb\xf4\xec" + "\xb4\x42\xa5\x82\xc8\x42\x38\x88\xaa\xed\x94\x6e\x5e\xec\x2b\xe6\x6e" + "\x12\x7f\x1e\x2e\x29\xb6\x6e\x68\xb9\xb4\xbe\xc4\xd1", + "\x79\x00\x4e\x64\x43\x89\xa1\x1b\x70\x9b\xc0\xa2\x3c\xb8\x59\x2f\x9f" + "\xc7\x96\x0b\xfa\x46\x13\x2c\xc1\xff\xb9\x74\x7d\xf3\x7d\xec", + "\xc0\x54\x66\x7d\x99\x2c\xc1\xe8\x4f\xa5\xb1\x3f\x64\x02\x12\x5b\x4b" + "\xb6\xfd\x29\x00\xdb\xcd\xaf\x8b\x86\x44\xc8\x2e\xda\xdc\x2b", + 1, 256, NULL, 0 }, + { 512, 256, 102, + "\xff\x97\xb7\x70\x20\x86\x1a\x0c\xe0\x0e\xff\x8d\xe9\xe7\x01\xae\xa8" + "\xc6\xde\x0f\xff\xd9\xb4\xc1\x34\x2a\x41\x6d\x35\x7f\xce\x35\xa7\x01" + "\x6c\x64\xcc\xd3\x8e\x2b\xdf\x67\x48\x02\x28\x1c\x22\x34\x15\x3d\xd8" + "\x3a\xac\xb9\x48\x96\x6d\xc8\x7e\xa4\x71\x8f\xff\x75", + "\xd7\xe2\xd2\x43\x7b\x70\x67\xa3\x0f\x04\x52\x90\x41\x96\x0e\x04\x1e" + "\x28\x17\x37\xd1\xe0\xda\xa8\xcd\x50\xcc\x0b\x26\x47\x16\xe1\x17\xaa" + "\x2f\xe9\xa7\xe3\x9c\x2f\x17\x8c\x60\x7f\xaa\x50\xc2", + "\x06\x2e\x9c\x46\x09\xa3\xb1\xd5\xdf\x27\x7a\x33\xac\x1c\x75\x01\xdf" + "\x81\xec\x1d\xdf\x46\x0b\x08\x50\xa2\xaa\x33\x2d\x07\xbf\xcb", + 1, 376, NULL, 0 }, + { 512, 256, 103, + "\xf4\xda\xf3\x70\xc4\xf4\x37\xd8\xa8\x53\x91\xec\x45\x5e\x34\x54\x0b" + "\xe8\xd3\x2d\xf8\xdf\xac\x05\xb1\x66\xae\x72\xec\x1c\xc6\x08\xf8\xc1" + "\x77\xb3\x0d\xde\x8b\xf0\x7c\x9d\x43\x47\x32\xc2\x6a\x6e\x53\x0b\x18" + "\x2c\x7a\xb2\x10\x93\xeb\x79\xd0\xbb\x5b\xe8\x5e\x53", + "\xf6\x04\x18\xf6\xc5\x4a\x25\xfa\xbf\x51\x82\x73\x08\x86\x19\xc0\x74" + "\x1c\x1c\x71\x87\xde\x93\xa0\xcf\x6a\x03\xf4\x56\x5f\x17\x65\xde\x65" + "\x67\x54\x54\x1b\x86\x01\x37\xf3\xf8\x45\x5d\xe7\xc4\x03", + "\x4d\x4a\x56\x8d\xd2\x6d\x27\x26\x7d\x07\x72\x54\x0e\xbb\x0b\x94\xd4" + "\x4f\xbb\xb1\x5a\x1b\xb7\x49\xa5\x70\xed\x63\x13\xee\x69\x5b", + 1, 384, NULL, 0 }, + { 512, 256, 104, + "\x3b\xe9\x5e\x87\x94\x21\xed\x78\x56\x26\x9e\xef\x39\xa2\x07\x0f\xae" + "\x40\x6c\xec\x5e\x30\xb5\x0d\x92\x79\x2d\xf5\xa3\x7d\xe9\x85\x95\x68" + "\x4e\xaa\x92\x05\x58\x7c\xa6\x07\xee\xac\x8f\x96\x59\x2f\x45\x8f\x63" + "\x43\x4b\x7d\xc8\x25\x96\xd3\xe4\xa1\xa1\x6c\x3d\x59", + "\xc6\x4f\x7c\x97\x0a\x44\x1c\x6c\x50\x38\x38\xa4\x91\x30\x8c\x78\x30" + "\x99\xea\xc5\x2b\xd3\x5a\x21\x79\x78\xa6\x4d\xce\xc8\x4d\x34\x18\x6a" + "\xb3\xb7\x4f\x20\x28\x5d\x6f\xea\x21\x65\xea\xb4\xda\x3d\x2c", + "\x4e\xa9\xe2\xf0\x88\x70\xc3\x0c\xb6\x8d\x41\x98\x16\xde\xb6\x81\xcf" + "\xd6\xba\x62\x90\x6e\x6b\x4a\x80\x88\xef\xcd\x9f\xa4\xed\xf9", + 1, 392, NULL, 0 }, + { 512, 256, 105, + "\xb5\xeb\x53\x58\x6b\x94\x85\x87\xdb\x3d\xd4\x6c\x43\xad\x65\x49\x8a" + "\x50\x79\x15\x75\x62\xe4\x07\x4c\x9c\x20\xd0\x97\xd0\xc9\x7f\x19\xdb" + "\x46\x61\xfb\x2d\xd1\x1b\x87\xa5\xcc\xba\x2c\x34\x56\x42\x61\x8f\x56" + "\x1d\x00\xbf\x87\xdf\xfc\x66\x76\x2e\x45\xe0\x15\x6f", + "\x9e\x8e\xbf\x96\x42\x99\x55\xd6\x0b\x92\x5a\x41\x11\x74\x5e\xc7\x02" + "\x8d\xe2\x4e\x69\x4a\x6d\x2e\xee\x1d\xbd\x5e\x82\x0a\xb9\xf0\x0b\xea" + "\xfd\xe0\x9f\x95\x09\x59\x33\xa0\x2f\x25\x12\x97\x28\x2b\x0c\xf6\x7c" + "\x51\x83\x97\x65\x58\x41\xf2\x30\xe1\xd9\xae\x5b\xa9\x31\x50\xd4\x37" + "\x5d\xc7\xc0\x73\x8b\x99\x85\x0b\x07\xd5\xa4\x42\x99\x4e\x68\xdc\x81" + "\x3d\x55\xed\xef\xa6\xcd\x06\x3c\xcb\x20\x27\x11\xd9\x7b\xa6\x74\xef" + "\xa0\x2d\xdb\xdc\x69\x23\x41\xe7\x7c\xfa", + "\x6a\x5b\x62\x72\xdf\x7f\xb6\x31\x52\x93\xce\x22\x48\x35\x11\xd7\x00" + "\xa8\x3c\x34\xdb\x6b\x0e\xe4\x39\x8b\x4a\x77\x02\x41\xfc\xf2", + 1, 896, NULL, 0 }, + { 512, 256, 106, + "\xfb\xc6\x78\x23\x5d\x59\x69\x80\x26\x87\x30\xa7\xaa\x8a\x3c\x25\x09" + "\x5f\x1d\xbc\xf7\xf8\x30\x99\x07\x16\xa5\x10\xc4\x4e\x62\xb1\xae\x8c" + "\xe0\x95\xc6\x5e\x18\x52\xbd\x8a\x09\xf2\x5a\xb9\x34\x15\xae\x73\x6f" + "\x22\xb2\xc6\x8e\xcd\xf3\xc0\xf7\x1e\x15\xa0\x57\xdc", + "\xef\x39\x57\x38\xf2\xb3\xbd\x7a\x06\x49\xee\xa7\x5e\x73\x4e\x5c\x79" + "\xba\xf2\x13\x58\xc7\xae\x96\xc6\x3a\x58\xe2\x60\x26\x6c\x7b\xfa\x86" + "\x96\x64\xc5\xd1\x0e\x87\xc2\x6d\x0f\x5e\xdb\x3b\x5f\x73\x90\x0c\x1d" + "\x9a\x96\xa5\xa2\xc2\x91\x25\x06\xc1\x9d\xff\x04\xe9\x00\xb8\xd5\xd6" + "\x3e\x1b\xb6\x06\xfe\x3d\x5a\x22\x9b\x64\x2b\x1a\xc5\x9e\x08\xa4\x68" + "\x7a\x7a\xad\xe3\xde\x16\xd1\x83\x13\x1d\xdd\x02\xda\x98\x8f\xf8\x26" + "\xe8\x8b\x74\xba\x0c\x5c\x41\xcf\xcd\x85\x70\xcc\xa5\x9f\xb3\x30\x4d" + "\x58\x8f\x7f\x7e\x63\xda\xd4\x7a", + "\x84\xdf\x33\x38\xf5\x02\xe6\x58\x66\xcf\x20\x77\xf6\x77\x63\x41\xa8" + "\x9d\xd2\x23\x49\x61\xb4\x74\xd5\xb7\x43\x8d\x18\x36\xfd\x6b", + 1, 1016, NULL, 0 }, + { 512, 256, 107, + "\xd8\x43\x7b\x09\x3e\x17\x0a\xfc\x30\xfb\x68\xd8\x73\xdb\x6d\xd6\x7e" + "\xe0\x37\x2b\x6c\x5f\xeb\x12\x4d\x54\x8a\xbb\xd9\x33\x04\xa0\x82\x23" + "\x5a\x17\x0a\x9d\x32\x62\x68\xd0\xcf\xc3\x42\x64\xd8\xeb\xcf\xcb\xcc" + "\x0d\xc0\xea\x77\x67\xb6\x50\xdb\xcf\xc7\x84\x8f\x91", + "\x0f\xb9\x1b\xb0\x43\xcf\x3d\x49\x34\x14\x82\xea\xcb\xab\x92\xda\x92" + "\x11\x7b\xb6\xec\x03\xb5\x18\xa9\x3c\x9d\x59\xe5\x47\x02\xc6\xa2\x1a" + "\xde\x4f\x25\x58\x70\xcb\x52\xda\x4a\x24\xb3\x68\x56\xb0\xcb\xf8\xb1" + "\x22\xd2\xfd\x5b\x02\x19\xae\xac\xb6\xe2\x92\xc9\x58\x63\x92\x17\x89" + "\x09\x2d\x5d\x65\xd5\xa9\x38\x92\x31\x67\x0e\x38\xbe\x31\xd8\xb7\x66" + "\x30\x65\x0c\x77\xed\xc2\x37\x73\xd5\xec\x91\x89\x91\x55\x60\xef\x6f" + "\x45\xa4\x27\x5f\x85\x95\x7d\x87\x62\x91\x6d\x81\x47\xef\x43\xea\x6f" + "\x43\x8d\x06\x62\x27\x35\x4d\xf2\x6c", + "\x4b\x9b\x51\xf2\xc7\xd7\x25\x26\x9a\x5b\xd7\x35\xd4\x25\x1c\xbf\xda" + "\x71\xea\x5c\xf5\xc0\x46\x37\x9b\x43\x9d\xdf\xf7\x66\xd2\x1a", + 1, 1024, NULL, 0 }, + { 512, 256, 108, + "\xd2\x92\x25\xe3\x04\x2b\x43\xd4\xeb\x7a\x93\x99\xf2\x24\x42\x4b\x5b" + "\x4d\xd9\x90\x31\xc8\xab\xf6\x09\xd3\xa6\xe3\x17\x58\x97\x13\x4c\xc7" + "\xba\x8a\x6b\xe2\x5d\x43\x6d\x41\xa7\x57\xa2\xda\xa4\xe1\xb0\x3f\x7c" + "\x30\x53\xee\x8c\xad\xa1\x95\x31\xd4\x8d\xab\x93\xe0", + "\x1f\xda\xd8\xcd\x37\xe1\x15\xef\x81\x31\xd4\x61\x9b\x5d\x61\xa9\x82" + "\x7a\x11\xb9\xc2\x5c\x5b\x82\x20\xb0\x02\xd5\xb7\xf6\xe2\xbc\x30\xa7" + "\xe2\xf3\xd1\x03\xdb\xca\xf8\x49\xa2\x55\xc2\x0e\xc5\xd1\xb4\x0a\x62" + "\x3e\x39\x8d\x76\xc5\xb7\xf0\x7f\xe3\x05\x18\x1c\xd5\xbe\xe2\x9d\xe3" + "\x36\x0f\xed\x55\xf6\x90\x63\xe6\xa8\xfd\x2e\xf3\x12\x91\xe4\xd7\xc3" + "\x90\x84\x49\x46\x6b\x4f\xf1\x16\x69\x59\xff\x1f\x2e\x86\xeb\x48\xa7" + "\xed\x11\xbd\xce\xc2\xfc\x2d\xfb\x66\x84\xb3\x6b\x8d\x0b\x68\xe0\xf1" + "\xb2\x3e\xf1\x1b\xf5\x90\x7d\x4a\x2b\x16\x2c\xec\x26\xf3\x1b\x6d\x5d" + "\x98\x92\xa2\x7b\x3f\xad\x7a\x55\x49\x85\x8d\xad\x53\x0b\xae\x19\x3d" + "\x9b\x60\xd4\x2d\x7a\xd2\xa6\x64\x76\xa3\xba\x4b\xf7\xa2\x7f\xf9\xd0" + "\xf8\x85\xa5\x40\xbf\x18\x1c\xaa\xd7\x1a\x23\x5e\xb3\x48\xa2\x3d\x05" + "\x3b\xa2\xdb\x7a\xed\x8d\x7d\x01\xd9\x6d\xc9\xf7\x80\xe2\xe4\x26\xc7" + "\x2b\xb6\x3f\xcb\x3f\xff\x44\xc1\x4b\xb7\xb0\xf8\xaf\x35\x52\xd6\x7e" + "\xe6\x76\x10\x92\xc7\x57\x62\x7d\x19\xc0\x80\x49\x9c\x24\x7d\x13\xb4" + "\x31\x69\x93\x97\x15\x9b\x1b\x71\xc6\x27\x45\x84\x95\x9d\x5c\x30" + "\xf0", + "\xf9\xad\x00\x42\x0c\xe9\x09\xcd\x05\x0f\xdb\x84\xdc\xc7\x0f\x00\xdf" + "\x97\x92\x89\x68\xd0\xac\xa5\x1d\xb2\x78\x4f\x03\x94\x89\x8d", + 1, 2040, NULL, 0 }, + { 512, 256, 109, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 110, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 111, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd0\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 112, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbd\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 113, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x52\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 114, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3f\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 115, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9a\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 116, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x31\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 117, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\xbf\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 118, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\x3f\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 119, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x86\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 120, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb4\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 121, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x85\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 122, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb7\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 123, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 124, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 125, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf2\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 126, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x22\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 127, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\x73\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 128, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\xa3\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 129, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x61\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 130, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xd1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 131, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x08\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 132, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x67\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 133, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc6\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 134, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x49\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 135, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc5\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 136, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x4a\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 137, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\x47\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 138, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\xc8\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 139, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x63", + 0, 0, NULL, 0 }, + { 512, 256, 140, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa2", + 0, 128, NULL, 0 }, + { 512, 256, 141, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x60", + 0, 0, NULL, 0 }, + { 512, 256, 142, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa1", + 0, 128, NULL, 0 }, + { 512, 256, 143, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x22", + 0, 0, NULL, 0 }, + { 512, 256, 144, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xe3", + 0, 128, NULL, 0 }, + { 512, 256, 145, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x86\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\xe2", + 0, 0, NULL, 0 }, + { 512, 256, 146, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x0e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\x23", + 0, 128, NULL, 0 }, + { 512, 256, 147, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9b\x9e\x3f\x87\x80\x96\x86\xf2\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 148, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x30\x1c\xbf\xb5\x66\x72\x0e\x22\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 149, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\xbf\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\xab\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 150, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\x3f\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x96\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 151, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd2\x9b\x9e\x3f\x87\x80\x96\x06\xf3\x41\x09\xfb\xc7\x18\xd6\x2b\xbb" + "\x09\xc2\x78\xcf\x05\xa2\x06\xad\xf2\x14\x63\xe1\x17\x03\x62", + 0, 0, NULL, 0 }, + { 512, 256, 152, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbf\x30\x1c\xbf\xb5\x66\x72\x8e\x23\xf1\x66\xe2\x48\x65\xc3\x16\xf2" + "\x16\x19\xc7\xc1\x50\x33\xcc\x6e\x8e\xbb\xcc\x8c\x5c\x5b\xa3", + 0, 128, NULL, 0 }, + { 512, 256, 153, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x2d\x64\x61\xc0\x78\x7f\x69\x79\x0c\xbe\xf6\x04\x38\xe7\x29\x54\x44" + "\xf6\x3d\x87\x30\xfa\x5d\xf9\x52\x0d\xeb\x9c\x1e\xe8\xfc\x9d", + 0, 0, NULL, 0 }, + { 512, 256, 154, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x40\xcf\xe3\x40\x4a\x99\x8d\xf1\xdc\x0e\x99\x1d\xb7\x9a\x3c\x69\x0d" + "\xe9\xe6\x38\x3e\xaf\xcc\x33\x91\x71\x44\x33\x73\xa3\xa4\x5c", + 0, 128, NULL, 0 }, + { 512, 256, 155, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 0, NULL, 0 }, + { 512, 256, 156, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + 0, 128, NULL, 0 }, + { 512, 256, 157, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 0, NULL, 0 }, + { 512, 256, 158, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 0, 128, NULL, 0 }, + { 512, 256, 159, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\x52\x1b\x1e\xbf\x07\x00\x16\x06\x73\xc1\x89\x7b\x47\x98\x56\x2b\x3b" + "\x89\x42\xf8\x4f\x85\x22\x86\x2d\x72\x94\xe3\x61\x97\x83\xe2", + 0, 0, NULL, 0 }, + { 512, 256, 160, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\x3f\xb0\x9c\x3f\x35\xe6\xf2\x8e\xa3\x71\xe6\x62\xc8\xe5\x43\x16\x72" + "\x96\x99\x47\x41\xd0\xb3\x4c\xee\x0e\x3b\x4c\x0c\xdc\xdb\x23", + 0, 128, NULL, 0 }, + { 512, 256, 161, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "", + "\xd3\x9a\x9f\x3e\x86\x81\x97\x87\xf2\x40\x08\xfa\xc6\x19\xd7\xaa\xba" + "\x08\xc3\x79\xce\x04\xa3\x07\xac\xf3\x15\x62\xe0\x16\x02\x63", + 0, 0, NULL, 0 }, + { 512, 256, 162, + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21" + "\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32" + "\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f", + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + "\xbe\x31\x1d\xbe\xb4\x67\x73\x0f\x22\xf0\x67\xe3\x49\x64\xc2\x97\xf3" + "\x17\x18\xc6\xc0\x51\x32\xcd\x6f\x8f\xba\xcd\x8d\x5d\x5a\xa2", + 0, 128, NULL, 0 }, + { 256, 512, 163, + "\x14\xd9\x37\x59\xfc\x28\xf3\x31\x9a\xb7\x4b\x81\x67\xc9\x74\xe8\x00" + "\xf0\x32\x34\x4d\xc2\x74\x7e\xc0\xf4\x94\x50\x61\xa4\x78\x27", + "", + "\x68\x93\x4d\xbe\x94\x8d\x9a\x77\xa5\xe0\xa9\x2e\xd9\x82\x54\xfa\x3b" + "\x6c\x93\xc8\xbf\x5e\xea\xa9\x12\xb7\xdf\xdf\x76\x2b\x37\x19\x2c\x5d" + "\x85\x23\xbc\xab\x9a\xd7\x1b\x09\xbf\x96\xd8\x45\x41\x88\xd0\x01\xc7" + "\xf2\x07\x7e\xb6\x41\x19\x9f\x57\x31\xb9\xf9\x46\x69", + 1, 0, NULL, 0 }, + { 256, 512, 164, + "\x9f\xa3\x71\xf3\x6f\xb2\x73\xd5\x14\xfd\x62\x8c\xb9\x38\x06\x7a\x4b" + "\xae\x32\xa1\x9a\x1e\x04\x5a\x7d\x6d\x7f\x6d\xe3\x75\x1c\xbf", + "\x31\x1b\xbf\x72\x2d\x32\x2c\xd7\xa0\x71\x0f\x48\x0f\xc6\x65\x18", + "\x16\x34\x5f\x6a\x6c\xa6\xe7\x8d\x4c\xca\xc3\x0b\x48\xd7\x66\x91\xd6" + "\x44\x24\x20\xef\xa1\x13\xc1\x5e\xf1\x27\xb5\x38\xb5\xb0\x24\x01\x8b" + "\x7d\x2d\xb4\xbc\x3e\xd3\x42\x42\x51\xab\x6b\x8b\x6c\x3c\xb1\x08\xb0" + "\xbe\xda\x84\x2d\xc3\xe6\x8e\x63\x40\x02\x87\xe5\xcd", + 1, 128, NULL, 0 }, + { 256, 512, 165, + "\x63\x13\xf1\x52\x6b\xc2\x20\xf2\x0d\xde\x1e\x64\xce\xd8\x59\x72\x79" + "\x58\x6d\x1e\x15\xaa\xd0\x5a\xd5\x91\xd8\x41\xb3\x69\x28\x4f", + "\xf7\x44\xfa\x39\x33\xe1\x6d\x8b\xf5\x24\xaf\xae\xb3\x4c\x71\x56\x53" + "\xa9\xcf\xb0\x1f\xa4\x5f\xe1\xfb\x68\xe7\x01\xfe\x14\x87\xca", + "\xb8\x8d\x1b\xa0\x3e\x27\x99\x20\x0a\x44\x75\x50\xd1\x8e\x31\x06\x97" + "\xa5\x79\x74\xf5\x13\xdf\x77\xeb\x07\xbb\xe3\x15\xba\x5f\xef\x39\x7e" + "\xeb\x81\xad\x90\x71\x68\x0b\xcc\x6c\x70\xf6\xb2\x52\xad\xe3\x5b\x4a" + "\x40\x40\x27\x9e\xc0\x1b\x86\xe4\x0b\x98\x77\x0e\x39", + 1, 256, NULL, 0 }, + { 256, 256, 166, + "\x1e\x22\x5c\xaf\xb9\x03\x39\xbb\xa1\xb2\x40\x76\xd4\x20\x6c\x3e\x79" + "\xc3\x55\x80\x5d\x85\x16\x82\xbc\x81\x8b\xaa\x4f\x5a\x77\x79", + "", + "\x23\xd4\x82\xa0\x5c\x90\x7e\xeb\x34\x6b\xa9\x8f\x83\xdb\x0f\x63\xc2" + "\xad\xfb\xd5\xb2\x94\x0f\x33\xc7\x96\x4c\x7f\x17\x99\xf1\x80", + 1, 0, NULL, 0 }, + { 256, 256, 167, + "\x6f\xa3\x53\x86\x8c\x82\xe5\xde\xee\xda\xc7\xf0\x94\x71\xa6\x1b\xf7" + "\x49\xab\x54\x98\x23\x9e\x94\x7e\x01\x2e\xee\x3c\x82\xd7\xc4", + "\xae\xed\x3e\x4d\x4c\xb9\xbb\xb6\x0d\x48\x2e\x98\xc1\x26\xc0\xf5", + "\x1c\xf9\xd2\xc9\xc1\xb5\x5a\x45\x19\x0b\x5b\xeb\x59\x0c\xd4\xcc\x95" + "\xe3\x85\x3d\xf8\xaa\xf9\xf4\xfe\xf9\xbb\xbb\xd7\x24\x35\xff", + 1, 128, NULL, 0 }, + { 256, 256, 168, + "\x18\x6e\x24\x8a\xd8\x24\xe1\xeb\x93\x32\x9a\x7f\xdc\xd5\x65\xb6\xcb" + "\x4e\xaf\x3f\x85\xb9\x0b\x91\x07\x77\x12\x8d\x8c\x53\x8d\x27", + "\x92\xef\x9f\xf5\x2f\x46\xec\xcc\x7e\x38\xb9\xee\x19\xfd\x2d\xe3\xb3" + "\x77\x26\xc8\xe6\xce\x9e\x1b\x96\xdb\x5d\xda\x4c\x31\x79\x02", + "\xd1\x27\xb7\x38\x5b\xad\xf0\xc7\x6f\x2b\x3d\x8a\xa9\xc7\x22\x33\x35" + "\x92\xe0\x1f\x46\x2f\xed\xd3\x5e\xc6\x64\xa6\xf6\xd5\x2d\x74", + 1, 256, NULL, 0 }, + { 520, 512, 169, + "\xdd\x1e\x0b\xdb\xb6\xb6\x08\x62\x17\x64\x84\xf3\x66\x9d\xa5\x31\x45" + "\x5f\x1c\xd7\x14\xf9\x99\xc2\x9f\x08\xb8\x51\x05\x5f\xee\x8d\x72\x18" + "\x6d\x37\x6c\x23\x6f\x4e\x16\xcb\xa7\xa2\x5c\xba\x87\x9f\xb2\x75\x3d" + "\xec\xa4\x45\x9a\xae\xbc\x6f\x6d\xe6\x25\xd9\x9a\xf3\x30", + "", + "\x7e\x4f\x7d\x84\x4b\x3b\xa0\xe0\x25\xb6\x6d\xe7\xcc\x62\x27\xbc\x50" + "\xd4\xe1\x74\x93\x02\x51\xbf\xff\x3d\xf3\x6c\x39\x00\xb5\xb7\x6b\x00" + "\x09\x5a\x89\x6d\x0f\x96\x84\x2e\x37\xb6\x13\x4d\xf4\x07\x60\x30\x76" + "\x99\x53\x4d\x66\x70\xf1\x38\x97\x4e\xe1\xc5\x8d\x94", + 1, 0, NULL, 0 }, + { 520, 512, 170, + "\x43\x2b\x31\x1e\xbc\xfd\x46\xec\xfc\xd3\xcc\x70\x6e\xbd\x05\xc7\x87" + "\xdf\xbe\x18\x55\xfd\xcf\xce\x8d\x50\xc9\xa0\x0f\x72\xb6\x5a\x8d\x42" + "\xac\xec\x33\x5b\x4e\x07\xd5\x44\xc9\x2f\xd7\xb1\xd3\x85\x43\xac\x6e" + "\x0f\xc0\x4c\x26\xd8\x8d\xe8\xdd\x97\x4a\xf6\x9e\x24\xd7", + "\x36\xb1\xfb\xe8\xf1\x33\x5e\x7c\x03\x99\xc2\x47\x30\x90\x64\x20", + "\x2c\xfb\x68\x8f\x30\xb1\x05\x34\xda\x93\x77\xa4\xb3\xfb\xee\x1d\xec" + "\x16\x1c\xb2\x88\xac\x8b\x75\x87\x93\x83\x8b\x45\xab\x95\x39\x79\xda" + "\xdf\x27\x81\x7f\x47\x7c\x9e\xbf\x23\xcf\xdc\xba\xcb\x60\xb8\x10\x38" + "\xe0\x8b\xc4\xfc\x31\x80\xbd\x2a\x1e\xe8\x05\x97\x6a", + 1, 128, NULL, 0 }, + { 520, 512, 171, + "\x17\xf7\x20\xf0\x9d\xf5\x97\x2a\xf9\xb9\xc6\x3e\x10\x04\x32\x84\x60" + "\x89\x00\xd5\x0b\x79\x55\xdb\x3b\x4e\x26\x79\xcb\x41\x20\xbe\x2c\x9b" + "\x9e\x2a\xa1\xa5\x74\x3e\xb5\x19\x79\x28\x22\xc3\x26\xb4\xd8\x90\xb5" + "\x55\x4d\x1c\xb0\xeb\x71\x08\x1b\x75\x69\xa2\xf0\x4d\xf7", + "\x57\x16\x7c\x25\x24\xa5\x52\x89\x68\x7b\x83\xa4\x0d\x3a\x69\xbc\x90" + "\xad\xc5\x3a\xd2\x47\x02\x0b\x88\x89\x7f\x9b\x95\xd1\x51\x6d", + "\x4f\x70\x26\x7b\x98\xfc\xeb\x4f\x66\x29\x01\xbd\x18\xfb\x4c\x81\xac" + "\x16\x42\x81\xdd\x0e\xce\x43\x02\x8a\x3c\x2a\x65\xca\x21\x3a\xed\xf1" + "\xbd\x20\x7f\x09\x39\xbd\x87\x9b\xbe\x20\xfd\x09\xcd\xeb\x20\x24\x6e" + "\x65\x39\x76\x6a\xdd\x08\xb3\xad\xc5\x14\x3d\x2b\xd9", + 1, 256, NULL, 0 }, + { 520, 256, 172, + "\x8a\x0c\x46\xeb\x8a\x29\x59\xe3\x98\x65\x33\x00\x79\x76\x33\x41\xe7" + "\x43\x9d\xab\x14\x96\x94\xee\x57\xe0\xd6\x1e\xc7\x3d\x94\x7e\x1d\x53" + "\x01\xcd\x97\x4e\x18\xa5\xe0\xd1\xcf\x0d\x2c\x37\xe8\xaa\xdd\x9f\xd5" + "\x89\xd5\x7e\xf3\x2e\x47\x02\x4a\x99\xbc\x3f\x70\xc0\x77", + "", + "\xe1\x65\x7f\x44\xbf\x84\x89\x5e\x6d\xb0\x81\x0a\x2c\xca\x61\xa6\xe1" + "\x05\xe1\x2e\xc0\x06\xf0\xb5\x96\x10\x20\x30\x1b\x57\x74\x4e", + 1, 0, NULL, 0 }, + { 520, 256, 173, + "\x28\x77\xeb\xb8\x1f\x80\x33\x4f\xd0\x05\x16\x33\x74\x46\xc5\xcf\x5a" + "\xd4\xa3\xa2\xe1\x97\x26\x9e\x5b\x0a\xd1\x88\x9d\xfe\x2b\x4b\x0a\xaa" + "\x67\x6f\xac\x55\xb3\x6c\xe3\xaf\xfc\x7f\x10\x92\xab\x89\xc5\x32\x73" + "\xa8\x37\xbd\x5b\xc9\x4d\x1a\x9d\x9e\x5b\x02\xe9\x85\x6f", + "\xba\x44\x8d\xb8\x8f\x15\x4f\x77\x50\x28\xfd\xec\xf9\xe6\x75\x2d", + "\x33\xd5\xa2\xd1\x99\x8a\x58\x68\x49\xee\xbf\x81\x34\x72\x84\x85\xfc" + "\xfc\x71\x24\x8f\x4a\x98\xe6\x22\xf8\x3b\x96\x78\x44\xc4\x0e", + 1, 128, NULL, 0 }, + { 520, 256, 174, + "\x21\x17\x8e\x26\xbc\x28\xff\xc2\x7c\x06\xf7\x62\xba\x19\x0a\x62\x70" + "\x75\x85\x6d\x7c\xa6\xfe\xab\x79\xac\x63\x14\x9b\x17\x12\x6e\x34\xfd" + "\x9e\x55\x90\xe0\xe9\x0a\xac\x80\x1d\xf0\x95\x05\xd8\xaf\x2d\xd0\xa2" + "\x70\x3b\x35\x2c\x57\x3a\xc9\xd2\xcb\x06\x39\x27\xf2\xaf", + "\x7d\x5f\x1d\x6b\x99\x34\x52\xb1\xb5\x3a\x43\x75\x76\x0d\x10\xa2\x0d" + "\x46\xa0\xab\x9e\xc3\x94\x3f\xc4\xb0\x7a\x2c\xe7\x35\xe7\x31", + "\x88\xd5\x79\xc2\x80\x19\x05\xb8\x18\x07\x0c\xce\xbd\x2c\x71\x92\xf9" + "\x7b\xb3\xe7\xac\xdc\xaf\x61\x3c\xec\xc7\x4d\x0e\x41\x12\x32", + 1, 256, NULL, 0 }, + { 0, 0, 0, NULL, NULL, NULL, 0, 0, NULL, 0 } +}; diff --git a/test/wycheproof/mac_test.h b/test/wycheproof/mac_test.h new file mode 100644 index 00000000..89b71f7e --- /dev/null +++ b/test/wycheproof/mac_test.h @@ -0,0 +1,47 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef MAC_TEST_H +#define MAC_TEST_H + +#include +#include + +struct mac_test { + size_t keySize; /* bits */ + size_t tagSize; /* bits */ + size_t tcId; + const char *key; + const char *msg; + const char *tag; + int resultValid; + size_t msgSize; /* bits */ + const char *iv; + size_t ivSize; /* bits */ +}; + +#endif /* MAC_TEST_H */ diff --git a/test/wycheproof/win_x64.mak b/test/wycheproof/win_x64.mak new file mode 100644 index 00000000..85d2aac0 --- /dev/null +++ b/test/wycheproof/win_x64.mak @@ -0,0 +1,98 @@ +# +# Copyright (c) 2022, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# + +APP = wycheproof +INSTNAME = intel-ipsec-mb + +!if !defined(PREFIX) +PREFIX = C:\Program Files +!endif + +!if exist("$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib") +IPSECLIB = "$(PREFIX)\$(INSTNAME)\libIPSec_MB.lib" +INCDIR = -I"$(PREFIX)\$(INSTNAME)" +!else +!if !defined(LIB_DIR) +LIB_DIR = ..\..\lib +!endif +IPSECLIB = "$(LIB_DIR)\libIPSec_MB.lib" +INCDIR = -I$(LIB_DIR) -I.\ +!endif + +!if !defined(DEBUG_OPT) +DEBUG_OPT = /Od +!endif + +!ifdef DEBUG +DCFLAGS = $(DEBUG_OPT) /DDEBUG /Z7 +DLFLAGS = /debug +!else +DCFLAGS = /O2 /Oi +DLFLAGS = +!endif + +# compiler +CC = cl + +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure snprintf() being used +CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) + +#linker +LNK = link +LFLAGS = /out:$(APP).exe $(DLFLAGS) + +# dependency +!ifndef DEPTOOL +DEPTOOL = ..\..\mkdep.bat +!endif +DEPFLAGS = $(INCDIR) + +OBJS = aes_gcm_test.json.obj aes_ccm_test.json.obj \ + chacha20_poly1305_test.json.obj \ + aes_cmac_test.json.obj gmac_test.json.obj \ + hmac_sha1_test.json.obj hmac_sha224_test.json.obj \ + hmac_sha256_test.json.obj hmac_sha384_test.json.obj \ + hmac_sha512_test.json.obj wycheproof.obj + +all: $(APP).exe + +$(APP).exe: $(OBJS) $(IPSECLIB) + $(LNK) $(LFLAGS) $(OBJS) $(IPSECLIB) + +tests.dep: $(OBJS) + @type *.obj.dep > $@ 2> nul + +.c.obj: + $(CC) /c $(CFLAGS) $< + $(DEPTOOL) $< $@ "$(DEPFLAGS)" > $@.dep + +clean: + del /q $(OBJS) tests.dep *.obj.dep $(APP).exe + +!if exist(tests.dep) +!include tests.dep +!endif diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof/wycheproof.c new file mode 100644 index 00000000..95167709 --- /dev/null +++ b/test/wycheproof/wycheproof.c @@ -0,0 +1,1470 @@ +/******************************************************************************* + Copyright (c) 2022, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#include +#include +#include +#include + +#include + +#include "mac_test.h" +#include "aead_test.h" + +static unsigned run_vectors = 0; +static unsigned skip_vectors = 0; + +static unsigned total_run_vectors = 0; +static unsigned total_skip_vectors = 0; + +static int process_job(IMB_MGR *p_mgr) +{ + IMB_JOB *job = IMB_SUBMIT_JOB(p_mgr); + + if (!job) { + const int err = imb_get_errno(p_mgr); + + /* check for error */ + if (err != 0) + return 0; + + /* flush to get the job processed */ + job = IMB_FLUSH_JOB(p_mgr); + + /* if flush returns nothing then it's an error */ + if (!job) + return 0; + } + + /* if returned job is not complete then it's an error */ + if (job->status != IMB_STATUS_COMPLETED) + return 0; + + return 1; +} + +static void +prep_iopad(const size_t scratch_size, void *scratch, + const size_t key_size, const void *key, + const int pattern) +{ + uint8_t *cb = (uint8_t *) scratch; + const uint8_t *kp = (const uint8_t *) key; + const size_t max_j = + (key_size > scratch_size) ? scratch_size : key_size; + size_t j; + + memset(scratch, pattern, scratch_size); + for (j = 0; j < max_j; j++) + cb[j] ^= kp[j]; +} + +#define PUTS_ONCE(_s) { \ + static int _ran_already = 0; \ + \ + if (!_ran_already) { \ + _ran_already = 1; \ + printf("\t@note %s\n", _s); \ + } \ + } + +/* + * ============================================================================= + * MAC TESTS + * ============================================================================= + */ +static void print_mac_test(const struct mac_test *v) +{ + if (v->iv != NULL) { + printf("MAC vector details:\n" + " tcId = %u\n" + " keySize = %u [bits]\n" + " tagSize = %u [bits]\n" + " msgSize = %u [bits]\n" + " ivSize = %u [bits]\n" + " resultValid = %d\n", + (unsigned)v->tcId, (unsigned)v->keySize, + (unsigned)v->tagSize, (unsigned)v->msgSize, + (unsigned)v->ivSize, (int)v->resultValid); + } else { + printf("MAC vector details:\n" + " tcId = %u\n" + " keySize = %u [bits]\n" + " tagSize = %u [bits]\n" + " msgSize = %u [bits]\n" + " resultValid = %d\n", + (unsigned)v->tcId, (unsigned)v->keySize, + (unsigned)v->tagSize, (unsigned)v->msgSize, + (int)v->resultValid); + } +} + +static int err_code = 0; + +static int +mac_submit_and_check(IMB_MGR *p_mgr, + const struct mac_test *v, + const void *res_tag, + const int job_api) +{ + if (job_api) { + /* submit job and get it processed */ + if (!process_job(p_mgr)) { + if (v->resultValid) { + print_mac_test(v); + printf("JOB-API submit/flush error!\n"); + printf("ERROR: %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + return 0; + } else { + /* error was expected */ + return 1; + } + } + } else { + if (err_code != 0) { + if (v->resultValid) { + print_mac_test(v); + printf("DIRECT-API error!\n"); + printf("ERROR: %s\n", + imb_get_strerror(err_code)); + return 0; + } else { + /* error was expected */ + err_code = 0; + return 1; + } + } + } + + const int tag_mismatch = memcmp(res_tag, v->tag, v->tagSize / 8); + + /* was mismatch expected? */ + if (v->resultValid == 0 && tag_mismatch) + return 1; + + /* check for TAG mismatch */ + if (tag_mismatch) { + printf("%s: TAG mismatch!\n", + job_api ? "JOB-API" : "DIRECT-API"); + print_mac_test(v); + return 0; + } + + return 1; +} + +static void errno_update(IMB_MGR *p_mgr) +{ + const int new_code = imb_get_errno(p_mgr); + + if (err_code == 0 && new_code != 0) + err_code = new_code; +} + +static void errno_reset(void) +{ + errno = 0; +} + +extern const struct mac_test aes_cmac_test_json[]; + +static int test_cmac(IMB_MGR *p_mgr) +{ + const struct mac_test *v = aes_cmac_test_json; + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint32_t skey1[4], skey2[4]; + uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if ((v->keySize / 8) == IMB_KEY_192_BYTES) { + /* unsupported - skip it*/ + PUTS_ONCE("AES-CMAC-192 not supported"); + skip_vectors++; + run_vectors--; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = scratch; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + if ((v->keySize / 8) == IMB_KEY_128_BYTES) { + job->hash_alg = IMB_AUTH_AES_CMAC; + IMB_AES_KEYEXP_128(p_mgr, v->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_128(p_mgr, expkey, skey1, + skey2); + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + } else if ((v->keySize / 8) == IMB_KEY_256_BYTES) { + job->hash_alg = IMB_AUTH_AES_CMAC_256; + IMB_AES_KEYEXP_256(p_mgr, v->key, expkey, dust); + IMB_AES_CMAC_SUBKEY_GEN_256(p_mgr, expkey, skey1, + skey2); + job->u.CMAC._key_expanded = expkey; + job->u.CMAC._skey1 = skey1; + job->u.CMAC._skey2 = skey2; + } + + /* clear space where computed TAG is put into */ + memset(scratch, 0, sizeof(scratch)); + + if (!mac_submit_and_check(p_mgr, v, scratch, 1)) + return 0; + } + + return 1; +} + +extern const struct mac_test gmac_test_json[]; + +static int test_gmac(IMB_MGR *p_mgr) +{ + const struct mac_test *v = gmac_test_json; + struct gcm_key_data gmac_key; + uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = scratch; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + if ((v->keySize / 8) == IMB_KEY_128_BYTES) { + job->hash_alg = IMB_AUTH_AES_GMAC_128; + IMB_AES128_GCM_PRE(p_mgr, v->key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = (const void *) v->iv; + job->u.GMAC.iv_len_in_bytes = v->ivSize / 8; + } else if ((v->keySize / 8) == IMB_KEY_192_BYTES) { + job->hash_alg = IMB_AUTH_AES_GMAC_192; + IMB_AES192_GCM_PRE(p_mgr, v->key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = (const void *) v->iv; + job->u.GMAC.iv_len_in_bytes = v->ivSize / 8; + } else if ((v->keySize / 8) == IMB_KEY_256_BYTES) { + job->hash_alg = IMB_AUTH_AES_GMAC_256; + IMB_AES256_GCM_PRE(p_mgr, v->key, &gmac_key); + job->u.GMAC._key = &gmac_key; + job->u.GMAC._iv = (const void *) v->iv; + job->u.GMAC.iv_len_in_bytes = v->ivSize / 8; + } + /* clear space where computed TAG is put into */ + memset(scratch, 0, sizeof(scratch)); + + if (!mac_submit_and_check(p_mgr, v, scratch, 1)) + return 0; + + /* exercise direct API test if available */ + memset(scratch, 0, sizeof(scratch)); + errno_reset(); + + if ((v->keySize / 8) == IMB_KEY_128_BYTES) { + struct gcm_context_data ctx; + + IMB_AES128_GCM_PRE(p_mgr, v->key, &gmac_key); + errno_update(p_mgr); + IMB_AES128_GMAC_INIT(p_mgr, &gmac_key, &ctx, + (const void *) v->iv, + v->ivSize / 8); + errno_update(p_mgr); + IMB_AES128_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES128_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tagSize / 8); + errno_update(p_mgr); + } + if ((v->keySize / 8) == IMB_KEY_192_BYTES) { + struct gcm_context_data ctx; + + IMB_AES192_GCM_PRE(p_mgr, v->key, &gmac_key); + errno_update(p_mgr); + IMB_AES192_GMAC_INIT(p_mgr, &gmac_key, &ctx, + (const void *) v->iv, + v->ivSize / 8); + errno_update(p_mgr); + IMB_AES192_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES192_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tagSize / 8); + errno_update(p_mgr); + } + if ((v->keySize / 8) == IMB_KEY_256_BYTES) { + struct gcm_context_data ctx; + + IMB_AES256_GCM_PRE(p_mgr, v->key, &gmac_key); + errno_update(p_mgr); + IMB_AES256_GMAC_INIT(p_mgr, &gmac_key, &ctx, + (const void *) v->iv, + v->ivSize / 8); + errno_update(p_mgr); + IMB_AES256_GMAC_UPDATE(p_mgr, &gmac_key, &ctx, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES256_GMAC_FINALIZE(p_mgr, &gmac_key, &ctx, + scratch, v->tagSize / 8); + errno_update(p_mgr); + } + + if (!mac_submit_and_check(p_mgr, v, scratch, 0)) + return 0; + } + + return 1; +} + +extern const struct mac_test hmac_sha1_test_json[]; + +static int test_hmac_sha1(IMB_MGR *p_mgr) +{ + const struct mac_test *v = hmac_sha1_test_json; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA1_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA1_BLOCK_SIZE]; + uint8_t key[IMB_SHA1_DIGEST_SIZE_IN_BYTES]; + uint8_t tag[IMB_SHA1_DIGEST_SIZE_IN_BYTES]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + const void *key_ptr = NULL; + size_t key_size = 0; + + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if (v->msgSize == 0) { + /* @todo skip */ + PUTS_ONCE("HMAC-SHA1 msgSize=0 not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_HMAC_SHA_1; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = tag; + + /* @note smaller tags sizes can be rejected */ + if ((v->tagSize / 8) > 0 && + (v->tagSize / 8) <= IMB_SHA1_DIGEST_SIZE_IN_BYTES) + job->auth_tag_output_len_in_bytes = + IMB_SHA1_DIGEST_SIZE_IN_BYTES; + else + job->auth_tag_output_len_in_bytes = + v->tagSize / 8; + + /* prepare key */ + if ((v->keySize / 8) <= IMB_SHA1_BLOCK_SIZE) { + key_ptr = v->key; + key_size = v->keySize / 8; + } else { + IMB_SHA1(p_mgr, v->key, v->keySize / 8, key); + key_ptr = key; + key_size = IMB_SHA1_DIGEST_SIZE_IN_BYTES; + } + + /* compute IPAD and OPAD */ + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); + IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); + IMB_SHA1_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + + /* clear space where computed TAG is put into */ + memset(tag, 0, sizeof(tag)); + + if (!mac_submit_and_check(p_mgr, v, tag, 1)) + return 0; + } + + return 1; +} + +extern const struct mac_test hmac_sha224_test_json[]; + +static int test_hmac_sha224(IMB_MGR *p_mgr) +{ + const struct mac_test *v = hmac_sha224_test_json; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA_256_BLOCK_SIZE]; + uint8_t key[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; + uint8_t tag[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + const void *key_ptr = NULL; + size_t key_size = 0; + + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if (v->msgSize == 0) { + /* @todo skip */ + PUTS_ONCE("HMAC-SHA224 msgSize=0 not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_HMAC_SHA_224; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + /* prepare key */ + if ((v->keySize / 8) <= IMB_SHA_256_BLOCK_SIZE) { + key_ptr = v->key; + key_size = v->keySize / 8; + } else { + IMB_SHA224(p_mgr, v->key, v->keySize / 8, key); + key_ptr = key; + key_size = IMB_SHA224_DIGEST_SIZE_IN_BYTES; + } + /* compute IPAD and OPAD */ + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); + IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); + IMB_SHA224_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + + /* clear space where computed TAG is put into */ + memset(tag, 0, sizeof(tag)); + + if (!mac_submit_and_check(p_mgr, v, tag, 1)) + return 0; + } + + return 1; +} + +extern const struct mac_test hmac_sha256_test_json[]; + +static int test_hmac_sha256(IMB_MGR *p_mgr) +{ + const struct mac_test *v = hmac_sha256_test_json; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA256_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA_256_BLOCK_SIZE]; + uint8_t key[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; + uint8_t tag[IMB_SHA256_DIGEST_SIZE_IN_BYTES]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + const void *key_ptr = NULL; + size_t key_size = 0; + + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if (v->msgSize == 0) { + /* @todo skip */ + PUTS_ONCE("HMAC-SHA256 msgSize=0 not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_HMAC_SHA_256; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + /* prepare key */ + if ((v->keySize / 8) <= IMB_SHA_256_BLOCK_SIZE) { + key_ptr = v->key; + key_size = v->keySize / 8; + } else { + IMB_SHA256(p_mgr, v->key, v->keySize / 8, key); + key_ptr = key; + key_size = IMB_SHA256_DIGEST_SIZE_IN_BYTES; + } + + /* compute IPAD and OPAD */ + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x36); + IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + prep_iopad(sizeof(scratch), scratch, key_size, key_ptr, 0x5c); + IMB_SHA256_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + + /* clear space where computed TAG is put into */ + memset(tag, 0, sizeof(tag)); + + if (!mac_submit_and_check(p_mgr, v, tag, 1)) + return 0; + } + + return 1; +} + +extern const struct mac_test hmac_sha384_test_json[]; + +static int test_hmac_sha384(IMB_MGR *p_mgr) +{ + const struct mac_test *v = hmac_sha384_test_json; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if (v->msgSize == 0) { + /* @todo skip */ + PUTS_ONCE("HMAC-SHA384 msgSize=0 not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_HMAC_SHA_384; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = scratch; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + /* compute IPAD and OPAD */ + prep_iopad(sizeof(scratch), scratch, + v->keySize / 8, v->key, 0x36); + IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + prep_iopad(sizeof(scratch), scratch, + v->keySize / 8, v->key, 0x5c); + IMB_SHA384_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + + /* clear space where computed TAG is put into */ + memset(scratch, 0, sizeof(scratch)); + + if (!mac_submit_and_check(p_mgr, v, scratch, 1)) + return 0; + } + + return 1; +} + +extern const struct mac_test hmac_sha512_test_json[]; + +static int test_hmac_sha512(IMB_MGR *p_mgr) +{ + const struct mac_test *v = hmac_sha512_test_json; + DECLARE_ALIGNED(uint8_t hmac_ipad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + DECLARE_ALIGNED(uint8_t hmac_opad[IMB_SHA512_DIGEST_SIZE_IN_BYTES], 16); + uint8_t scratch[IMB_SHA_512_BLOCK_SIZE]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for ( ; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + + /* tag too long */ + if (v->tagSize > (sizeof(scratch) * 8)) { + print_mac_test(v); + return 0; + } + + if (v->msgSize == 0) { + /* @todo skip */ + PUTS_ONCE("HMAC-SHA512 msgSize=0 not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + job->hash_alg = IMB_AUTH_HMAC_SHA_512; + job->cipher_mode = IMB_CIPHER_NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->src = (const void *) v->msg; + job->hash_start_src_offset_in_bytes = 0; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->auth_tag_output = scratch; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + + /* compute IPAD and OPAD */ + prep_iopad(sizeof(scratch), scratch, + v->keySize / 8, v->key, 0x36); + IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_ipad); + + prep_iopad(sizeof(scratch), scratch, + v->keySize / 8, v->key, 0x5c); + IMB_SHA512_ONE_BLOCK(p_mgr, scratch, hmac_opad); + + job->u.HMAC._hashed_auth_key_xor_ipad = hmac_ipad; + job->u.HMAC._hashed_auth_key_xor_opad = hmac_opad; + + /* clear space where computed TAG is put into */ + memset(scratch, 0, sizeof(scratch)); + + if (!mac_submit_and_check(p_mgr, v, scratch, 1)) + return 0; + } + + return 1; +} + +/* + * ============================================================================= + * AEAD TESTS + * ============================================================================= + */ + +static void print_aead_test(const struct aead_test *v) +{ + printf("AEAD vector details:\n" + " tcId = %u\n" + " ivSize = %u [bits]\n" + " keySize = %u [bits]\n" + " tagSize = %u [bits]\n" + " aadSize = %u [bits]\n" + " msgSize = %u [bits]\n" + " resultValid = %d\n", + (unsigned)v->tcId, (unsigned)v->ivSize, + (unsigned)v->keySize, (unsigned)v->tagSize, + (unsigned)v->aadSize, (unsigned)v->msgSize, + (int)v->resultValid); +} + +static int +aead_submit_and_check(IMB_MGR *p_mgr, + const struct aead_test *v, + const void *res_tag, + const void *res_text, + const int job_api, + const int is_encrypt) +{ + if (job_api) { + /* submit job and get it processed */ + if (!process_job(p_mgr)) { + if (v->resultValid) { + print_aead_test(v); + printf("JOB-API submit/flush error!\n"); + return 0; + } else { + /* error was expected */ + return 1; + } + } + } else { + if (err_code != 0) { + if (v->resultValid) { + print_aead_test(v); + printf("DIRECT-API error!\n"); + printf("ERROR: %s\n", + imb_get_strerror(err_code)); + return 0; + } else { + /* error was expected */ + err_code = 0; + return 1; + } + } + } + + const int tag_mismatch = memcmp(res_tag, v->tag, v->tagSize / 8); + const int text_mismatch = is_encrypt ? + memcmp(res_text, v->ct, v->msgSize / 8) : + memcmp(res_text, v->msg, v->msgSize / 8); + + if (v->resultValid == 0 && (tag_mismatch || text_mismatch)) + return 1; + + /* check for TAG mismatch */ + if (tag_mismatch) { + printf("%s %s: TAG mismatch!\n", + job_api ? "JOB-API" : "DIRECT-API", + is_encrypt ? "encrypt" : "decrypt"); + print_aead_test(v); + return 0; + } + + /* check for text mismatch */ + if (text_mismatch) { + printf("%s %s mismatch!\n", + job_api ? "JOB-API" : "DIRECT-API", + is_encrypt ? "encrypt: cipher-text" : + "decrypt: plain-text"); + print_aead_test(v); + return 0; + } + + return 1; +} + +extern const struct aead_test aes_gcm_test_json[]; + +static int test_aead_gcm(IMB_MGR *p_mgr) +{ + const struct aead_test *v = NULL; + struct gcm_key_data gcm_key; + struct gcm_context_data ctx; + uint8_t text[512], tag[16]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (v = aes_gcm_test_json; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT(v->tagSize <= (sizeof(tag) * 8)); + IMB_ASSERT(v->msgSize <= (sizeof(text) * 8)); + + /* tag too long */ + if (v->tagSize > (sizeof(tag) * 8)) { + print_aead_test(v); + return 0; + } + /* message too long */ + if (v->msgSize > (sizeof(text) * 8)) { + print_aead_test(v); + return 0; + } + + switch (v->keySize / 8) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_PRE(p_mgr, v->key, &gcm_key); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_PRE(p_mgr, v->key, &gcm_key); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_PRE(p_mgr, v->key, &gcm_key); + break; + default: + printf("Invalid key size: %u bytes!\n", + (unsigned)v->keySize / 8); + print_aead_test(v); + return 0; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + /* encrypt test */ + job->cipher_mode = IMB_CIPHER_GCM; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->msg; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_AES_GMAC; + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aadSize / 8; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 1)) + return 0; + + /* decrypt test */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = IMB_CIPHER_GCM; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->ct; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_AES_GMAC; + job->enc_keys = &gcm_key; + job->dec_keys = &gcm_key; + job->u.GCM.aad = v->aad; + job->u.GCM.aad_len_in_bytes = v->aadSize / 8; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 0)) + return 0; + + /* test direct API */ + + /* encrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + errno_reset(); + + switch (v->keySize / 8) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES128_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES128_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES192_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES192_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES256_GCM_ENC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES256_GCM_ENC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + default: + printf("Invalid key size: %u bytes!\n", + (unsigned)v->keySize / 8); + print_aead_test(v); + return 0; + } + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 0, 1)) + return 0; + + /* decrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + errno_reset(); + switch (v->keySize / 8) { + case IMB_KEY_128_BYTES: + IMB_AES128_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES128_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->ct, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES128_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + case IMB_KEY_192_BYTES: + IMB_AES192_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES192_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->ct, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES192_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + case IMB_KEY_256_BYTES: + IMB_AES256_GCM_INIT_VAR_IV(p_mgr, &gcm_key, &ctx, + (const void *) v->iv, + v->ivSize / 8, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_AES256_GCM_DEC_UPDATE(p_mgr, &gcm_key, &ctx, text, + (const void *) v->ct, + v->msgSize / 8); + errno_update(p_mgr); + IMB_AES256_GCM_DEC_FINALIZE(p_mgr, &gcm_key, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + break; + default: + printf("Invalid key size: %u bytes!\n", + (unsigned)v->keySize / 8); + print_aead_test(v); + return 0; + } + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 0, 0)) + return 0; + } + + return 1; +} + +extern const struct aead_test chacha20_poly1305_test_json[]; + +static int test_aead_chacha20_poly1305(IMB_MGR *p_mgr) +{ + const struct aead_test *v = NULL; + struct chacha20_poly1305_context_data ctx; + uint8_t text[512], tag[16]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (v = chacha20_poly1305_test_json; v->msg != NULL; + v++, run_vectors++) { + IMB_ASSERT(v->tagSize <= (sizeof(tag) * 8)); + IMB_ASSERT(v->msgSize <= (sizeof(text) * 8)); + + /* tag too long */ + if (v->tagSize > (sizeof(tag) * 8)) { + print_aead_test(v); + return 0; + } + /* message too long */ + if (v->msgSize > (sizeof(text) * 8)) { + print_aead_test(v); + return 0; + } + + /* test JOB API */ + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + /* encrypt test */ + job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->msg; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_CHACHA20_POLY1305; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->hash_start_src_offset_in_bytes = 0; + job->enc_keys = (const void *) v->key; + job->dec_keys = (const void *) v->key; + job->u.CHACHA20_POLY1305.aad = (const void *) v->aad; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = v->aadSize / 8; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 1)) + return 0; + + /* decrypt test */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = IMB_CIPHER_CHACHA20_POLY1305; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->ct; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_CHACHA20_POLY1305; + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->hash_start_src_offset_in_bytes = 0; + job->enc_keys = (const void *) v->key; + job->dec_keys = (const void *) v->key; + job->u.CHACHA20_POLY1305.aad = (const void *) v->aad; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = v->aadSize / 8; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 0)) + return 0; + + /* test direct API */ + + /* encrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + errno_reset(); + + IMB_CHACHA20_POLY1305_INIT(p_mgr, (const void *) v->key, &ctx, + (const void *) v->iv, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_CHACHA20_POLY1305_ENC_UPDATE(p_mgr, (const void *) v->key, + &ctx, text, + (const void *) v->msg, + v->msgSize / 8); + errno_update(p_mgr); + IMB_CHACHA20_POLY1305_ENC_FINALIZE(p_mgr, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 0, 1)) + return 0; + + /* decrypt direction */ + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + errno_reset(); + + IMB_CHACHA20_POLY1305_INIT(p_mgr, (const void *) v->key, &ctx, + (const void *) v->iv, + (const void *) v->aad, + v->aadSize / 8); + errno_update(p_mgr); + IMB_CHACHA20_POLY1305_DEC_UPDATE(p_mgr, (const void *) v->key, + &ctx, text, + (const void *) v->ct, + v->msgSize / 8); + errno_update(p_mgr); + IMB_CHACHA20_POLY1305_DEC_FINALIZE(p_mgr, &ctx, tag, + v->tagSize / 8); + errno_update(p_mgr); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 0, 0)) + return 0; + } + + return 1; +} + +extern const struct aead_test aes_ccm_test_json[]; + +static int test_aead_ccm(IMB_MGR *p_mgr) +{ + const struct aead_test *v = NULL; + DECLARE_ALIGNED(uint32_t expkey[4*15], 16); + DECLARE_ALIGNED(uint32_t dust[4*15], 16); + uint8_t text[512], tag[16]; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) + ; + + for (v = aes_ccm_test_json; v->msg != NULL; v++, run_vectors++) { + IMB_ASSERT(v->tagSize <= (sizeof(tag) * 8)); + IMB_ASSERT(v->msgSize <= (sizeof(text) * 8)); + + /* tag too long */ + if (v->tagSize > (sizeof(tag) * 8)) { + print_aead_test(v); + return 0; + } + /* message too long */ + if (v->msgSize > (sizeof(text) * 8)) { + print_aead_test(v); + return 0; + } + + if ((v->aadSize / 8) > 46) { + /* unsupported AAD sizes - skip it */ + PUTS_ONCE("AES-CCM AAD > 46 bytes not supported"); + run_vectors--; + skip_vectors++; + continue; + } + + switch (v->keySize / 8) { + case IMB_KEY_128_BYTES: + IMB_AES_KEYEXP_128(p_mgr, v->key, expkey, dust); + break; + case IMB_KEY_256_BYTES: + IMB_AES_KEYEXP_256(p_mgr, v->key, expkey, dust); + break; + case IMB_KEY_192_BYTES: + /* unsupported key size - skip it */ + PUTS_ONCE("AES-CCM-192 not supported"); + run_vectors--; + skip_vectors++; + continue; + default: + printf("Invalid key size: %u bytes!\n", + (unsigned)v->keySize / 8); + print_aead_test(v); + return 0; + } + + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + /* encrypt test */ + job->cipher_mode = IMB_CIPHER_CCM; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->msg; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_AES_CCM; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->u.CCM.aad_len_in_bytes = v->aadSize / 8; + job->u.CCM.aad = v->aad; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 1)) + return 0; + + /* decrypt test */ + job = IMB_GET_NEXT_JOB(p_mgr); + + job->cipher_mode = IMB_CIPHER_CCM; + job->cipher_direction = IMB_DIR_DECRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->key_len_in_bytes = v->keySize / 8; + job->src = (const void *) v->ct; + job->dst = text; + job->msg_len_to_cipher_in_bytes = v->msgSize / 8; + job->cipher_start_src_offset_in_bytes = UINT64_C(0); + job->msg_len_to_hash_in_bytes = v->msgSize / 8; + job->hash_start_src_offset_in_bytes = UINT64_C(0); + job->iv = (const void *) v->iv; + job->iv_len_in_bytes = v->ivSize / 8; + job->auth_tag_output = tag; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; + job->hash_alg = IMB_AUTH_AES_CCM; + job->enc_keys = expkey; + job->dec_keys = expkey; + job->u.CCM.aad_len_in_bytes = v->aadSize / 8; + job->u.CCM.aad = v->aad; + + memset(text, 0, sizeof(text)); + memset(tag, 0, sizeof(tag)); + + /* submit job and check */ + if (!aead_submit_and_check(p_mgr, v, tag, text, 1, 0)) + return 0; + } /* for(ccm_vectors) */ + + return 1; +} + +static int test_all(IMB_MGR *p_mgr) +{ + const struct { + int (*fn)(IMB_MGR *); + const char *name; + } test_tab[] = { + { test_aead_gcm, "AEAD AES-GCM" }, + { test_aead_ccm, "AEAD AES-CCM" }, + { test_aead_chacha20_poly1305, "AEAD CHACHA20-POLY1305" }, + { test_cmac, "AES-CMAC" }, + { test_gmac, "GMAC" }, + { test_hmac_sha1, "HMAC-SHA1" }, + { test_hmac_sha224, "HMAC-SHA224" }, + { test_hmac_sha256, "HMAC-SHA256" }, + { test_hmac_sha384, "HMAC-SHA384" }, + { test_hmac_sha512, "HMAC-SHA512" } + }; + unsigned i; + int ret = 1; + + for (i = 0; i < IMB_DIM(test_tab); i++) { + run_vectors = 0; + skip_vectors = 0; + if (test_tab[i].fn(p_mgr) == 0) { + printf("Testing %s: FAILED\n", test_tab[i].name); + ret = 0; + } else { + printf("Testing %s: PASSED (run: %u, skipped: %u)\n", + test_tab[i].name, run_vectors, skip_vectors); + } + total_run_vectors += run_vectors; + total_skip_vectors += skip_vectors; + } + return ret; +} + + +/* + * ============================================================================= + * MAIN + * ============================================================================= + */ + +static void +usage(const char *name) +{ + printf("Usage: %s [args], where args are zero or more\n" + "--aesni-emu test AESNI emulation interface\n" + "--avx512 test AVX512 interface\n" + "--avx2 test AVX2 interface\n" + "--avx test AVX interface\n" + "--sse test SSE interface\n" + "--shani-off don't use SHA extensions " + "(auto-detect is default)\n", name); +} + +int main(int argc, const char **argv) +{ + IMB_ARCH arch_to_run = IMB_ARCH_NUM; + uint64_t flags = 0; + const uint64_t feat_flags = imb_get_feature_flags(); + IMB_MGR *p_mgr = NULL; + int i; + + if (imb_get_version() < IMB_VERSION(0, 50, 0)) { + printf("Library version detection unsupported!\n"); + } else { + printf("Tool version: %s\n", IMB_VERSION_STR); + printf("Library version: %s\n", imb_get_version_str()); + } + + for (i = 1; i < argc; i++) { + if ((strcmp(argv[i], "-h") == 0) || + (strcmp(argv[i], "--help") == 0)) { + usage(argv[0]); + return EXIT_SUCCESS; + } else if (strcmp(argv[i], "--aesni-emu") == 0) { + flags |= IMB_FLAG_AESNI_OFF; + arch_to_run = IMB_ARCH_NOAESNI; + } else if (strcmp(argv[i], "--sse") == 0) { + arch_to_run = IMB_ARCH_SSE; + } else if (strcmp(argv[i], "--avx") == 0) { + arch_to_run = IMB_ARCH_AVX; + } else if (strcmp(argv[i], "--avx2") == 0) { + arch_to_run = IMB_ARCH_AVX2; + } else if (strcmp(argv[i], "--avx512") == 0) { + arch_to_run = IMB_ARCH_AVX512; + } else if (strcmp(argv[i], "--shani-off") == 0) { + flags |= IMB_FLAG_SHANI_OFF; + } + } + + p_mgr = alloc_mb_mgr(flags); + if (p_mgr == NULL) { + printf("Error allocating MB_MGR structure: %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + return EXIT_FAILURE; + } + + switch (arch_to_run) { + case IMB_ARCH_NOAESNI: + if (((feat_flags & IMB_FEATURE_AESNI_EMU) == 0) && + (imb_get_errno(p_mgr) == IMB_ERR_NO_AESNI_EMU)) { + printf("AESNI Emulation is not enabled!\n"); + free_mb_mgr(p_mgr); + return EXIT_FAILURE; + } + init_mb_mgr_sse(p_mgr); + break; + case IMB_ARCH_SSE: + init_mb_mgr_sse(p_mgr); + break; + case IMB_ARCH_AVX: + init_mb_mgr_avx(p_mgr); + break; + case IMB_ARCH_AVX2: + init_mb_mgr_avx2(p_mgr); + break; + case IMB_ARCH_AVX512: + init_mb_mgr_avx512(p_mgr); + break; + default: + /* auto-detect */ + init_mb_mgr_auto(p_mgr, &arch_to_run); + break; + } + + if (imb_get_errno(p_mgr) != 0) { + printf("Error initializing MB_MGR structure! %s\n", + imb_get_strerror(imb_get_errno(p_mgr))); + free_mb_mgr(p_mgr); + return EXIT_FAILURE; + } + + if (!test_all(p_mgr)) { + printf("Wycheproof test complete: FAILED\n"); + free_mb_mgr(p_mgr); + return EXIT_FAILURE; + } + + printf("Test complete: PASSED (run: %u, skipped: %u)\n", + total_run_vectors, total_skip_vectors); + + free_mb_mgr(p_mgr); + return EXIT_SUCCESS; +} diff --git a/win_x64.mak b/win_x64.mak index 6c097cb9..8fac2541 100644 --- a/win_x64.mak +++ b/win_x64.mak @@ -28,11 +28,13 @@ all: cd lib & $(MAKE) /f win_x64.mak cd test & $(MAKE) /f win_x64.mak + cd test\wycheproof & $(MAKE) /f win_x64.mak cd perf & $(MAKE) /f win_x64.mak clean: cd lib & $(MAKE) /f win_x64.mak clean cd test & $(MAKE) /f win_x64.mak clean + cd test\wycheproof & $(MAKE) /f win_x64.mak clean cd perf & $(MAKE) /f win_x64.mak clean install: -- GitLab From 17536e00a0ba5900a10031af8ef2cacbcc8c0004 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 13 Sep 2022 15:31:51 +0000 Subject: [PATCH 351/369] avx2: [sha256/224] use SHANI extensions in avx2 type-2 for SHA224, HMAC-SHA224, SHA256 and HMAC-SHA256 --- lib/avx2_t2/mb_mgr_avx2_t2.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/avx2_t2/mb_mgr_avx2_t2.c b/lib/avx2_t2/mb_mgr_avx2_t2.c index d9f3c605..8b7a267f 100644 --- a/lib/avx2_t2/mb_mgr_avx2_t2.c +++ b/lib/avx2_t2/mb_mgr_avx2_t2.c @@ -47,6 +47,7 @@ #include "include/error.h" #include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_sse_type2.h" /* shani */ #include "include/arch_avx_type1.h" #include "include/arch_avx2_type1.h" #include "include/arch_avx2_type2.h" @@ -170,12 +171,13 @@ #define SUBMIT_JOB_PON_DEC_NO_CTR submit_job_pon_dec_no_ctr_avx /* SHA1/224/256/384/512 */ +/* note: SHA1 MB is better than SHANI on Xeon processors */ #define SUBMIT_JOB_SHA1 submit_job_sha1_avx2 #define FLUSH_JOB_SHA1 flush_job_sha1_avx2 -#define SUBMIT_JOB_SHA224 submit_job_sha224_avx2 -#define FLUSH_JOB_SHA224 flush_job_sha224_avx2 -#define SUBMIT_JOB_SHA256 submit_job_sha256_avx2 -#define FLUSH_JOB_SHA256 flush_job_sha256_avx2 +#define SUBMIT_JOB_SHA224 submit_job_sha224_ni_sse +#define FLUSH_JOB_SHA224 flush_job_sha224_ni_sse +#define SUBMIT_JOB_SHA256 submit_job_sha256_ni_sse +#define FLUSH_JOB_SHA256 flush_job_sha256_ni_sse #define SUBMIT_JOB_SHA384 submit_job_sha384_avx2 #define FLUSH_JOB_SHA384 flush_job_sha384_avx2 #define SUBMIT_JOB_SHA512 submit_job_sha512_avx2 @@ -184,10 +186,10 @@ /* HMAC-SHA1/224/256/384/512 */ #define SUBMIT_JOB_HMAC submit_job_hmac_avx2 #define FLUSH_JOB_HMAC flush_job_hmac_avx2 -#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_avx2 -#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_avx2 -#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_avx2 -#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_avx2 +#define SUBMIT_JOB_HMAC_SHA_224 submit_job_hmac_sha_224_ni_sse +#define FLUSH_JOB_HMAC_SHA_224 flush_job_hmac_sha_224_ni_sse +#define SUBMIT_JOB_HMAC_SHA_256 submit_job_hmac_sha_256_ni_sse +#define FLUSH_JOB_HMAC_SHA_256 flush_job_hmac_sha_256_ni_sse #define SUBMIT_JOB_HMAC_SHA_384 submit_job_hmac_sha_384_avx2 #define FLUSH_JOB_HMAC_SHA_384 flush_job_hmac_sha_384_avx2 #define SUBMIT_JOB_HMAC_SHA_512 submit_job_hmac_sha_512_avx2 @@ -243,12 +245,10 @@ static void reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_hmac_sha1_reset(state->hmac_sha_1_ooo, AVX2_NUM_SHA1_LANES); /* Init HMAC/SHA224 out-of-order fields */ - ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, - AVX2_NUM_SHA256_LANES); + ooo_mgr_hmac_sha224_reset(state->hmac_sha_224_ooo, 2); - /* Init HMAC/SHA256 out-of-order fields */ - ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, - AVX2_NUM_SHA256_LANES); + /* Init HMAC/SHA_256 out-of-order fields */ + ooo_mgr_hmac_sha256_reset(state->hmac_sha_256_ooo, 2); /* Init HMAC/SHA384 out-of-order fields */ ooo_mgr_hmac_sha384_reset(state->hmac_sha_384_ooo, @@ -279,10 +279,10 @@ static void reset_ooo_mgrs(IMB_MGR *state) ooo_mgr_sha1_reset(state->sha_1_ooo, AVX2_NUM_SHA1_LANES); /* Init SHA224 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_224_ooo, AVX2_NUM_SHA256_LANES); + ooo_mgr_sha256_reset(state->sha_224_ooo, 2); /* Init SHA256 out-of-order fields */ - ooo_mgr_sha256_reset(state->sha_256_ooo, AVX2_NUM_SHA256_LANES); + ooo_mgr_sha256_reset(state->sha_256_ooo, 2); /* Init SHA384 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_384_ooo, AVX2_NUM_SHA512_LANES); -- GitLab From cfebfc3b3edf55d86d2d32462b4382518c735085 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 13 Sep 2022 15:57:18 +0000 Subject: [PATCH 352/369] lib: [snow3g] use SSE implementations of UIA2 and UEA2 for AVX and AVX2 JOB API For UIA2, on non-AVX512 architectures if SAFE_LOOKUP is disabled then fallback to the default implementation as it is bit faster. --- lib/avx2_t1/mb_mgr_avx2_t1.c | 36 +++++++++++++++++++++++++++++- lib/avx2_t2/mb_mgr_avx2_t2.c | 36 +++++++++++++++++++++++++++++- lib/avx_t1/mb_mgr_avx_t1.c | 4 ++-- lib/avx_t2/mb_mgr_avx_t2.c | 4 ++-- lib/include/mb_mgr_job_api.h | 8 +++---- lib/no-aesni/mb_mgr_sse_no_aesni.c | 2 ++ 6 files changed, 80 insertions(+), 10 deletions(-) diff --git a/lib/avx2_t1/mb_mgr_avx2_t1.c b/lib/avx2_t1/mb_mgr_avx2_t1.c index b4575cb0..44782ec2 100644 --- a/lib/avx2_t1/mb_mgr_avx2_t1.c +++ b/lib/avx2_t1/mb_mgr_avx2_t1.c @@ -36,6 +36,7 @@ #include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" +#include "include/snow3g_submit.h" #include "include/gcm.h" #include "include/chacha20_poly1305.h" @@ -46,7 +47,7 @@ #include "include/aesni_emu.h" #include "include/error.h" -#include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_sse_type1.h" /* poly1305, snow3g */ #include "include/arch_avx_type1.h" #include "include/arch_avx2_type1.h" @@ -214,6 +215,33 @@ #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_avx2_t1(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_avx2_t1(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx2_t1 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx2_t1 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local @@ -288,6 +316,12 @@ static void reset_ooo_mgrs(IMB_MGR *state) /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/avx2_t2/mb_mgr_avx2_t2.c b/lib/avx2_t2/mb_mgr_avx2_t2.c index 8b7a267f..23bea9ab 100644 --- a/lib/avx2_t2/mb_mgr_avx2_t2.c +++ b/lib/avx2_t2/mb_mgr_avx2_t2.c @@ -36,6 +36,7 @@ #include "include/kasumi_interface.h" #include "include/zuc_internal.h" #include "include/snow3g.h" +#include "include/snow3g_submit.h" #include "include/gcm.h" #include "include/chacha20_poly1305.h" @@ -46,7 +47,7 @@ #include "include/aesni_emu.h" #include "include/error.h" -#include "include/arch_sse_type1.h" /* poly1305 */ +#include "include/arch_sse_type1.h" /* poly1305, snow3g */ #include "include/arch_sse_type2.h" /* shani */ #include "include/arch_avx_type1.h" #include "include/arch_avx2_type1.h" @@ -217,6 +218,33 @@ #define SUBMIT_JOB_SNOW_V snow_v_avx #define SUBMIT_JOB_SNOW_V_AEAD snow_v_aead_init_avx +/* SNOW3G UE2 & UIA2 */ +static IMB_JOB * +submit_snow3g_uea2_job_avx2_t2(IMB_MGR *state, IMB_JOB *job) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + if ((job->msg_len_to_cipher_in_bits & 7) || + (job->cipher_start_offset_in_bits & 7)) + return def_submit_snow3g_uea2_job(state, job); + + return submit_job_snow3g_uea2_sse(snow3g_uea2_ooo, job); +} + +static IMB_JOB * +flush_snow3g_uea2_job_avx2_t2(IMB_MGR *state) +{ + MB_MGR_SNOW3G_OOO *snow3g_uea2_ooo = state->snow3g_uea2_ooo; + + return flush_job_snow3g_uea2_sse(snow3g_uea2_ooo); +} + +#define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx2_t2 +#define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx2_t2 + +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse + /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local @@ -289,6 +317,12 @@ static void reset_ooo_mgrs(IMB_MGR *state) /* Init SHA512 out-of-order fields */ ooo_mgr_sha512_reset(state->sha_512_ooo, AVX2_NUM_SHA512_LANES); + + /* Init SNOW3G-UEA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uea2_ooo, 4); + + /* Init SNOW3G-UIA out-of-order fields */ + ooo_mgr_snow3g_reset(state->snow3g_uia2_ooo, 4); } IMB_DLL_LOCAL void diff --git a/lib/avx_t1/mb_mgr_avx_t1.c b/lib/avx_t1/mb_mgr_avx_t1.c index 6c51ecd2..d84f4c2d 100644 --- a/lib/avx_t1/mb_mgr_avx_t1.c +++ b/lib/avx_t1/mb_mgr_avx_t1.c @@ -238,8 +238,8 @@ flush_snow3g_uea2_job_avx_t1(IMB_MGR *state) #define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx_t1 #define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx_t1 -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx_t1 -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx_t1 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local diff --git a/lib/avx_t2/mb_mgr_avx_t2.c b/lib/avx_t2/mb_mgr_avx_t2.c index 08a63a52..977ca576 100644 --- a/lib/avx_t2/mb_mgr_avx_t2.c +++ b/lib/avx_t2/mb_mgr_avx_t2.c @@ -241,8 +241,8 @@ flush_snow3g_uea2_job_avx_t2(IMB_MGR *state) #define SUBMIT_JOB_SNOW3G_UEA2 submit_snow3g_uea2_job_avx_t2 #define FLUSH_JOB_SNOW3G_UEA2 flush_snow3g_uea2_job_avx_t2 -#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_avx_t2 -#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_avx_t2 +#define SUBMIT_JOB_SNOW3G_UIA2 submit_job_snow3g_uia2_sse +#define FLUSH_JOB_SNOW3G_UIA2 flush_job_snow3g_uia2_sse /* AES-DOCSIS */ #define ETHERNET_FCS ethernet_fcs_avx_local diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index a728dfb1..513e190a 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -761,7 +761,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if defined (SSE) || defined (AVX512) +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -819,7 +819,7 @@ SUBMIT_JOB_HASH(IMB_MGR *state, IMB_JOB *job) return SUBMIT_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job, job->auth_tag_output_len_in_bytes); case IMB_AUTH_SNOW3G_UIA2_BITLEN: -#if defined (SSE) || defined (AVX512) +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) return SUBMIT_JOB_SNOW3G_UIA2(snow3g_uia2_ooo, job); #else IMB_SNOW3G_F9_1_BUFFER(state, (const snow3g_key_schedule_t *) @@ -937,7 +937,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) MB_MGR_SHA_256_OOO *sha_256_ooo = state->sha_256_ooo; MB_MGR_SHA_512_OOO *sha_384_ooo = state->sha_384_ooo; MB_MGR_SHA_512_OOO *sha_512_ooo = state->sha_512_ooo; -#if defined(SSE) || defined (AVX512) +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) MB_MGR_SNOW3G_OOO *snow3g_uia2_ooo = state->snow3g_uia2_ooo; #endif @@ -984,7 +984,7 @@ FLUSH_JOB_HASH(IMB_MGR *state, IMB_JOB *job) case IMB_AUTH_ZUC256_EIA3_BITLEN: return FLUSH_JOB_ZUC256_EIA3(zuc256_eia3_ooo, job->auth_tag_output_len_in_bytes); -#if defined(SSE) || defined (AVX512) +#if (defined(SAFE_LOOKUP) || defined(AVX512)) && !defined(SSE_AESNI_EMU) case IMB_AUTH_SNOW3G_UIA2_BITLEN: return FLUSH_JOB_SNOW3G_UIA2(snow3g_uia2_ooo); #endif diff --git a/lib/no-aesni/mb_mgr_sse_no_aesni.c b/lib/no-aesni/mb_mgr_sse_no_aesni.c index 268576aa..f071419d 100644 --- a/lib/no-aesni/mb_mgr_sse_no_aesni.c +++ b/lib/no-aesni/mb_mgr_sse_no_aesni.c @@ -29,6 +29,8 @@ #include #include +#define SSE_AESNI_EMU + #include "intel-ipsec-mb.h" #include "include/ipsec_ooo_mgr.h" #include "include/kasumi_interface.h" -- GitLab From 0ef621d1853fff077a0e8ae296531a748df4e4ba Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 14 Sep 2022 16:07:29 +0000 Subject: [PATCH 353/369] avx512: [aes-ccm] fix for issue #107 in avx512 type-2 implementation --- ..._ccm_auth_submit_flush_x16_vaes_avx512.asm | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm index 9e94ab39..bc5be452 100644 --- a/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm +++ b/lib/avx512_t2/mb_mgr_aes128_ccm_auth_submit_flush_x16_vaes_avx512.asm @@ -495,7 +495,30 @@ endstruc COPY_IV_KEYS_TO_NULL_LANES tmp2, tmp4, tmp3, xmm4, xmm5, k6 ;; Find min length for lanes 0-7 - vphminposuw min_len_idx, XWORD(ccm_lens) + vphminposuw min_len_idx, XWORD(ccm_lens) + jmp %%_ccm_round + +%%_ccm_round_flush: + ;; find null lanes + ;; - vphminposuw already issued + ;; - lens updated + vpxorq zmm7, zmm7, zmm7 + vmovdqu64 zmm8, [state + _aes_ccm_job_in_lane + (0*PTR_SZ)] + vmovdqu64 zmm9, [state + _aes_ccm_job_in_lane + (8*PTR_SZ)] + vpcmpq k4, zmm8, zmm7, 0 ; EQ + vpcmpq k5, zmm9, zmm7, 0 ; EQ + kshiftlw k6, k5, 8 + korw k6, k6, k4 ; masks of NULL jobs in k4 (8), k5 (8) and k6 (16) + knotw k7, k6 ; mask of non-NULL jobs + kmovw DWORD(tmp), k7 + bsf DWORD(tmp2), DWORD(tmp) ; index of the 1st set bit in tmp + + ;; copy good lane data into NULL lanes + mov tmp, [state + _aes_ccm_args_in + tmp2*8] + vpbroadcastq zmm8, tmp + vmovdqa64 [state + _aes_ccm_args_in + (0*PTR_SZ)]{k4}, zmm8 + vmovdqa64 [state + _aes_ccm_args_in + (8*PTR_SZ)]{k5}, zmm8 + %endif ; end FLUSH %%_ccm_round: @@ -655,8 +678,11 @@ endstruc vpbroadcastw ccm_lens{k1}, WORD(tmp) vmovdqa64 [state + _aes_cmac_lens], ccm_lens vphminposuw min_len_idx, XWORD(ccm_lens) - +%ifidn %%SUBMIT_FLUSH, SUBMIT jmp %%_ccm_round +%else + jmp %%_ccm_round_flush +%endif %%_prepare_partial_block_to_auth: ; Check if partial block needs to be hashed @@ -687,8 +713,11 @@ endstruc %%_finish_partial_block_copy: vmovdqa [init_block_addr], xtmp0 mov [state + _aes_ccm_args_in + min_idx * 8], init_block_addr - +%ifidn %%SUBMIT_FLUSH, SUBMIT jmp %%_ccm_round +%else + jmp %%_ccm_round_flush +%endif %endmacro align 64 -- GitLab From 3afa39dfc0ff418e107fddfd86f358c3187882f8 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 14 Sep 2022 21:54:01 +0000 Subject: [PATCH 354/369] lib: [aes-cmac] fix for potential out of scope read - up to 16 bytes can be read outside the buffer on flush operation - number of conditions must be met to make it happen (similar to issue #107) - SSE, AVX implementations affected - all architecture interfaces affected except avx512 type-2 --- ...mb_mgr_aes128_cmac_submit_flush_x8_avx.asm | 21 +++++++++++++++++++ ...mb_mgr_aes128_cmac_submit_flush_x4_sse.asm | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm b/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm index a4e20b97..f3a8bdb4 100644 --- a/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm +++ b/lib/avx_t1/mb_mgr_aes128_cmac_submit_flush_x8_avx.asm @@ -292,6 +292,22 @@ APPEND(skip_,I): %endrep ;; Find min length vphminposuw xmm1, xmm0 + jmp %%_cmac_round + +%%_cmac_round_flush: + ;; - good lane already known + ;; - copy good_lane input pointer to empty lanes + ;; - lens updated and vphminposuw executed + mov tmp2, [state + _aes_cmac_args_in + good_lane*8] + xor tmp3, tmp3 +%assign I 0 +%rep 8 + cmp qword [state + _aes_cmac_job_in_lane + I*8], tmp3 + jne APPEND(skip2_,I) + mov [state + _aes_cmac_args_in + I*8], tmp2 +APPEND(skip2_,I): +%assign I (I+1) +%endrep %endif ; end FLUSH @@ -350,7 +366,12 @@ APPEND(skip_,I): lea m_last, [state + _aes_cmac_scratch + tmp3] mov [state + _aes_cmac_args_in + idx*8], m_last +%ifidn %%SUBMIT_FLUSH, SUBMIT jmp %%_cmac_round +%else + mov good_lane, idx + jmp %%_cmac_round_flush +%endif %%_copy_complete_digest: ; Job complete, copy digest to AT output diff --git a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm index 358c67c1..be832f5b 100644 --- a/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm +++ b/lib/sse_t1/mb_mgr_aes128_cmac_submit_flush_x4_sse.asm @@ -301,6 +301,22 @@ APPEND(skip_,I): %endrep ;; Find min length phminposuw xmm1, xmm0 + jmp %%_cmac_round + +%%_cmac_round_flush: + ;; - good lane already known + ;; - copy good_lane input pointer to empty lanes + ;; - lens updated and vphminposuw executed + mov tmp2, [state + _aes_cmac_args_in + good_lane*8] + xor tmp3, tmp3 +%assign I 0 +%rep NUM_LANES + cmp qword [state + _aes_cmac_job_in_lane + I*8], tmp3 + jne APPEND(skip2_,I) + mov [state + _aes_cmac_args_in + I*8], tmp2 +APPEND(skip2_,I): +%assign I (I+1) +%endrep %endif ; end FLUSH @@ -369,7 +385,12 @@ APPEND(skip_,I): lea m_last, [state + _aes_cmac_scratch + tmp3] mov [state + _aes_cmac_args_in + idx*8], m_last +%ifidn %%SUBMIT_FLUSH, SUBMIT jmp %%_cmac_round +%else + mov good_lane, idx + jmp %%_cmac_round_flush +%endif %%_copy_complete_digest: ; Job complete, copy digest to AT output -- GitLab From 5cc000652df44d4135bce84a77d992afaa283f9a Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 15 Sep 2022 12:44:06 +0000 Subject: [PATCH 355/369] lib: [self-test] self-test complete manager feature flag added --- README | 16 +++++++++++++++- README.md | 17 +++++++++++++++-- lib/intel-ipsec-mb.h | 1 + lib/x86_64/self_test.c | 3 +++ test/acvp_app_main.c | 23 ++++++++++++++++++----- test/ipsec_xvalid.c | 8 ++++++++ test/main.c | 7 +++++++ test/wycheproof/wycheproof.c | 7 +++++++ 8 files changed, 74 insertions(+), 8 deletions(-) diff --git a/README b/README index 5dd448b1..02336484 100644 --- a/README +++ b/README @@ -741,12 +741,26 @@ answer test) on following algorithms: - HMAC-SHA384 - HMAC-SHA512 -Detecting library self-test failure in the application: +Example detection of library self-test completion & error in the +application: IMB_ARCH arch; IMB_MGR *p_mgr = alloc_mb_mgr(0); init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ + + /* + * check for self-test successful completion + * - requires library version v1.3 or newer + */ + if (imb_get_version() >= IMB_VERSION(1,3,0)) + printf("SELF-TEST: %s\n", + (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + "PASS" : "FAIL"); + else + printf("SELF-TEST: N/A (requires library >= v1.3)\n"); + + /* check for initialization self-test error */ if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { /* self-test error */ exit(EXIT_FAILURE); diff --git a/README.md b/README.md index 9c7c3e62..6ee63aaa 100644 --- a/README.md +++ b/README.md @@ -676,14 +676,27 @@ The self-test consists of Cryptographic algorithm test (known answer test) on fo - HMAC-SHA384 - HMAC-SHA512 -Detecting library self-test failure in the application: +Example detection of library self-test completion & error in the application: ``` IMB_ARCH arch; IMB_MGR *p_mgr = alloc_mb_mgr(0); init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ + +/* + * check for self-test successful completion + * - requires library version v1.3 or newer + */ +if (imb_get_version() >= IMB_VERSION(1,3,0)) + printf("SELF-TEST: %s\n", + (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + "PASS" : "FAIL"); +else + printf("SELF-TEST: N/A (requires library >= v1.3)\n"); + +/* check for initialization self-test error */ if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { /* self-test error */ exit(EXIT_FAILURE); } -``` \ No newline at end of file +``` diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index c55b33b8..3801f094 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1031,6 +1031,7 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_FEATURE_AVX512_IFMA (1ULL << 17) #define IMB_FEATURE_BMI2 (1ULL << 18) #define IMB_FEATURE_AESNI_EMU (1ULL << 19) +#define IMB_FEATURE_SELF_TEST (1ULL << 20) /** * CPU flags needed for each implementation diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index a126f1bf..29db2847 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -1571,6 +1571,8 @@ static int self_test_aead(IMB_MGR *p_mgr) IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) { + p_mgr->features &= ~IMB_FEATURE_SELF_TEST; + if (!self_test_ciphers(p_mgr)) return 0; @@ -1580,5 +1582,6 @@ IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) if (!self_test_aead(p_mgr)) return 0; + p_mgr->features |= IMB_FEATURE_SELF_TEST; return 1; } diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index da7c246e..9e709c90 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1481,23 +1481,36 @@ int main(int argc, char **argv) goto exit; } + if (imb_get_version() >= IMB_VERSION(1, 3, 0)) { + if ((mb_mgr != NULL) && /* scan-build needs this check */ + (mb_mgr->features & IMB_FEATURE_SELF_TEST)) + printf("SELF-TEST: PASS\n"); + else + printf("SELF-TEST: FAIL\n"); + } else { + printf("SELF-TEST: N/A (requires >= v1.3)\n"); + } + if (imb_get_errno(mb_mgr) != 0) { fprintf(stderr, "Error initializing MB_MGR structure! %s\n", imb_get_strerror(imb_get_errno(mb_mgr))); - free_mb_mgr(mb_mgr); goto exit; } /* Parse request file, run crypto tests and write out response file */ acvp_run_vectors_from_file(ctx, req_filename, resp_filename); + ret = EXIT_SUCCESS; + +exit: /* Free MB_MGR and test session */ - free_mb_mgr(mb_mgr); + if (mb_mgr != NULL) + free_mb_mgr(mb_mgr); + if (ctx != NULL) acvp_free_test_session(ctx); - ret = EXIT_SUCCESS; -exit: - free(req_filename); + if (req_filename != NULL) + free(req_filename); return ret; } diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index ef870c28..a4a339bc 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -2479,6 +2479,10 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } + if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + if (!(enc_mgr->features & IMB_FEATURE_SELF_TEST)) + fprintf(stderr, "SELF-TEST: FAIL\n"); + if (imb_get_errno(enc_mgr) != 0) { fprintf(stderr, "Error initializing enc MB_MGR structure! %s\n", imb_get_strerror(imb_get_errno(enc_mgr))); @@ -2522,6 +2526,10 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } + if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + if (!(dec_mgr->features & IMB_FEATURE_SELF_TEST)) + fprintf(stderr, "SELF-TEST: FAIL\n"); + if (imb_get_errno(dec_mgr) != 0) { fprintf(stderr, "Error initializing dec MB_MGR structure! %s\n", imb_get_strerror(imb_get_errno(dec_mgr))); diff --git a/test/main.c b/test/main.c index 8ff63103..c14af81b 100644 --- a/test/main.c +++ b/test/main.c @@ -444,6 +444,13 @@ main(int argc, char **argv) break; } + if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + printf("SELF-TEST: %s\n", + (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + "PASS" : "FAIL"); + else + printf("SELF-TEST: N/A (requires library >= v1.3)\n"); + if (imb_get_errno(p_mgr) != 0) { printf("Error initializing MB_MGR structure! %s\n", imb_get_strerror(imb_get_errno(p_mgr))); diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof/wycheproof.c index 95167709..70d331ab 100644 --- a/test/wycheproof/wycheproof.c +++ b/test/wycheproof/wycheproof.c @@ -1449,6 +1449,13 @@ int main(int argc, const char **argv) break; } + if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + printf("SELF-TEST: %s\n", + (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + "PASS" : "FAIL"); + else + printf("SELF-TEST: N/A (requires library >= v1.3)\n"); + if (imb_get_errno(p_mgr) != 0) { printf("Error initializing MB_MGR structure! %s\n", imb_get_strerror(imb_get_errno(p_mgr))); -- GitLab From 1d470a11a5e69983ff09d1d754b3082f885a433d Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Fri, 16 Sep 2022 14:44:51 +0100 Subject: [PATCH 356/369] test: cleanup issues produced by cppcheck --- test/ctr_test.c | 8 ++++---- test/gcm_test.c | 5 +++-- test/hmac_sha1_test.c | 12 ++++++------ test/hmac_sha256_sha512_test.c | 12 ++++++------ 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/test/ctr_test.c b/test/ctr_test.c index 153f2ba4..4c8c9310 100644 --- a/test/ctr_test.c +++ b/test/ctr_test.c @@ -1530,7 +1530,7 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } if (memcmp(out_text, targets[i] + sizeof(padding), @@ -1558,7 +1558,7 @@ test_ctr_burst(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; @@ -1656,7 +1656,7 @@ test_ctr_cipher_burst(struct IMB_MGR *mb_mgr, job = &jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } if (memcmp(out_text, targets[i] + sizeof(padding), @@ -1684,7 +1684,7 @@ test_ctr_cipher_burst(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; diff --git a/test/gcm_test.c b/test/gcm_test.c index ba1bd767..cdbc9c54 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -1701,7 +1701,7 @@ aes_gcm_burst(IMB_MGR *mb_mgr, job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); return -1; } } @@ -2160,7 +2160,8 @@ test_gcm_vectors_burst(struct gcm_ctr_vector const *vector, uint8_t **T_test = NULL; const uint8_t *iv = vector->IV; uint64_t iv_len = vector->IVlen; - uint32_t i, j, num_jobs = GCM_MAX_JOBS; + uint32_t i, j; + const uint32_t num_jobs = GCM_MAX_JOBS; /* Allocate space for the calculated ciphertext */ ct_test = malloc(num_jobs * sizeof(void *)); diff --git a/test/hmac_sha1_test.c b/test/hmac_sha1_test.c index fa45adc9..bd800f66 100644 --- a/test/hmac_sha1_test.c +++ b/test/hmac_sha1_test.c @@ -457,7 +457,7 @@ test_hmac_sha1(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; @@ -580,7 +580,7 @@ check_burst_jobs: job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } @@ -595,7 +595,7 @@ check_burst_jobs: num_jobs - completed_jobs, jobs); if (completed_jobs == 0) { - printf("Expected %d jobs, received %d\n", + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } @@ -720,7 +720,7 @@ test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, job = &jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } @@ -731,7 +731,7 @@ test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; @@ -757,7 +757,7 @@ test_hmac_sha1_std_vectors(struct IMB_MGR *mb_mgr, const int vectors_cnt = DIM(hmac_sha1_vectors); int vect; - printf("HMAC-SHA1 standard test vectors (N jobs = %d):\n", num_jobs); + printf("HMAC-SHA1 standard test vectors (N jobs = %u):\n", num_jobs); for (vect = 1; vect <= vectors_cnt; vect++) { const int idx = vect - 1; #ifdef DEBUG diff --git a/test/hmac_sha256_sha512_test.c b/test/hmac_sha256_sha512_test.c index 8549934e..b1d8c0d0 100644 --- a/test/hmac_sha256_sha512_test.c +++ b/test/hmac_sha256_sha512_test.c @@ -1022,7 +1022,7 @@ test_hmac_shax(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; @@ -1232,7 +1232,7 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, job = jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } @@ -1248,7 +1248,7 @@ test_hmac_shax_burst(struct IMB_MGR *mb_mgr, num_jobs - completed_jobs, jobs); if (completed_jobs == 0) { - printf("Expected %d jobs, received %d\n", + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } @@ -1468,7 +1468,7 @@ test_hmac_shax_hash_burst(struct IMB_MGR *mb_mgr, job = &jobs[i]; if (job->status != IMB_STATUS_COMPLETED) { - printf("job %d status not complete!\n", i+1); + printf("job %u status not complete!\n", i+1); goto end; } @@ -1480,7 +1480,7 @@ test_hmac_shax_hash_burst(struct IMB_MGR *mb_mgr, } if (jobs_rx != num_jobs) { - printf("Expected %d jobs, received %d\n", num_jobs, jobs_rx); + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); goto end; } ret = 0; @@ -1507,7 +1507,7 @@ test_hmac_shax_std_vectors(struct IMB_MGR *mb_mgr, const int vectors_cnt = DIM(hmac_sha256_sha512_vectors); int vect; - printf("HMAC-SHA%d standard test vectors (N jobs = %d):\n", + printf("HMAC-SHA%d standard test vectors (N jobs = %u):\n", sha_type, num_jobs); for (vect = 1; vect <= vectors_cnt; vect++) { const int idx = vect - 1; -- GitLab From dae20d1e27c06aae8882a3466dc30533cf26c68d Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Mon, 19 Sep 2022 11:32:02 +0000 Subject: [PATCH 357/369] test: [wycheproof] --no-gfni command line option added --- test/wycheproof/wycheproof.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof/wycheproof.c index 70d331ab..53d1d27e 100644 --- a/test/wycheproof/wycheproof.c +++ b/test/wycheproof/wycheproof.c @@ -1375,7 +1375,9 @@ usage(const char *name) "--avx test AVX interface\n" "--sse test SSE interface\n" "--shani-off don't use SHA extensions " - "(auto-detect is default)\n", name); + "(auto-detect by default)\n" + "--gfni-off don't use GFNI extensions " + "(auto-detect by default)\n", name); } int main(int argc, const char **argv) @@ -1411,6 +1413,8 @@ int main(int argc, const char **argv) arch_to_run = IMB_ARCH_AVX512; } else if (strcmp(argv[i], "--shani-off") == 0) { flags |= IMB_FLAG_SHANI_OFF; + } else if (strcmp(argv[i], "--gfni-off") == 0) { + flags |= IMB_FLAG_GFNI_OFF; } } -- GitLab From 8473a4b21c059c33e7b0e76a0b2aef69cf384811 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 20 Sep 2022 11:14:53 +0100 Subject: [PATCH 358/369] test: Initialize tag buffers before use Fix for issues reported by Coverity. --- test/acvp_app_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index 9e709c90..c94c2b2f 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -244,7 +244,7 @@ static int aes_gcm_handler(ACVP_TEST_CASE *test_case) } } } else /* DECRYPT */ { - uint8_t res_tag[MAX_TAG_LENGTH]; + uint8_t res_tag[MAX_TAG_LENGTH] = {0}; if (direct_api == 1) { gcm_init_var_iv(&key, &ctx, tc->iv, tc->iv_len, @@ -377,7 +377,7 @@ static int aes_gmac_handler(ACVP_TEST_CASE *test_case) } } } else /* DECRYPT */ { - uint8_t res_tag[MAX_TAG_LENGTH]; + uint8_t res_tag[MAX_TAG_LENGTH] = {0}; if (direct_api == 1) { gmac_init_var(&key, &ctx, tc->iv, tc->iv_len); -- GitLab From 0d5803c282308ecccc5a9c0d111552e1a4d359e8 Mon Sep 17 00:00:00 2001 From: Marcel Cornu Date: Tue, 20 Sep 2022 13:01:02 +0100 Subject: [PATCH 359/369] test: Fix various static analysis issues --- test/chacha20_poly1305_test.c | 5 +++++ test/gcm_test.c | 5 +++++ test/job_api_fuzz_test.c | 18 ++++++++++-------- test/wycheproof/wycheproof.c | 15 +++++++-------- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/test/chacha20_poly1305_test.c b/test/chacha20_poly1305_test.c index 4009a0ef..d952baea 100644 --- a/test/chacha20_poly1305_test.c +++ b/test/chacha20_poly1305_test.c @@ -658,6 +658,11 @@ test_single_job_sgl(struct IMB_MGR *mb_mgr, const uint32_t num_segments = DIV_ROUND_UP(buffer_sz, seg_sz); sgl_segs = malloc(sizeof(struct IMB_SGL_IOV) * num_segments); + if (sgl_segs == NULL) { + fprintf(stderr, "Could not allocate memory for SGL segments\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } if (last_seg_sz == 0) last_seg_sz = seg_sz; diff --git a/test/gcm_test.c b/test/gcm_test.c index cdbc9c54..092ae92a 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -2700,6 +2700,11 @@ test_single_job_sgl(struct IMB_MGR *mb_mgr, last_seg_sz = seg_sz; sgl_segs = malloc(sizeof(struct IMB_SGL_IOV) * num_segments); + if (sgl_segs == NULL) { + fprintf(stderr, "Could not allocate memory for SGL segments\n"); + test_suite_update(ctx, 0, 1); + goto exit; + } in_buffer = malloc(buffer_sz); if (in_buffer == NULL) { diff --git a/test/job_api_fuzz_test.c b/test/job_api_fuzz_test.c index a2b7fbb3..6ae3283c 100644 --- a/test/job_api_fuzz_test.c +++ b/test/job_api_fuzz_test.c @@ -450,14 +450,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_HASH_ALG hash; IMB_CIPHER_MODE cipher; IMB_CIPHER_DIRECTION dir; - char *ar; - const char *api; IMB_MGR *p_mgr = NULL; IMB_ARCH arch; unsigned i; + const char *ar = getenv("ARCH"); + const char *api = getenv("API"); const char *n_jobs = getenv("NUM_JOBS"); const char *key_length = getenv("KEY_LEN"); + const char *cipher_dir = getenv("DIR"); unsigned num_jobs; unsigned key_len; const size_t buffsize = BUFF_SIZE; @@ -480,11 +481,15 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) if (num_jobs > 32 || num_jobs == 0 || key_len == 0) return 0; - if (getenv("DIR")) { - if (strcmp(getenv("DIR"), "ENCRYPT") == 0) + if (cipher_dir != NULL) { + if (strcmp(cipher_dir, "ENCRYPT") == 0) dir = IMB_DIR_ENCRYPT; - else if (strcmp(getenv("DIR"), "DECRYPT") == 0) + else if (strcmp(cipher_dir, "DECRYPT") == 0) dir = IMB_DIR_DECRYPT; + else { + printf("Invalid cipher direction!\n"); + return EXIT_FAILURE; + } } else { dir = IMB_DIR_ENCRYPT; } @@ -495,7 +500,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) printf("Error allocating MB_MGR structure!\n"); return EXIT_FAILURE; } - ar = getenv("ARCH"); if (ar == NULL) { init_mb_mgr_auto(p_mgr, &arch); } else { @@ -515,8 +519,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) /* create job array */ IMB_JOB *jobs[32] = {NULL}; - api = getenv("API"); - if (api == NULL || (strcmp(api, "SINGLE") == 0)) { single = true; } else if (strcmp(api, "BURST") == 0) { diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof/wycheproof.c index 53d1d27e..c8256651 100644 --- a/test/wycheproof/wycheproof.c +++ b/test/wycheproof/wycheproof.c @@ -399,10 +399,10 @@ static int test_hmac_sha1(IMB_MGR *p_mgr) const void *key_ptr = NULL; size_t key_size = 0; - IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); /* tag too long */ - if (v->tagSize > (sizeof(scratch) * 8)) { + if (v->tagSize > (sizeof(tag) * 8)) { print_mac_test(v); return 0; } @@ -433,8 +433,7 @@ static int test_hmac_sha1(IMB_MGR *p_mgr) job->auth_tag_output_len_in_bytes = IMB_SHA1_DIGEST_SIZE_IN_BYTES; else - job->auth_tag_output_len_in_bytes = - v->tagSize / 8; + job->auth_tag_output_len_in_bytes = v->tagSize / 8; /* prepare key */ if ((v->keySize / 8) <= IMB_SHA1_BLOCK_SIZE) { @@ -484,10 +483,10 @@ static int test_hmac_sha224(IMB_MGR *p_mgr) const void *key_ptr = NULL; size_t key_size = 0; - IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); /* tag too long */ - if (v->tagSize > (sizeof(scratch) * 8)) { + if (v->tagSize > (sizeof(tag) * 8)) { print_mac_test(v); return 0; } @@ -560,10 +559,10 @@ static int test_hmac_sha256(IMB_MGR *p_mgr) const void *key_ptr = NULL; size_t key_size = 0; - IMB_ASSERT((v->tagSize / 8) <= sizeof(scratch)); + IMB_ASSERT((v->tagSize / 8) <= sizeof(tag)); /* tag too long */ - if (v->tagSize > (sizeof(scratch) * 8)) { + if (v->tagSize > (sizeof(tag) * 8)) { print_mac_test(v); return 0; } -- GitLab From 9731adbe51e8828378f500fa5a56be3145946a07 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Mon, 19 Sep 2022 11:57:12 +0100 Subject: [PATCH 360/369] Version bump to 1.3 --- lib/intel-ipsec-mb.h | 4 ++-- rpm/SUSE/intel-ipsec-mb.spec | 2 +- rpm/intel-ipsec-mb.spec | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 3801f094..6abeee77 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -91,8 +91,8 @@ typedef struct { /** * Library version */ -#define IMB_VERSION_STR "1.2.0" -#define IMB_VERSION_NUM 0x10200 +#define IMB_VERSION_STR "1.3.0" +#define IMB_VERSION_NUM 0x10300 /** * Macro to translate version number diff --git a/rpm/SUSE/intel-ipsec-mb.spec b/rpm/SUSE/intel-ipsec-mb.spec index 0ee0733c..e1c7d673 100644 --- a/rpm/SUSE/intel-ipsec-mb.spec +++ b/rpm/SUSE/intel-ipsec-mb.spec @@ -18,7 +18,7 @@ # Versions numbers %global major 1 -%global minor 2 +%global minor 3 %global patch 0 # GitHub properties diff --git a/rpm/intel-ipsec-mb.spec b/rpm/intel-ipsec-mb.spec index 61ecbcf3..1ba73250 100644 --- a/rpm/intel-ipsec-mb.spec +++ b/rpm/intel-ipsec-mb.spec @@ -25,7 +25,7 @@ # Versions numbers %global major 1 -%global minor 2 +%global minor 3 %global patch 0 %global fullversion %{major}.%{minor}.%{patch} @@ -109,6 +109,9 @@ ln -s libIPSec_MB.so.%{fullversion} libIPSec_MB.so %{_libdir}/libIPSec_MB.so %changelog +* Tue Sep 20 2022 Stephen Mcintyre 1.3.0-1 +- Update for release package v1.3 + * Fri Feb 11 2022 Stephen Mcintyre 1.2.0-1 - Update for release package v1.2 -- GitLab From f3e99483aaf8bd475d0fda02fb61a09b76a4d378 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 16 Sep 2022 16:18:45 +0000 Subject: [PATCH 361/369] lib: [self-test] two self-test feature flags created: presence and test status Compile option added that removes self-test feature and ignores status of the self-test - only to be used for development or debug - enabled with 'make -j EXTRA_CFLAGS=-DNO_SELF_TEST_DEV' --- README | 16 ++++++++++------ README.md | 16 ++++++++++------ lib/intel-ipsec-mb.h | 3 ++- lib/x86_64/self_test.c | 23 +++++++++++++++++------ test/acvp_app_main.c | 5 ++--- test/ipsec_xvalid.c | 8 ++++---- test/main.c | 4 ++-- test/wycheproof/wycheproof.c | 4 ++-- 8 files changed, 49 insertions(+), 30 deletions(-) diff --git a/README b/README index 02336484..84ea575b 100644 --- a/README +++ b/README @@ -750,15 +750,19 @@ application: init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ /* - * check for self-test successful completion + * check for self-test presence and successful * - requires library version v1.3 or newer */ - if (imb_get_version() >= IMB_VERSION(1,3,0)) - printf("SELF-TEST: %s\n", - (p_mgr->features & IMB_FEATURE_SELF_TEST) ? - "PASS" : "FAIL"); - else + if (p_mgr->features & IMB_FEATURE_SELF_TEST) { + /* self-test feature present */ + if (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) { + printf("SELF-TEST: PASS\n"); + } else { + printf("SELF-TEST: FAIL\n"); + } + } else { printf("SELF-TEST: N/A (requires library >= v1.3)\n"); + } /* check for initialization self-test error */ if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { diff --git a/README.md b/README.md index 6ee63aaa..32a3b22b 100644 --- a/README.md +++ b/README.md @@ -684,15 +684,19 @@ IMB_MGR *p_mgr = alloc_mb_mgr(0); init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ /* - * check for self-test successful completion + * check for self-test presence and successful * - requires library version v1.3 or newer */ -if (imb_get_version() >= IMB_VERSION(1,3,0)) - printf("SELF-TEST: %s\n", - (p_mgr->features & IMB_FEATURE_SELF_TEST) ? - "PASS" : "FAIL"); -else +if (p_mgr->features & IMB_FEATURE_SELF_TEST) { + /* self-test feature present */ + if (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) { + printf("SELF-TEST: PASS\n"); + } else { + printf("SELF-TEST: FAIL\n"); + } +} else { printf("SELF-TEST: N/A (requires library >= v1.3)\n"); +} /* check for initialization self-test error */ if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 6abeee77..0dff3c12 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -1031,7 +1031,8 @@ typedef uint32_t (*crc32_fn_t)(const void *, const uint64_t); #define IMB_FEATURE_AVX512_IFMA (1ULL << 17) #define IMB_FEATURE_BMI2 (1ULL << 18) #define IMB_FEATURE_AESNI_EMU (1ULL << 19) -#define IMB_FEATURE_SELF_TEST (1ULL << 20) +#define IMB_FEATURE_SELF_TEST (1ULL << 20) /* self-test feature present */ +#define IMB_FEATURE_SELF_TEST_PASS (1ULL << 21) /* self-test passed */ /** * CPU flags needed for each implementation diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 29db2847..fb661b8a 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -1571,17 +1571,28 @@ static int self_test_aead(IMB_MGR *p_mgr) IMB_DLL_LOCAL int self_test(IMB_MGR *p_mgr) { - p_mgr->features &= ~IMB_FEATURE_SELF_TEST; + int ret = 1; + + p_mgr->features |= IMB_FEATURE_SELF_TEST; + p_mgr->features &= ~IMB_FEATURE_SELF_TEST_PASS; if (!self_test_ciphers(p_mgr)) - return 0; + ret = 0; if (!self_test_hash(p_mgr)) - return 0; + ret = 0; if (!self_test_aead(p_mgr)) - return 0; + ret = 0; - p_mgr->features |= IMB_FEATURE_SELF_TEST; - return 1; + if (ret) + p_mgr->features |= IMB_FEATURE_SELF_TEST_PASS; + +#ifdef NO_SELF_TEST_DEV + p_mgr->features &= ~(IMB_FEATURE_SELF_TEST | + IMB_FEATURE_SELF_TEST_PASS); + ret = 1; +#endif + + return ret; } diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index c94c2b2f..cc27c555 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1481,9 +1481,8 @@ int main(int argc, char **argv) goto exit; } - if (imb_get_version() >= IMB_VERSION(1, 3, 0)) { - if ((mb_mgr != NULL) && /* scan-build needs this check */ - (mb_mgr->features & IMB_FEATURE_SELF_TEST)) + if ((mb_mgr != NULL) && (mb_mgr->features & IMB_FEATURE_SELF_TEST)) { + if (mb_mgr->features & IMB_FEATURE_SELF_TEST_PASS) printf("SELF-TEST: PASS\n"); else printf("SELF-TEST: FAIL\n"); diff --git a/test/ipsec_xvalid.c b/test/ipsec_xvalid.c index a4a339bc..0e32f646 100644 --- a/test/ipsec_xvalid.c +++ b/test/ipsec_xvalid.c @@ -2479,8 +2479,8 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } - if (imb_get_version() >= IMB_VERSION(1, 3, 0)) - if (!(enc_mgr->features & IMB_FEATURE_SELF_TEST)) + if (enc_mgr->features & IMB_FEATURE_SELF_TEST) + if (!(enc_mgr->features & IMB_FEATURE_SELF_TEST_PASS)) fprintf(stderr, "SELF-TEST: FAIL\n"); if (imb_get_errno(enc_mgr) != 0) { @@ -2526,8 +2526,8 @@ run_test(const IMB_ARCH enc_arch, const IMB_ARCH dec_arch, exit(EXIT_FAILURE); } - if (imb_get_version() >= IMB_VERSION(1, 3, 0)) - if (!(dec_mgr->features & IMB_FEATURE_SELF_TEST)) + if (dec_mgr->features & IMB_FEATURE_SELF_TEST) + if (!(dec_mgr->features & IMB_FEATURE_SELF_TEST_PASS)) fprintf(stderr, "SELF-TEST: FAIL\n"); if (imb_get_errno(dec_mgr) != 0) { diff --git a/test/main.c b/test/main.c index c14af81b..9cfb68c6 100644 --- a/test/main.c +++ b/test/main.c @@ -444,9 +444,9 @@ main(int argc, char **argv) break; } - if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + if (p_mgr->features & IMB_FEATURE_SELF_TEST) printf("SELF-TEST: %s\n", - (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) ? "PASS" : "FAIL"); else printf("SELF-TEST: N/A (requires library >= v1.3)\n"); diff --git a/test/wycheproof/wycheproof.c b/test/wycheproof/wycheproof.c index c8256651..790fc6e6 100644 --- a/test/wycheproof/wycheproof.c +++ b/test/wycheproof/wycheproof.c @@ -1452,9 +1452,9 @@ int main(int argc, const char **argv) break; } - if (imb_get_version() >= IMB_VERSION(1, 3, 0)) + if (p_mgr->features & IMB_FEATURE_SELF_TEST) printf("SELF-TEST: %s\n", - (p_mgr->features & IMB_FEATURE_SELF_TEST) ? + (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) ? "PASS" : "FAIL"); else printf("SELF-TEST: N/A (requires library >= v1.3)\n"); -- GitLab From a02f5129fb84d3b428a42e5272e8d532ca6ce87e Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Fri, 16 Sep 2022 14:56:18 +0000 Subject: [PATCH 362/369] doc: [README] DLL injection section added --- README | 36 ++++++++++++++++++++++++++++++++++++ README.md | 23 +++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/README b/README index 84ea575b..8d8b51ab 100644 --- a/README +++ b/README @@ -45,6 +45,7 @@ CONTENTS 10. Disclaimer (ZUC, KASUMI, SNOW3G) 11. Legal Disclaimer 12. FIPS Compliance +13. DLL Injection @@ -769,3 +770,38 @@ application: /* self-test error */ exit(EXIT_FAILURE); } + + + +13.DLL INJECTION ATTACK + + +Problem + +The Windows OS has an insecure predefined search order and set of +defaults when trying to locate a resource. If the resource location is +not specified by the software, an attacker need only place a malicious +version in one of the locations Windows will search, and it will be +loaded instead. Although this weakness can occur with any resource, it +is especially common with DLL files. + +Solutions + +Applications using intel-ipsec-mb DLL library may need to apply one of +the solutions to prevent from DLL injection attack. + +Two solutions are available: - Using a Fully Qualified Path is the most +secure way to load a DLL +- Signature verification of the DLL + +Resources and Solution Details + +- Security remarks section of LoadLibraryEx documentation by + Microsoft: + https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexa#security-remarks +- Microsoft Dynamic Link Library Security article: + https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-security +- Hijack Execution Flow: DLL Search Order Hijacking: + https://attack.mitre.org/techniques/T1574/001 +- Hijack Execution Flow: DLL Side-Loading: + https://attack.mitre.org/techniques/T1574/002 diff --git a/README.md b/README.md index 32a3b22b..a0426423 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ Contents 10. Disclaimer (ZUC, KASUMI, SNOW3G) 11. Legal Disclaimer 12. FIPS Compliance +13. DLL Injection 1\. Overview ============ @@ -704,3 +705,25 @@ if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { exit(EXIT_FAILURE); } ``` + +13\.DLL Injection Attack +======================= + +### Problem + +The Windows OS has an insecure predefined search order and set of defaults when trying to locate a resource. If the resource location is not specified by the software, an attacker need only place a malicious version in one of the locations Windows will search, and it will be loaded instead. Although this weakness can occur with any resource, it is especially common with DLL files. + +### Solutions + +Applications using intel-ipsec-mb DLL library may need to apply one of the solutions to prevent from DLL injection attack. + +Two solutions are available: +- Using a Fully Qualified Path is the most secure way to load a DLL +- Signature verification of the DLL + +### Resources and Solution Details + +- Security remarks section of LoadLibraryEx documentation by Microsoft: +- Microsoft Dynamic Link Library Security article: +- Hijack Execution Flow: DLL Search Order Hijacking: +- Hijack Execution Flow: DLL Side-Loading: -- GitLab From eb347f7fba8a7ce9b296758865068fd09c881aab Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Wed, 21 Sep 2022 21:17:15 +0000 Subject: [PATCH 363/369] lib: [spellcheck] update to codespell 2.12 plus fixes reported by it --- Makefile | 2 +- lib/avx2_t1/gcm_avx_gen4.asm | 2 +- lib/avx512_t1/gcm_avx512.asm | 2 +- lib/avx512_t2/pon_vaes_avx512.asm | 14 ++--- lib/avx_t1/gcm_avx_gen2.asm | 2 +- lib/avx_t1/pon_by8_avx.asm | 94 +++++++++++++++---------------- lib/include/kasumi_internal.h | 10 ++-- lib/sse_t1/gcm_sse.inc | 2 +- lib/sse_t1/pon_by8_sse.asm | 44 +++++++-------- perf/win_x64.mak | 2 +- test/gcm_test.c | 2 +- test/test_api.py | 4 +- test/win_x64.mak | 2 +- test/wycheproof/win_x64.mak | 2 +- 14 files changed, 92 insertions(+), 92 deletions(-) diff --git a/Makefile b/Makefile index 37f1f92b..08b1da71 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ TAGS: # -I FILE -- File containing words to be ignored # CODESPELL ?= codespell -CS_IGNORE_WORDS ?= iinclude,struc,fo,ue,od,ba +CS_IGNORE_WORDS ?= iinclude,struc,fo,ue,od,ba,padd .PHONY: spellcheck spellcheck: diff --git a/lib/avx2_t1/gcm_avx_gen4.asm b/lib/avx2_t1/gcm_avx_gen4.asm index a4cb7be8..189280bd 100644 --- a/lib/avx2_t1/gcm_avx_gen4.asm +++ b/lib/avx2_t1/gcm_avx_gen4.asm @@ -491,7 +491,7 @@ default rel %ifidn %%ENC_DEC, DEC vmovdqa xmm3, xmm1 - vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + vpxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) mov r15, %%PLAIN_CYPH_LEN add r15, r13 diff --git a/lib/avx512_t1/gcm_avx512.asm b/lib/avx512_t1/gcm_avx512.asm index f8c44a8c..352a89c5 100644 --- a/lib/avx512_t1/gcm_avx512.asm +++ b/lib/avx512_t1/gcm_avx512.asm @@ -483,7 +483,7 @@ default rel %ifidn %%ENC_DEC, DEC vmovdqa xmm3, xmm1 %endif - vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + vpxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) mov r15, %%PLAIN_CYPH_LEN add r15, r13 diff --git a/lib/avx512_t2/pon_vaes_avx512.asm b/lib/avx512_t2/pon_vaes_avx512.asm index 1ab0de6c..1f712f15 100644 --- a/lib/avx512_t2/pon_vaes_avx512.asm +++ b/lib/avx512_t2/pon_vaes_avx512.asm @@ -200,7 +200,7 @@ endstruc %endmacro %macro AES128_CTR_PON_ENC 1 -%define %%CIPH %1 ; [in] cipher "CTR" or "NO_CTR" +%define %%CIPHER %1 ; [in] cipher "CTR" or "NO_CTR" sub rsp, STACKFRAME_size @@ -260,7 +260,7 @@ endstruc ; get output buffer mov dst, [job + _dst] -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ; Encrypt buffer and calculate BIP in the same function mov arg2, dst @@ -325,7 +325,7 @@ endstruc vpxord xmm1, xmm0 vmovq bip, xmm1 -%endif ; CIPH = CTR +%endif ; CIPHER = CTR mov tmp_1, [job + _auth_tag_output] mov [tmp_1], DWORD(bip) @@ -351,7 +351,7 @@ endstruc %endmacro %macro AES128_CTR_PON_DEC 1 -%define %%CIPH %1 ; [in] cipher "CTR" or "NO_CTR" +%define %%CIPHER %1 ; [in] cipher "CTR" or "NO_CTR" sub rsp, STACKFRAME_size @@ -384,7 +384,7 @@ endstruc ; Save job pointer mov [rsp + _job_save], job -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; Decrypt message and calculate BIP in same function mov arg2, [job + _dst] mov arg3, [job + _iv] @@ -410,7 +410,7 @@ endstruc %ifndef LINUX add rsp, 8*6 %endif -%else ; %%CIPH == CTR +%else ; %%CIPHER == CTR ; Calculate BIP (XOR message) vmovq xmm1, bip @@ -449,7 +449,7 @@ endstruc vmovd DWORD(bip), xmm1 -%endif ; CIPH == CTR +%endif ; CIPHER == CTR cmp bytes_to_crc, 4 jle %%_skip_crc diff --git a/lib/avx_t1/gcm_avx_gen2.asm b/lib/avx_t1/gcm_avx_gen2.asm index a79c89d2..db90c7b6 100644 --- a/lib/avx_t1/gcm_avx_gen2.asm +++ b/lib/avx_t1/gcm_avx_gen2.asm @@ -528,7 +528,7 @@ mksection .text %ifidn %%ENC_DEC, DEC vmovdqa xmm3, xmm1 - vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + vpxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) mov r15, %%PLAIN_CYPH_LEN add r15, r13 diff --git a/lib/avx_t1/pon_by8_avx.asm b/lib/avx_t1/pon_by8_avx.asm index fe6dbf32..74f2f27d 100644 --- a/lib/avx_t1/pon_by8_avx.asm +++ b/lib/avx_t1/pon_by8_avx.asm @@ -247,7 +247,7 @@ mksection .text ;;; ============================================================================ ;;; PON stitched algorithm round on a single AES block (16 bytes): -;;; AES-CTR (optional, depending on %%CIPH) +;;; AES-CTR (optional, depending on %%CIPHER) ;;; - prepares counter block ;;; - encrypts counter block ;;; - loads text @@ -273,10 +273,10 @@ mksection .text %define %%TXMM2 %11 ; [clobbered] XMM temporary %define %%CRC_TYPE %12 ; [in] "first_crc" or "next_crc" or "no_crc" %define %%DIR %13 ; [in] "ENC" or "DEC" -%define %%CIPH %14 ; [in] "CTR" or "NO_CTR" +%define %%CIPHER %14 ; [in] "CTR" or "NO_CTR" %define %%CTR_CHECK %15 ; [in/out] GP with 64bit counter (to identify overflow) -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; prepare counter blocks for encryption vpshufb %%TXMM0, %%CTR, [rel byteswap_const] ;; perform 1 increment on whole 128 bits @@ -302,23 +302,23 @@ mksection .text vmovdqu %%TXMM1, [%%INP] %endif -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; AES rounds AES_ENC_ROUNDS %%KP, %%N_ROUNDS, %%TXMM0 ;; xor plaintext/ciphertext against encrypted counter blocks vpxor %%TXMM0, %%TXMM0, %%TXMM1 -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; register copy is needed as no_load/no_store options need it vmovdqa %%TXMM0, %%TXMM1 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %ifnidn %%CRC_TYPE, no_crc %ifidn %%CRC_TYPE, next_crc ;; Finish split CRC_MUL() operation vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXMM2 %endif -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; CRC calculation for ENCRYPTION/DECRYPTION ;; - always XOR against plaintext block %ifidn %%DIR, ENC @@ -326,32 +326,32 @@ mksection .text %else vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXMM0 %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; CRC calculation for NO CIPHER option vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXMM1 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %endif ;; CRC_TYPE != NO_CRC ;; store the result in the output buffer %ifnidn %%OUTP, no_store -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR vmovdqu [%%OUTP], %%TXMM0 -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR vmovdqu [%%OUTP], %%TXMM1 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %endif ;; update BIP value - always use cipher text for BIP %ifnidn %%XBIP_IN_OUT, no_bip -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR %ifidn %%DIR, ENC vpxor %%XBIP_IN_OUT, %%XBIP_IN_OUT, %%TXMM0 %else vpxor %%XBIP_IN_OUT, %%XBIP_IN_OUT, %%TXMM1 %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR vpxor %%XBIP_IN_OUT, %%XBIP_IN_OUT, %%TXMM1 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %endif ;; !NO_BIP ;; increment in/out pointers @@ -365,7 +365,7 @@ mksection .text ;;; ============================================================================ ;;; PON stitched algorithm round on a single AES block (16 bytes): -;;; AES-CTR (optional, depending on %%CIPH) +;;; AES-CTR (optional, depending on %%CIPHER) ;;; - prepares counter block ;;; - encrypts counter block ;;; - loads text @@ -399,7 +399,7 @@ mksection .text %define %%T10 %19 ; [clobbered] XMM temporary %define %%CRC_TYPE %20 ; [in] "first_crc" or "next_crc" or "no_crc" %define %%DIR %21 ; [in] "ENC" or "DEC" -%define %%CIPH %22 ; [in] "CTR" or "NO_CTR" +%define %%CIPHER %22 ; [in] "CTR" or "NO_CTR" %define %%CTR_CHECK %23 ; [in/out] GP with 64bit counter (to identify overflow) %define %%CTR1 %%T3 @@ -412,7 +412,7 @@ mksection .text %define %%TXT3 %%T9 %define %%TXT4 %%T10 -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; prepare counter blocks for encryption vmovdqa %%T0, [rel ddq_add_1] vmovdqa %%T2, [rel byteswap_const] @@ -493,7 +493,7 @@ mksection .text vmovdqu %%TXT3, [%%INP + 32] vmovdqu %%TXT4, [%%INP + 48] -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR AES_ENC_ROUNDS_4 %%KP, %%N_ROUNDS, %%CTR1, %%CTR2, %%CTR3, %%CTR4, %%T0 ;; xor plaintext/ciphertext against encrypted counter blocks @@ -501,13 +501,13 @@ mksection .text vpxor %%CTR2, %%CTR2, %%TXT2 vpxor %%CTR3, %%CTR3, %%TXT3 vpxor %%CTR4, %%CTR4, %%TXT4 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %ifidn %%CRC_TYPE, next_crc ;; Finish split CRC_MUL() operation vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%T2 %endif -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR %ifidn %%DIR, ENC ;; CRC calculation for ENCRYPTION (blocks 1 & 2) ;; - XOR CRC against plaintext block @@ -527,7 +527,7 @@ mksection .text CRC_CLMUL %%XCRC_IN_OUT, %%XCRC_MUL, %%T2 %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; CRC calculation for NO CIPHER option (blocks 1 & 2) ;; - XOR CRC against plaintext block vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXT1 @@ -536,23 +536,23 @@ mksection .text vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXT2 CRC_CLMUL %%XCRC_IN_OUT, %%XCRC_MUL, %%T2 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR ;; store ciphertext/plaintext -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR vmovdqu [%%OUTP], %%CTR1 vmovdqu [%%OUTP + 16], %%CTR2 vmovdqu [%%OUTP + 32], %%CTR3 vmovdqu [%%OUTP + 48], %%CTR4 -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR vmovdqu [%%OUTP], %%TXT1 vmovdqu [%%OUTP + 16], %%TXT2 vmovdqu [%%OUTP + 32], %%TXT3 vmovdqu [%%OUTP + 48], %%TXT4 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR ;; update BIP value -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; - always use ciphertext for BIP %ifidn %%DIR, ENC vpxor %%T0, %%CTR1, %%CTR2 @@ -561,10 +561,10 @@ mksection .text vpxor %%T0, %%TXT1, %%TXT2 vpxor %%T1, %%TXT3, %%TXT4 %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR vpxor %%T0, %%TXT1, %%TXT2 vpxor %%T1, %%TXT3, %%TXT4 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR vpxor %%XBIP_IN_OUT, %%XBIP_IN_OUT, %%T0 vpxor %%XBIP_IN_OUT, %%XBIP_IN_OUT, %%T1 @@ -572,7 +572,7 @@ mksection .text add %%INP, 64 add %%OUTP, 64 -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR %ifidn %%DIR, ENC ;; CRC calculation for ENCRYPTION (blocks 3 & 4) ;; - XOR CRC against plaintext block @@ -588,14 +588,14 @@ mksection .text CRC_CLMUL %%XCRC_IN_OUT, %%XCRC_MUL, %%T2 vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%CTR4 %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; CRC calculation for NO CIPHER option (blocks 3 & 4) ;; - XOR CRC against plaintext block vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXT3 CRC_CLMUL %%XCRC_IN_OUT, %%XCRC_MUL, %%T2 vpxor %%XCRC_IN_OUT, %%XCRC_IN_OUT, %%TXT4 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR %endmacro ; DO_PON_4 @@ -604,7 +604,7 @@ mksection .text %macro CIPHER_BIP_REST 14 %define %%NUM_BYTES %1 ; [in/clobbered] number of bytes to cipher %define %%DIR %2 ; [in] "ENC" or "DEC" -%define %%CIPH %3 ; [in] "CTR" or "NO_CTR" +%define %%CIPHER %3 ; [in] "CTR" or "NO_CTR" %define %%PTR_IN %4 ; [in/clobbered] GPR pointer to input buffer %define %%PTR_OUT %5 ; [in/clobbered] GPR pointer to output buffer %define %%PTR_KEYS %6 ; [in] GPR pointer to expanded keys @@ -623,7 +623,7 @@ mksection .text jb %%_partial_block_left DO_PON %%PTR_KEYS, NUM_AES_ROUNDS, %%XCTR_IN_OUT, %%PTR_IN, %%PTR_OUT, %%XBIP_IN_OUT, \ - no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPH, %%CTR_CHECK + no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPHER, %%CTR_CHECK sub %%NUM_BYTES, 16 jz %%_bip_done jmp %%_cipher_last_blocks @@ -635,7 +635,7 @@ mksection .text ;; XMMT2 = data in ;; XMMT1 = data out DO_PON %%PTR_KEYS, NUM_AES_ROUNDS, %%XCTR_IN_OUT, no_load, no_store, no_bip, \ - no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPH, %%CTR_CHECK + no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPHER, %%CTR_CHECK ;; bip update for partial block (mask out bytes outside the message) lea %%GPT1, [rel mask_out_top_bytes + 16] @@ -863,8 +863,8 @@ mksection .text ;;; - calls other macros and directly uses registers ;;; defined at the top of the file %macro AES128_CTR_PON 2 -%define %%DIR %1 ; [in] direction "ENC" or "DEC" -%define %%CIPH %2 ; [in] cipher "CTR" or "NO_CTR" +%define %%DIR %1 ; [in] direction "ENC" or "DEC" +%define %%CIPHER %2 ; [in] cipher "CTR" or "NO_CTR" push r12 push r13 @@ -921,7 +921,7 @@ mksection .text %%_crc_not_zero: sub bytes_to_crc, 4 ; subtract size of the CRC itself -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; - read 16 bytes of IV ;; - convert to little endian format ;; - save least significant 8 bytes in GP register for overflow check @@ -938,7 +938,7 @@ mksection .text ;; get output buffer mov p_out, [job + _dst] -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; get key pointers mov p_keys, [job + _enc_keys] %endif @@ -950,7 +950,7 @@ mksection .text vmovdqa xcrckey, [rel rk1] ; rk1 and rk2 in xcrckey ;; get number of bytes to cipher -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR mov num_bytes, [job + _msg_len_to_cipher_in_bytes] %else ;; Message length to cipher is 0 @@ -967,7 +967,7 @@ mksection .text %ifidn %%DIR, DEC ;; decrypt the buffer first mov tmp, num_bytes - CIPHER_BIP_REST tmp, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST tmp, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 ;; correct in/out pointers - go back to start of the buffers @@ -1042,7 +1042,7 @@ mksection .text DO_PON_4 p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ xcrc, xcrckey, xtmp1, xtmp2, xtmp3, xtmp4, xtmp5, xtmp6, \ xtmp7, xtmp8, xtmp9, xtmp10, xtmp11, first_crc, %%DIR, \ - %%CIPH, ctr_check + %%CIPHER, ctr_check sub num_bytes, 64 sub bytes_to_crc, 64 %ifidn %%DIR, ENC @@ -1057,7 +1057,7 @@ mksection .text DO_PON_4 p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ xcrc, xcrckey, xtmp1, xtmp2, xtmp3, xtmp4, xtmp5, xtmp6, \ xtmp7, xtmp8, xtmp9, xtmp10, xtmp11, next_crc, %%DIR, \ - %%CIPH, ctr_check + %%CIPHER, ctr_check sub num_bytes, 64 sub bytes_to_crc, 64 %ifidn %%DIR, ENC @@ -1068,7 +1068,7 @@ mksection .text %%_crc_below_64_bytes: DO_PON p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ xcrc, xcrckey, xtmp1, xtmp2, xtmp3, first_crc, %%DIR, \ - %%CIPH, ctr_check + %%CIPHER, ctr_check sub num_bytes, 16 sub bytes_to_crc, 16 @@ -1078,7 +1078,7 @@ mksection .text jb %%_exit_loop DO_PON p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ xcrc, xcrckey, xtmp1, xtmp2, xtmp3, next_crc, %%DIR, \ - %%CIPH, ctr_check + %%CIPHER, ctr_check sub num_bytes, 16 sub bytes_to_crc, 16 %ifidn %%DIR, ENC @@ -1092,7 +1092,7 @@ mksection .text ;; decrypt rest of the message including CRC and optional padding mov tmp, num_bytes - CIPHER_BIP_REST tmp, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST tmp, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 mov tmp, num_bytes ; correct in/out pointers - to point before cipher & BIP @@ -1157,7 +1157,7 @@ mksection .text or DWORD(decrypt_not_done), DWORD(decrypt_not_done) jnz %%_do_not_cipher_the_rest %endif - CIPHER_BIP_REST num_bytes, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST num_bytes, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 %%_do_not_cipher_the_rest: diff --git a/lib/include/kasumi_internal.h b/lib/include/kasumi_internal.h index a781f381..7e9a0b7f 100755 --- a/lib/include/kasumi_internal.h +++ b/lib/include/kasumi_internal.h @@ -756,7 +756,7 @@ kasumi_f8_1_buffer(const kasumi_key_sched_t *pCtx, const uint64_t IV, uint8_t *pBufferOut = (uint8_t *) pOut; uint32_t lengthInBytes = length; - /* IV Endianity */ + /* IV Endianness */ a.b64[0] = BSWAP64(IV); /* First encryption to create modifier */ @@ -857,7 +857,7 @@ kasumi_f8_1_buffer_bit(const kasumi_key_sched_t *pCtx, const uint64_t IV, SafeBuf safeOutBuf = {0}; SafeBuf safeInBuf = {0}; - /* IV Endianity */ + /* IV Endianness */ a.b64[0] = BSWAP64(IV); /* First encryption to create modifier */ @@ -1002,7 +1002,7 @@ kasumi_f8_2_buffer(const kasumi_key_sched_t *pCtx, kasumi_union_t temp; - /* IV Endianity */ + /* IV Endianness */ a1.b64[0] = BSWAP64(IV1); a2.b64[0] = BSWAP64(IV2); @@ -1185,7 +1185,7 @@ kasumi_f8_3_buffer(const kasumi_key_sched_t *pCtx, kasumi_union_t a2, b2; /* the modifier */ kasumi_union_t a3, b3; /* the modifier */ - /* IV Endianity */ + /* IV Endianness */ a1.b64[0] = BSWAP64(IV1); a2.b64[0] = BSWAP64(IV2); a3.b64[0] = BSWAP64(IV3); @@ -1304,7 +1304,7 @@ kasumi_f8_4_buffer(const kasumi_key_sched_t *pCtx, const uint64_t IV1, kasumi_union_t a4, b4; /* the modifier */ uint16_t *pTemp[4] = {b1.b16, b2.b16, b3.b16, b4.b16}; - /* IV Endianity */ + /* IV Endianness */ b1.b64[0] = BSWAP64(IV1); b2.b64[0] = BSWAP64(IV2); b3.b64[0] = BSWAP64(IV3); diff --git a/lib/sse_t1/gcm_sse.inc b/lib/sse_t1/gcm_sse.inc index 8e671328..042ef073 100644 --- a/lib/sse_t1/gcm_sse.inc +++ b/lib/sse_t1/gcm_sse.inc @@ -565,7 +565,7 @@ %ifidn %%ENC_DEC, DEC movdqa xmm3, xmm1 - pxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn) + pxor xmm9, xmm1 ; Ciphertext XOR E(K, Yn) mov r15, %%PLAIN_CYPH_LEN add r15, r13 diff --git a/lib/sse_t1/pon_by8_sse.asm b/lib/sse_t1/pon_by8_sse.asm index 6e067ba4..f0a4f0b5 100644 --- a/lib/sse_t1/pon_by8_sse.asm +++ b/lib/sse_t1/pon_by8_sse.asm @@ -207,7 +207,7 @@ mksection .text ;;; ============================================================================ ;;; PON stitched algorithm round on a single AES block (16 bytes): -;;; AES-CTR (optional, depending on %%CIPH) +;;; AES-CTR (optional, depending on %%CIPHER) ;;; - prepares counter blocks ;;; - encrypts counter blocks ;;; - loads text @@ -233,10 +233,10 @@ mksection .text %define %%TXMM2 %11 ; [clobbered] XMM temporary %define %%CRC_TYPE %12 ; [in] "first_crc" or "next_crc" or "no_crc" %define %%DIR %13 ; [in] "ENC" or "DEC" -%define %%CIPH %14 ; [in] "CTR" or "NO_CTR" +%define %%CIPHER %14 ; [in] "CTR" or "NO_CTR" %define %%CTR_CHECK %15 ; [in/out] GP with 64bit counter (to identify overflow) -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; prepare counter blocks for encryption movdqa %%TXMM0, %%CTR pshufb %%TXMM0, [rel byteswap_const] @@ -261,18 +261,18 @@ mksection .text movdqu %%TXMM1, [%%INP] %endif -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; AES rounds AES_ENC_ROUNDS %%KP, %%N_ROUNDS, %%TXMM0 ;; xor plaintext/ciphertext against encrypted counter blocks pxor %%TXMM0, %%TXMM1 -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; if no encryption needs to be done, move from input to output reg movdqa %%TXMM0, %%TXMM1 -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR %ifidn %%DIR, ENC ;; CRC calculation for ENCRYPTION %ifidn %%CRC_TYPE, first_crc @@ -298,7 +298,7 @@ mksection .text pxor %%XCRC_IN_OUT, %%TXMM0 %endif %endif ; DECRYPT -%else ;; CIPH = NO_CTR +%else ;; CIPHER = NO_CTR ;; CRC calculation for DECRYPTION %ifidn %%CRC_TYPE, first_crc ;; in the first run just XOR initial CRC with the first block @@ -311,7 +311,7 @@ mksection .text pxor %%XCRC_IN_OUT, %%TXMM1 %endif -%endif ;; CIPH = CTR +%endif ;; CIPHER = CTR ;; store the result in the output buffer %ifnidn %%OUTP, no_store @@ -343,7 +343,7 @@ mksection .text %macro CIPHER_BIP_REST 14 %define %%NUM_BYTES %1 ; [in/clobbered] number of bytes to cipher %define %%DIR %2 ; [in] "ENC" or "DEC" -%define %%CIPH %3 ; [in] "CTR" or "NO_CTR" +%define %%CIPHER %3 ; [in] "CTR" or "NO_CTR" %define %%PTR_IN %4 ; [in/clobbered] GPR pointer to input buffer %define %%PTR_OUT %5 ; [in/clobbered] GPR pointer to output buffer %define %%PTR_KEYS %6 ; [in] GPR pointer to expanded keys @@ -361,7 +361,7 @@ mksection .text jb %%_partial_block_left DO_PON %%PTR_KEYS, NUM_AES_ROUNDS, %%XCTR_IN_OUT, %%PTR_IN, %%PTR_OUT, %%XBIP_IN_OUT, \ - no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPH, %%CTR_CHECK + no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPHER, %%CTR_CHECK sub %%NUM_BYTES, 16 jz %%_bip_done jmp %%_cipher_last_blocks @@ -373,7 +373,7 @@ mksection .text ;; XMMT2 = data in ;; XMMT1 = data out DO_PON %%PTR_KEYS, NUM_AES_ROUNDS, %%XCTR_IN_OUT, no_load, no_store, no_bip, \ - no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPH, %%CTR_CHECK + no_crc, no_crc, %%XMMT1, %%XMMT2, %%XMMT3, no_crc, %%DIR, %%CIPHER, %%CTR_CHECK ;; BIP update for partial block (mask out bytes outside the message) lea %%GPT1, [rel mask_out_top_bytes + 16] @@ -608,8 +608,8 @@ mksection .text ;;; - calls other macros and directly uses registers ;;; defined at the top of the file %macro AES128_CTR_PON 2 -%define %%DIR %1 ; [in] direction "ENC" or "DEC" -%define %%CIPH %2 ; [in] cipher "CTR" or "NO_CTR" +%define %%DIR %1 ; [in] direction "ENC" or "DEC" +%define %%CIPHER %2 ; [in] cipher "CTR" or "NO_CTR" push r12 push r13 @@ -666,7 +666,7 @@ mksection .text %%_crc_not_zero: sub bytes_to_crc, 4 ; subtract size of the CRC itself -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; - read 16 bytes of IV ;; - convert to little endian format ;; - save least significant 8 bytes in GP register for overflow check @@ -683,7 +683,7 @@ mksection .text ;; get output buffer mov p_out, [job + _dst] -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR ;; get key pointers mov p_keys, [job + _enc_keys] %endif @@ -695,7 +695,7 @@ mksection .text movdqa xcrckey, [rel rk1] ; rk1 and rk2 in xcrckey ;; get number of bytes to cipher -%ifidn %%CIPH, CTR +%ifidn %%CIPHER, CTR mov num_bytes, [job + _msg_len_to_cipher_in_bytes] %else ;; Message length to cipher is 0 @@ -712,7 +712,7 @@ mksection .text %ifidn %%DIR, DEC ;; decrypt the buffer first mov tmp, num_bytes - CIPHER_BIP_REST tmp, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST tmp, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 ;; correct in/out pointers - go back to start of the buffers @@ -783,7 +783,7 @@ mksection .text %%_at_least_32_bytes: DO_PON p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ - xcrc, xcrckey, xtmp1, xtmp2, xtmp3, first_crc, %%DIR, %%CIPH, ctr_check + xcrc, xcrckey, xtmp1, xtmp2, xtmp3, first_crc, %%DIR, %%CIPHER, ctr_check sub num_bytes, 16 sub bytes_to_crc, 16 @@ -791,7 +791,7 @@ mksection .text cmp bytes_to_crc, 16 jb %%_exit_loop DO_PON p_keys, NUM_AES_ROUNDS, xcounter, p_in, p_out, xbip, \ - xcrc, xcrckey, xtmp1, xtmp2, xtmp3, next_crc, %%DIR, %%CIPH, ctr_check + xcrc, xcrckey, xtmp1, xtmp2, xtmp3, next_crc, %%DIR, %%CIPHER, ctr_check sub num_bytes, 16 sub bytes_to_crc, 16 %ifidn %%DIR, ENC @@ -805,7 +805,7 @@ mksection .text ;; decrypt rest of the message including CRC and optional padding mov tmp, num_bytes - CIPHER_BIP_REST tmp, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST tmp, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 mov tmp, num_bytes ; correct in/out pointers - to point before cipher & BIP @@ -869,7 +869,7 @@ mksection .text or DWORD(decrypt_not_done), DWORD(decrypt_not_done) jnz %%_do_not_cipher_the_rest %endif - CIPHER_BIP_REST num_bytes, %%DIR, %%CIPH, p_in, p_out, p_keys, xbip, \ + CIPHER_BIP_REST num_bytes, %%DIR, %%CIPHER, p_in, p_out, p_keys, xbip, \ xcounter, xtmp1, xtmp2, xtmp3, ctr_check, tmp2, tmp3 %%_do_not_cipher_the_rest: diff --git a/perf/win_x64.mak b/perf/win_x64.mak index 99d7847a..2f28549c 100644 --- a/perf/win_x64.mak +++ b/perf/win_x64.mak @@ -62,7 +62,7 @@ DLFLAGS = CC = cl -# _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure strtok() being used +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure strtok() being used CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) LNK = link diff --git a/test/gcm_test.c b/test/gcm_test.c index 092ae92a..b89c16c6 100644 --- a/test/gcm_test.c +++ b/test/gcm_test.c @@ -1283,7 +1283,7 @@ static const struct gcm_ctr_vector gcm_vectors[] = { extra_vector(22), }; -/* Variable IV vectrors (not 12 bytes) */ +/* Variable IV vectors (not 12 bytes) */ static const struct gcm_ctr_vector gcm_iv_vectors[] = { /* * field order {K, Klen, IV, IVlen, A, Alen, P, Plen, C, T, Tlen}; diff --git a/test/test_api.py b/test/test_api.py index 63adaaab..9d4f4851 100755 --- a/test/test_api.py +++ b/test/test_api.py @@ -72,7 +72,7 @@ fixed_end_main = """test_suite_update(&ts, run - errors, errors); def ERR(err_str): print("ERROR: {}".format(err_str)) -# This serches for patterns in header file and returns dict with results +# This searches for patterns in header file and returns dict with results # Functions works only for 3 args exactly # lines : list of all lines from LIB_HEADER # keyword : position of keyword in pattern (0/1/2) @@ -399,7 +399,7 @@ if __name__ == "__main__": function_list.append((name, f_type)) # -------------------------------------------------------------------------- - # Match arg types with arg names and args with ivalid values + # Match arg types with arg names and args with invalid values full_data = prep_func_arg_type_matches(function_list, defines, prototypes) test_cases = assign_errors_to_inv_parameters_by_arg_name(full_data) diff --git a/test/win_x64.mak b/test/win_x64.mak index 3d075b66..67959bbb 100644 --- a/test/win_x64.mak +++ b/test/win_x64.mak @@ -59,7 +59,7 @@ DLFLAGS = # compiler CC = cl -# _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure snprintf() being used +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure snprintf() being used CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) #linker diff --git a/test/wycheproof/win_x64.mak b/test/wycheproof/win_x64.mak index 85d2aac0..12d3caac 100644 --- a/test/wycheproof/win_x64.mak +++ b/test/wycheproof/win_x64.mak @@ -58,7 +58,7 @@ DLFLAGS = # compiler CC = cl -# _CRT_SECURE_NO_WARNINGS disables warning C4996 about unsecure snprintf() being used +# _CRT_SECURE_NO_WARNINGS disables warning C4996 about insecure snprintf() being used CFLAGS = /nologo /DNO_COMPAT_IMB_API_053 /D_CRT_SECURE_NO_WARNINGS $(DCFLAGS) /Y- /W3 /WX- /Gm- /fp:precise /EHsc $(EXTRA_CFLAGS) $(INCDIR) #linker -- GitLab From c556123cbba8a98eace7ba34cf77043419a32bc6 Mon Sep 17 00:00:00 2001 From: Pablo de Lara Date: Thu, 15 Sep 2022 09:46:07 +0100 Subject: [PATCH 364/369] doc: [README] add CAVP algorithm parameters section Co-authored-by: Tomasz Kantecki --- README | 83 ++++++++++++++++++++++++++++++++++++++++++++++------- README.md | 85 ++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 147 insertions(+), 21 deletions(-) diff --git a/README b/README index 8d8b51ab..b08dac9f 100644 --- a/README +++ b/README @@ -704,16 +704,79 @@ process. It implements validation of the following algorithms: - AES-CBC - AES-CTR - AES-CMAC -- SHA1 -- SHA224 -- SHA256 -- SHA384 -- SHA512 -- HMAC-SHA1 -- HMAC-SHA224 -- HMAC-SHA256 -- HMAC-SHA384 -- HMAC-SHA512 +- SHA1 (SHA-1) +- SHA224 (SHA2-224) +- SHA256 (SHA2-256) +- SHA384 (SHA2-384) +- SHA512 (SHA2-512) +- HMAC-SHA1 (HMAC-SHA-1) +- HMAC-SHA224 (HMAC-SHA2-224) +- HMAC-SHA256 (HMAC-SHA2-256) +- HMAC-SHA384 (HMAC-SHA2-384) +- HMAC-SHA512 (HMAC-SHA2-512) + +CAVP Algorithm Parameters + +NOTE: all sizes in bits + + +--------------------------------------------------------------------------------------------+ + | Algorithm | Standard | Parameters | + |---------------------+-----------+----------------------------------------------------------| + | AES-GCM | SP800-38D | Key size: 128, 192, 256 | + | | | Direction: encrypt and decrypt | + | | | ivLen: [min = 8, max = 1024, increment 8] | + | | | tagLen: 32, 64, 96, 104, 112, 120, 128 | + | | | payloadLen: [min = 0, max = 65536, increment = 8] | + | | | aadLen: [min = 0, max = 65536, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | AES-CBC | SP800-38A | Key size: 128, 192, 256 | + | | | Direction: encrypt and decrypt | + |---------------------+-----------+----------------------------------------------------------| + | AES-CTR | SP800-38A | Key size: 128, 192, 256 | + | | | Direction: encrypt and decrypt | + | | | payloadLen: [min = 8, max = 128, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | SHA1 (SHA-1) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | SHA224 (SHA2-224) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | SHA256 (SHA2-256) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | SHA384 (SHA2-384) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | SHA512 (SHA2-512) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | AES-CCM | SP800-38C | Key size: 128, 256 | + | | | Direction: encrypt and decrypt | + | | | ivLen: [min = 56, max = 104, increment 8] | + | | | tagLen: 32, 48, 64, 80, 96, 112, 128 | + | | | payloadLen: [min = 0, max = 256, increment = 8] | + | | | aadLen: [min = 0, max = 368, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | AES-GMAC | SP800-38B | Key size: 128, 192, 256 | + | | | ivLen: [min = 8, max = 1024, increment 8] | + | | | tagLen: 32, 64, 96, 104, 112, 120, 128 | + | | | aadLen: [min = 0, max = 65536, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | AES-CMAC | SP800-38B | Key size: 128, 256 | + | | | msgLen: [min = 8, max = 65528, increment 8] | + | | | macLen: [min = 8, max = 128, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | HMAC-SHA1 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | + | (HMAC-SHA-1) | | macLen: [min = 32, max = 160, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | HMAC-SHA224 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | + | (HMAC-SHA2-224) | | macLen: [min = 32, max = 224, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | HMAC-SHA256 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | + | (HMAC-SHA2-256) | | macLen: [min = 32, max = 256, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | HMAC-SHA384 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | + | (HMAC-SHA2-384) | | macLen: [min = 32, max = 384, increment = 8] | + |---------------------+-----------+----------------------------------------------------------| + | HMAC-SHA512 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | + | (HMAC-SHA2-512) | | macLen: [min = 32, max = 512, increment = 8] | + +--------------------------------------------------------------------------------------------+ Self-Test diff --git a/README.md b/README.md index a0426423..2555d681 100644 --- a/README.md +++ b/README.md @@ -642,17 +642,80 @@ ACVP test application located in `test` directory is to support CAVP process. It - AES-CBC - AES-CTR - AES-CMAC -- SHA1 -- SHA224 -- SHA256 -- SHA384 -- SHA512 -- HMAC-SHA1 -- HMAC-SHA224 -- HMAC-SHA256 -- HMAC-SHA384 -- HMAC-SHA512 - +- SHA1 (SHA-1) +- SHA224 (SHA2-224) +- SHA256 (SHA2-256) +- SHA384 (SHA2-384) +- SHA512 (SHA2-512) +- HMAC-SHA1 (HMAC-SHA-1) +- HMAC-SHA224 (HMAC-SHA2-224) +- HMAC-SHA256 (HMAC-SHA2-256) +- HMAC-SHA384 (HMAC-SHA2-384) +- HMAC-SHA512 (HMAC-SHA2-512) + +### CAVP Algorithm Parameters + +**Note:** all sizes in bits +``` ++--------------------------------------------------------------------------------------------+ +| Algorithm | Standard | Parameters | +|---------------------+-----------+----------------------------------------------------------| +| AES-GCM | SP800-38D | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +| | | ivLen: [min = 8, max = 1024, increment 8] | +| | | tagLen: 32, 64, 96, 104, 112, 120, 128 | +| | | payloadLen: [min = 0, max = 65536, increment = 8] | +| | | aadLen: [min = 0, max = 65536, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CBC | SP800-38A | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +|---------------------+-----------+----------------------------------------------------------| +| AES-CTR | SP800-38A | Key size: 128, 192, 256 | +| | | Direction: encrypt and decrypt | +| | | payloadLen: [min = 8, max = 128, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA1 (SHA-1) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA224 (SHA2-224) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA256 (SHA2-256) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA384 (SHA2-384) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| SHA512 (SHA2-512) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CCM | SP800-38C | Key size: 128, 256 | +| | | Direction: encrypt and decrypt | +| | | ivLen: [min = 56, max = 104, increment 8] | +| | | tagLen: 32, 48, 64, 80, 96, 112, 128 | +| | | payloadLen: [min = 0, max = 256, increment = 8] | +| | | aadLen: [min = 0, max = 368, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-GMAC | SP800-38B | Key size: 128, 192, 256 | +| | | ivLen: [min = 8, max = 1024, increment 8] | +| | | tagLen: 32, 64, 96, 104, 112, 120, 128 | +| | | aadLen: [min = 0, max = 65536, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| AES-CMAC | SP800-38B | Key size: 128, 256 | +| | | msgLen: [min = 8, max = 65528, increment 8] | +| | | macLen: [min = 8, max = 128, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA1 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA-1) | | macLen: [min = 32, max = 160, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA224 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-224) | | macLen: [min = 32, max = 224, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA256 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-256) | | macLen: [min = 32, max = 256, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA384 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-384) | | macLen: [min = 32, max = 384, increment = 8] | +|---------------------+-----------+----------------------------------------------------------| +| HMAC-SHA512 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | +| (HMAC-SHA2-512) | | macLen: [min = 32, max = 512, increment = 8] | ++--------------------------------------------------------------------------------------------+ +``` ### Self-Test In order to support CMVP, the library implements Self-Test functionality that is available with all compilation options. -- GitLab From c3f041b012829f8ce5ad321b6f9bb0023084e714 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 22 Sep 2022 13:50:27 +0000 Subject: [PATCH 365/369] doc: release notes update for v1.3 --- ReleaseNotes.txt | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 55e68064..379e710c 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,7 +1,7 @@ ======================================================================== Release Notes for Intel(R) Multi-Buffer Crypto for IPsec Library -Unreleased +v1.3 September 2022 ======================================================================== Library @@ -18,6 +18,8 @@ Library - Asynchronous burst API added that supports all cipher and hash modes - SNOW3G-UEA2 SSE multi-buffer implementation added - SNOW3G-UIA2 SSE multi-buffer initialization and key-stream generation added +- SNOW3G-UEA2 and SNOW3G-UIA2 SSE implementation used in JOB API for + AVX and AVX2 architectures - API documentation added (doxygen generated) - New SGL job API (AES-GCM and CHACHA20-POLY1305 only) - Enforced EVEX PMADD52 encoding in AVX512 code @@ -25,6 +27,13 @@ Library - SSE, AVX, AVX2 and AVX512 managers were split to better cover different types - Added library self-test functionality - enbranch64 not emitted on Windows builds (CET related) +- use SHANI extensions in AVX2 type-2 and AVX type-2 for SHA224, HMAC-SHA224, + SHA256 and HMAC-SHA256 +- use SHANI extensions in AVX type-2 for SHA1, HMAC-SHA1 +- no-GFNI option added to help with testing +- single buffer SHANI implementation of SHA1 and SHA256 added +- single buffer SHANI implementation used in HMAC-SHA1, HMAC-SHA224 and + HMAC-SHA256 flush operation Test Applications - GHASH JOB API support added in the test application, fuzzing and xvalid tools @@ -35,6 +44,11 @@ Test Applications - Cross validation (xvalid) tool improvements in pattern search functionality - FreeBSD added to github CI - Added AVX-SSE transition check to the cross validation tool (xvalid) +- Wycheproof AES-GCM, AES-CCM, CHACHA20-POLY1305, AES-CMAC, AES-GMAC, HMAC-SHA1, + HMAC-SHA224, HMAC-SHA256, HMAC-SHA384 and HMAC-SHA512 test vectors added + to a new test tool +- no-GFNI option added +- Fuzzing application extended to cover new burst API's Performance Application - GHASH support added (through JOB and direct API) @@ -42,12 +56,15 @@ Performance Application - Support added for SHA1/224/256/384/512 - Burst API support added for supported algorithms - SGL support added (AES-GCM and CHACHA20-POLY1305 only) +- no-GFNI option added Fixes - Fixed 23-byte IV expansion for ZUC-256 (issue #102) - Fixed incorrect 8-buffer SNOW3G key-stream generation (issue #104) - Numerous AVX-SSE transition fixes with SAFE_OPTIONS=n - [ZUC-EIA3] allow unaligned digest load/stores +- AES-CCM authentication flush may load out of scope data (issue #107) +- AES-CMAC authentication flush may load out of scope data (similar to issue #107) v1.2 February 2022 ======================================================================== -- GitLab From 006e5a1f306c8c2191451321862d953496599772 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Tue, 27 Sep 2022 14:16:24 +0000 Subject: [PATCH 366/369] perf: [diff-tool] increase column width and remove spaces in column names --- perf/ipsec_diff_tool.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/perf/ipsec_diff_tool.py b/perf/ipsec_diff_tool.py index f69823e8..d21b049f 100755 --- a/perf/ipsec_diff_tool.py +++ b/perf/ipsec_diff_tool.py @@ -35,7 +35,7 @@ import sys # Number of parameters (ARCH, CIPHER_MODE, DIR, HASH_ALG, KEY_SIZE) PAR_NUM = 5 -COL_WIDTH = 14 +COL_WIDTH = 19 CYCLE_COST = False PACKET_SIZE = 0 SLOPE = False @@ -153,16 +153,16 @@ class VarList(list): #commandline flags and prints the appropriate values if CYCLE_COST: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "CYCLE COST A", "CYCLE COST B"] + "KEYSZ", "CYCLE_COST_A", "CYCLE_COST_B"] print("Buffer size: {} bytes".format(PACKET_SIZE)) elif THROUGHPUT: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "THROUGHPUT A", "THROUGHPUT B"] + "KEYSZ", "THROUGHPUT_A", "THROUGHPUT_B"] print("Buffer size: {} bytes".format(PACKET_SIZE)) print("Clock speed: {} MHz\nThroughput unit: Mbps".format(CLOCK_SPEED)) else: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "SLOPE A", "INTERCEPT A", "SLOPE B", "INTERCEPT B"] + "KEYSZ", "SLOPE_A", "INTERCEPT_A", "SLOPE_B", "INTERCEPT_B"] print("".join(j.ljust(COL_WIDTH) for j in headings)) @@ -206,16 +206,16 @@ class VarList(list): """ if CYCLE_COST: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "CYCLE COST A"] + "KEYSZ", "CYCLE_COST_A"] print("Buffer size: {} bytes".format(PACKET_SIZE)) elif THROUGHPUT: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "THROUGHPUT A"] + "KEYSZ", "THROUGHPUT_A"] print("Buffer size: {} bytes".format(PACKET_SIZE)) print("Clock speed: {} MHz\nThroughput unit: Mbps".format(CLOCK_SPEED)) else: headings = ["NO", "ARCH", "CIPHER", "DIR", "HASH", - "KEYSZ", "SLOPE A", "INTERCEPT A"] + "KEYSZ", "SLOPE_A", "INTERCEPT_A"] print("".join(j.ljust(COL_WIDTH) for j in headings)) for i, obj in enumerate(self): number = i+1 -- GitLab From 47b4dbd4cb09ccf51908ed0f2db1e8ae45c90d49 Mon Sep 17 00:00:00 2001 From: Stephen Mcintyre Date: Tue, 4 Oct 2022 13:44:34 +0100 Subject: [PATCH 367/369] test [ACVP]: update arch selection issue --- test/acvp_app_main.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/acvp_app_main.c b/test/acvp_app_main.c index cc27c555..cd1b55a7 100644 --- a/test/acvp_app_main.c +++ b/test/acvp_app_main.c @@ -1338,15 +1338,15 @@ int main(int argc, char **argv) "Missing argument for --arch\n"); goto exit; } - if (strcmp(argv[i], "SSE") == 0) + if (strcmp(argv[i + 1], "SSE") == 0) test_arch = IMB_ARCH_SSE; - else if (strcmp(argv[i], "AVX") == 0) + else if (strcmp(argv[i + 1], "AVX") == 0) test_arch = IMB_ARCH_AVX; - else if (strcmp(argv[i], "AVX2") == 0) + else if (strcmp(argv[i + 1], "AVX2") == 0) test_arch = IMB_ARCH_AVX2; - else if (strcmp(argv[i], "AVX512") == 0) + else if (strcmp(argv[i + 1], "AVX512") == 0) test_arch = IMB_ARCH_AVX512; - else if (strcmp(argv[i], "NO-AESNI") == 0) + else if (strcmp(argv[i + 1], "NO-AESNI") == 0) test_arch = IMB_ARCH_NOAESNI; else { fprintf(stderr, "Unsupported architecture\n"); -- GitLab From 7ccf0bef69ba911b55110dfbbd62aedbfceca980 Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 29 Sep 2022 15:13:36 +0000 Subject: [PATCH 368/369] lib: version change from v1.3 to v1.4-dev --- lib/intel-ipsec-mb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/intel-ipsec-mb.h b/lib/intel-ipsec-mb.h index 0dff3c12..ac8a58eb 100644 --- a/lib/intel-ipsec-mb.h +++ b/lib/intel-ipsec-mb.h @@ -91,8 +91,8 @@ typedef struct { /** * Library version */ -#define IMB_VERSION_STR "1.3.0" -#define IMB_VERSION_NUM 0x10300 +#define IMB_VERSION_STR "1.4.0-dev" +#define IMB_VERSION_NUM 0x10400 /** * Macro to translate version number -- GitLab From dcb145c0578188eed9300ea711c406d81e0373ec Mon Sep 17 00:00:00 2001 From: Tomasz Kantecki Date: Thu, 29 Sep 2022 15:22:21 +0000 Subject: [PATCH 369/369] doc: [README] remove text version of README and keep the markdown one only --- Makefile | 5 +- README | 870 ------------------------------------------------------- 2 files changed, 1 insertion(+), 874 deletions(-) delete mode 100644 README diff --git a/Makefile b/Makefile index 08b1da71..ffd3c6fb 100644 --- a/Makefile +++ b/Makefile @@ -57,9 +57,6 @@ help: doxy: $(MAKE) -C lib doxy -README: README.md - pandoc -f markdown -t plain $< -o $@ - .PHONY: TAGS TAGS: find ./ -name "*.[ch]" -print | etags - @@ -82,7 +79,7 @@ spellcheck: $(CODESPELL) -d -L $(CS_IGNORE_WORDS) \ -S "*.obj,*.o,*.a,*.so,*.lib,*~,*.so,*.so.*,*.d,ipsec_perf" \ -S "ipsec_MB_testapp,ipsec_xvalid_test" \ - ./lib ./perf ./test README README.md SECURITY.md CONTRIBUTING \ + ./lib ./perf ./test README.md SECURITY.md CONTRIBUTING \ Makefile win_x64.mak ReleaseNotes.txt LICENSE $(CS_EXTRA_OPTS) # cppcheck analysis check diff --git a/README b/README deleted file mode 100644 index b08dac9f..00000000 --- a/README +++ /dev/null @@ -1,870 +0,0 @@ -[Coverity Status] [Linux Build Shared gcc] [Linux Build Static gcc] -[Linux Build Shared clang] [Linux Build Static clang] [Linux Build -Shared clang AESNI emulation] [FreeBSD Build Shared clang] [FreeBSD -Build Shared gcc] - - - -INTEL(R) MULTI-BUFFER CRYPTO FOR IPSEC LIBRARY - - -The library provides software crypto acceleration primarily targeting -packet processing applications. It can be used for application such as: -IPsec, TLS, Wireless (RAN), Cable or MPEG DRM. - -The library is hosted on GitHub and is used as software crypto provider -in DPDK, Intel(R) QAT Engine and FD.io. - -Using crypto interfaces from the above frameworks gives freedom to -change providers without subsequent application modifications. The -library can also be used directly through its native API. - -Key differentiating features: - -- operation chaining (encryption and authentication) -- advanced cryptographic pipelining - - job manager with scheduling and dispatching functions - - API hides underlying implementation details from an application -- multi-buffer and function stitching innovations -- low level implementations using latest instruction extensions - - - -CONTENTS - - -1. Overview -2. Processor Extensions -3. Recommendations -4. Package Content -5. Documentation -6. Compilation -7. Security Considerations & Options for Increased Security -8. Installation -9. Backwards compatibility -10. Disclaimer (ZUC, KASUMI, SNOW3G) -11. Legal Disclaimer -12. FIPS Compliance -13. DLL Injection - - - -1. OVERVIEW - - -Intel Multi-Buffer Crypto for IPsec Library is highly-optimized software -implementations of the core cryptographic processing for IPsec, which -provides industry-leading performance on a range of Intel(R) Processors. - -For information on how the library works, see the Intel White Paper: -“Fast Multi-buffer IPsec Implementations on Intel Architecture -Processors”. Jim Guilford, Sean Gulley, et. al. - -The easiest way to find it is to search the Internet for the title and -Intel White Paper. - -Table 1. List of supported cipher algorithms and their implementations. - - +----------------------------------------------------------------------+ - | | Implementation | - | Encryption +-----------------------------------------------------| - | | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(5)| - |----------------+--------+--------+--------+--------+--------+--------| - | AES128-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | AES192-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | AES256-GCM | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | AES128-CCM | N | Y by8 | Y by8 | N | N | Y by16 | - | AES256-CCM | N | Y by8 | Y by8 | N | N | Y by16 | - | AES128-CBC | N | Y(1) | Y(3) | N | N | Y(6) | - | AES192-CBC | N | Y(1) | Y(3) | N | N | Y(6) | - | AES256-CBC | N | Y(1) | Y(3) | N | N | Y(6) | - | AES128-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES192-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES256-CTR | N | Y by8 | Y by8 | N | N | Y by16 | - | AES128-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | - | AES192-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | - | AES256-ECB | N | Y(1) | Y by8 | Y(10) | N | Y by16 | - | NULL | Y | N | N | N | N | N | - | AES128-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | - | AES256-DOCSIS | N | Y(2) | Y(4) | N | Y(7) | Y(8) | - | DES-DOCSIS | Y | N | N | N | Y x16 | N | - | 3DES | Y | N | N | N | Y x16 | N | - | DES | Y | N | N | N | Y x16 | N | - | KASUMI-F8 | Y | N | N | N | N | N | - | ZUC-EEA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | - | ZUC-EEA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | - | SNOW3G-UEA2 | N | Y x4 | Y | Y | Y x16 | Y x16 | - | AES128-CBCS(9) | N | Y(1) | Y(3) | N | N | Y(6) | - | Chacha20 | N | Y | Y | Y | Y | N | - | Chacha20 AEAD | N | Y | Y | Y | Y | N | - | SNOW-V | N | Y | Y | N | N | N | - | SNOW-V AEAD | N | Y | Y | N | N | N | - | PON-CRC-BIP | N | Y by8 | Y by8 | N | N | Y | - +----------------------------------------------------------------------+ - -Notes: -(1,2) - By default, decryption is by4 and encryption is x4. -On CPU’s supporting GFNI, decryption is by8 and encryption is x8. -(3,4) - decryption is by8 and encryption is x8 -(5) - AVX512 plus VAES, VPCLMULQDQ and GFNI extensions -(6) - decryption is by16 and encryption is x16 -(7) - same as AES128-CBC for AVX, combines cipher and CRC32 -(8) - decryption is by16 and encryption is x16 -(9) - currently 1:9 crypt:skip pattern supported -(10) - by default, decryption and encryption are AVX by8. -On CPUs supporting VAES, decryption and encryption are AVX2-VAES by16. - -Legend: -byY - single buffer Y blocks at a time -xY - Y buffers at a time - -As an example of how to read table 1 and 2, if one uses AVX512 interface -to perform AES128-CBC encryption then there is no native AVX512 -implementation for this cipher. In such case, the library uses best -available implementation which is AVX for AES128-CBC. - -Table 2. List of supported integrity algorithms and their -implementations. - - +-------------------------------------------------------------------------+ - | | Implementation | - | Integrity +-----------------------------------------------------| - | | x86_64 | SSE | AVX | AVX2 | AVX512 | VAES(3)| - |-------------------+--------+--------+--------+--------+--------+--------| - | AES-XCBC-96 | N | Y x4 | Y x8 | N | N | Y x16 | - | HMAC-MD5-96 | Y(1) | Y x4x2 | Y x4x2 | Y x8x2 | N | N | - | HMAC-SHA1-96 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | HMAC-SHA2-224_112 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | HMAC-SHA2-256_128 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | HMAC-SHA2-384_192 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | HMAC-SHA2-512_256 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | SHA1 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | SHA2-224 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | SHA2-256 | N | Y(2)x4 | Y x4 | Y x8 | Y x16 | N | - | SHA2-384 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | SHA2-512 | N | Y x2 | Y x2 | Y x4 | Y x8 | N | - | AES128-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | AES192-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | AES256-GMAC | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | NULL | Y | N | N | N | N | N | - | AES128-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 | - | AES256-CCM | N | Y(5)x4 | Y x8 | N | N | Y x16 | - | AES128-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 | - | AES256-CMAC-96 | Y | Y(5)x4 | Y x8 | N | N | Y x16 | - | KASUMI-F9 | Y | N | N | N | N | N | - | ZUC-EIA3 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | - | ZUC-EIA3-256 | N | Y x4 | Y x4 | Y x8 | Y x16 | Y x16 | - | SNOW3G-UIA2(8) | N | Y by4 | Y by4 | N | Y by32 | Y by32 | - | DOCSIS-CRC32(4) | N | Y | Y | N | Y | Y | - | HEC | N | Y | Y | N | N | N | - | POLY1305 | Y | N | N | N | Y | Y | - | POLY1305 AEAD | Y | N | N | N | Y | Y | - | SNOW-V AEAD | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | GHASH | N | Y by8 | Y by8 | Y by8 | Y by8 | Y by48 | - | CRC(6) | N | Y by8 | Y by8 | N | N | Y by16 | - | PON-CRC-BIP(7) | N | Y | Y | N | N | Y | - +-------------------------------------------------------------------------+ - -Notes: -(1) - MD5 over one block implemented in C -(2) - Implementation using SHANI extensions is x2 -(3) - AVX512 plus VAES, VPCLMULQDQ, GFNI and IFMA extensions -(4) - used only with AES256-DOCSIS and AES128-DOCSIS ciphers -(5) - x8 on selected CPU’s supporting GFNI -(6) - Supported CRC types: - CRC32: Ethernet FCS, SCTP, WIMAX OFDMA -- CRC24: LTE A, LTE B -- CRC16: X25, FP data -- CRC11: FP header -- CRC10: IUUP data -- CRC8: WIMAX OFDMA HCS -- CRC7: FP header -- CRC6: IUUP header -(7) - used only with PON-AES128-CTR cipher -(8) - x4/x16 for init keystream generation, then by4/by32 - -Legend: -byY- single buffer Y blocks at a time -xY- Y buffers at a time - -Table 3. Encryption and integrity algorithm combinations - - +---------------------------------------------------------------------+ - | Encryption | Allowed Integrity Algorithms | - |---------------+-----------------------------------------------------| - | AES128-GCM | AES128-GMAC | - |---------------+-----------------------------------------------------| - | AES192-GCM | AES192-GMAC | - |---------------+-----------------------------------------------------| - | AES256-GCM | AES256-GMAC | - |---------------+-----------------------------------------------------| - | AES128-CCM | AES128-CCM | - |---------------+-----------------------------------------------------| - | AES256-CCM | AES256-CCM | - |---------------+-----------------------------------------------------| - | AES128-CBC, | AES-XCBC-96, | - | AES192-CBC, | HMAC-SHA1-96, HMAC-SHA2-224_112, HMAC-SHA2-256_128, | - | AES256-CBC, | HMAC-SHA2-384_192, HMAC-SHA2-512_256, | - | AES128-CTR, | AES128-CMAC-96, | - | AES192-CTR, | NULL, | - | AES256-CTR, | KASUMI-F9, | - | AES128-ECB, | ZUC-EIA3, ZUC-EIA3-256, | - | AES192-ECB, | SNOW3G-UIA3, | - | AES256-ECB, | POLY1305, | - | NULL, | AES128-GMAC, AES192-GMAC, AES256-GMAC, GHASH | - | AES128-DOCSIS,| | - | AES256-DOCSIS,| | - | DES-DOCSIS, | | - | 3DES, | | - | DES, | | - | Chacha20, | | - | KASUMI-F8, | | - | ZUC-EEA3, | | - | ZUC-EEA3-256, | | - | SNOW3G-UEA3 | | - | SNOW-V | | - |---------------+-----------------------------------------------------| - | AES128-DOCSIS,| DOCSIS-CRC32 | - | AES256-DOCSIS | | - |---------------+-----------------------------------------------------| - | PON-AES128-CTR| PON-CRC-BIP | - |---------------+-----------------------------------------------------| - | CHACHA20 AEAD | POLY1305 AEAD | - +---------------+-----------------------------------------------------+ - | SNOW-V AEAD | SNOW-V AEAD (GHASH) | - +---------------+-----------------------------------------------------+ - - - -2. PROCESSOR EXTENSIONS - - -Table 4. Processor extensions used in the library - - +-------------------------------------------------------------------------+ - | Algorithm | Interface | Extensions | - |-------------------+-----------+-----------------------------------------| - | HMAC-SHA1-96, | AVX512 | AVX512F, AVX512BW, AVX512VL | - | HMAC-SHA2-224_112,| | | - | HMAC-SHA2-256_128,| | | - | HMAC-SHA2-384_192,| | | - | HMAC-SHA2-512_256 | | | - |-------------------+-----------+-----------------------------------------| - | DES, 3DES, | AVX512 | AVX512F, AVX512BW | - | DOCSIS-DES | | | - |-------------------+-----------+-----------------------------------------| - | HMAC-SHA1-96, | SSE | SHANI | - | HMAC-SHA2-224_112,| | - presence is autodetected and library | - | HMAC-SHA2-256_128,| | falls back to SSE implementation | - | HMAC-SHA2-384_192,| | if not present | - | HMAC-SHA2-512_256 | | | - +-------------------+-----------+-----------------------------------------+ - - - -3. RECOMMENDATIONS - - -Legacy or to be avoided algorithms listed in the table below are -implemented in the library in order to support legacy applications. -Please use corresponding alternative algorithms instead. - - +--------------------------------------------------------------+ - | # | Algorithm | Recommendation | Alternative | - |---+---------------------+----------------+-------------------| - | 1 | DES encryption | Avoid | AES encryption | - |---+---------------------+----------------+-------------------| - | 2 | 3DES encryption | Avoid | AES encryption | - |---+---------------------+----------------+-------------------| - | 3 | HMAC-MD5 integrity | Legacy | HMAC-SHA256 | - |---+---------------------+----------------+-------------------| - | 4 | AES-ECB encryption | Avoid | AES-CBC, AES-CNTR | - |---+---------------------+----------------+-------------------| - | 3 | HMAC-SHA1 integrity | Avoid | HMAC-SHA256 | - +--------------------------------------------------------------+ - -Intel(R) Multi-Buffer Crypto for IPsec Library depends on C library and -it is recommended to use its latest version. - -Applications using the Intel(R) Multi-Buffer Crypto for IPsec Library -rely on Operating System to provide process isolation. As the result, it -is recommended to use latest Operating System patches and security -updates. - - - -4. PACKAGE CONTENT - - -- test - Library test applications -- perf - Library performance application -- lib - Library source files -- lib/sse - Intel(R) SSE optimized routines -- lib/avx - Intel(R) AVX optimized routines -- lib/avx2 - Intel(R) AVX2 optimized routines -- lib/avx512 - Intel(R) AVX512 optimized routines -- lib/no-aesni - Non-AESNI accelerated routines - -NOTE: -There is just one branch used in the project. All development is done on -the master branch. -Code taken from the tip of the master branch should not be considered -fit for production. - -Refer to the releases tab for stable code versions: -https://github.com/intel/intel-ipsec-mb/releases - - - -5. DOCUMENTATION - - -Full documentation can be found at: -https://intel.github.io/intel-ipsec-mb - -To generate documentation locally, run: -> make doxy - - - -6. COMPILATION - - -Linux (64-bit only) - -Required tools: -- GNU make -- NASM version 2.14 (or newer) -- gcc (GCC) 4.8.3 (or newer) - -Shared library: -> make - -Static library: -> make SHARED=n - -Clean the build: -> make clean -or -> make clean SHARED=n - -Build with debugging information: -> make DEBUG=y - -Build with AESNI emulation support (disabled by default): -> make AESNI_EMU=y - -NOTE: Building with debugging information is not advised for production -use. - -For more build options and their explanation run: -> make help - - -Windows MSVS (x64 only) - -Required tools: -- Microsoft (R) Visual Studio 2015: -- NMAKE: Microsoft (R) Program Maintenance Utility Version 14.00.24210.0 -- CL: Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24215.1 for -x64 -- LIB: Microsoft (R) Library Manager Version 14.00.24215.1 -- LINK: Microsoft (R) Incremental Linker Version 14.00.24215.1 -- Note: Building on later versions should work but is not verified -- NASM version 2.14 (or newer) - -Shared library (DLL): -> nmake /f win_x64.mak - -Static library: -> nmake /f win_x64.mak SHARED=n - -Clean the build: -> nmake /f win_x64.mak clean -or -> nmake /f win_x64.mak clean SHARED=n - -Build without safety features: -- SAFE_DATA clears sensitive information stored temporarily on stack, -registers or internal data structures -- SAFE_PARAM adds extra checks on input parameters -- SAFE_LOOKUP uses constant-time lookups (enabled by default) -- SAFE_OPTIONS additional option to disable all safe options. Enabled by -default. -Disable to turn off: SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP. - -> nmake /f win_x64.mak SAFE_DATA=n SAFE_PARAM=n -> nmake /f win_x64.mak SAFE_OPTIONS=n - -Build with debugging information: -> nmake /f win_x64.mak DEBUG=y - -Build with AESNI emulation support (disabled by default): -> nmake /f win_x64.mak AESNI_EMU=y - -NOTE: Building with debugging information is not advised for production -use. - -For more build options and their explanation run: -> nmake /f win_x64.mak help - - -Windows Mingw-w64 (64-bit only) - -Required tools: -- GNU mingw32-make.exe -- NASM version 2.14 (or newer) -- gcc (GCC) 10.3.0 (or newer) - -Shared library: -> mingw32-make.exe - -Static library: -> mingw32-make.exe SHARED=n - -Clean the build: -> mingw32-make.exe clean -or -> mingw32-make.exe clean SHARED=n - -Build with debugging information: -> mingw32-make.exe DEBUG=y - -NOTE: Building with debugging information is not advised for production -use. - -For more build options and their explanation run: -> mingw32-make.exe help - - -FreeBSD (64-bit only) - -Required tools: -- GNU make -- NASM version 2.14 (or newer) -- gcc (GCC) 4.8.3 (or newer) / clang 5.0 (or newer) - -Shared library: -> gmake - -Static library: -> gmake SHARED=n - -Clean the build: -> gmake clean -or -> gmake clean SHARED=n - -Build with debugging information: -> gmake DEBUG=y - -NOTE: Building with debugging information is not advised for production -use. - -For more build options and their explanation run: -> gmake help - - - -7. SECURITY CONSIDERATIONS & OPTIONS FOR INCREASED SECURITY - - -Security Considerations - -The security of a system that uses cryptography depends on the strength -of the cryptographic algorithms as well as the strength of the keys. -Cryptographic key strength is dependent on several factors, with some of -the most important factors including the length of the key, the entropy -of the key bits, and maintaining the secrecy of the key. - -The selection of an appropriate algorithm and mode of operation -critically affects the security of a system. Appropriate selection -criteria is beyond the scope of this document and should be determined -based upon usage, appropriate standards and consultation with a -cryptographic expert. This library includes some algorithms, which are -considered cryptographically weak and are included only for legacy and -interoperability reasons. See the “Recommendations” section for more -details. - -Secure creation of key material is not a part of this library. This -library assumes that cryptographic keys have been created using approved -methods with an appropriate and secure entropy source. Users of this -library are referred to NIST SP800-133 Revision 1, Recommendation for -Cryptographic Key Generation, found at -https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-133r1.pdf - -Even with the use of strong cryptographic algorithms and robustly -generated keys, software implementations of cryptographic algorithms may -be attacked at the implementation through cache-timing attacks, -buffer-over-reads, and other software vulnerabilities. Counter-measures -against these types of attacks are possible but require additional -processing cycles. Whether a particular system should provide such -counter-measures depends on the threats to that system, and cannot be -determined by a general library such as this one. In order to provide -the most flexible implementation, this library allows certain -counter-measures to be enabled or disabled at compile time. These -options are listed below as the “Options for Increased Security” and are -enabled through various build flags. - -Options for Increased Security - -There are three build options that are used to increase safety in the -code and help protect external functions from incorrect input data. The -SAFE_DATA, SAFE_PARAM and SAFE_LOOKUP options are enabled by default. -Due to the potential performance impact associated to the extra code, -these can be disabled by setting the parameter equal to “n” (e.g. make -SAFE_LOOKUP=n). - -No specific code has been added, and no specific validation or security -tests have been performed to help protect against or check for -side-channel attacks. - -SAFE_DATA - -Stack and registers containing sensitive information, such as keys or -IVs, are cleared upon completion of a function call. - -SAFE_PARAM - -Input parameters are checked, looking generally for NULL pointers or an -incorrect input length. - -SAFE_LOOKUP - -Lookups which depend on sensitive information are implemented with -constant time functions. - -Algorithms where these constant time functions are used are the -following: -- AESNI emulation -- DES: SSE, AVX and AVX2 implementations -- KASUMI: all architectures -- SNOW3G: all architectures - -If SAFE_LOOKUP is not enabled in the build (e.g. make SAFE_LOOKUP=n) -then the algorithms listed above may be susceptible to timing attacks -which could expose the cryptographic key. - -SAFE_OPTIONS - -SAFE_OPTIONS is a parameter that can be used to disable/enable all -supported safe options (i.e. SAFE_DATA, SAFE_PARAM, SAFE_LOOKUP). It is -set to y by default and all safe options are enabled. SAFE_OPTIONS=n -disables all safe options. - -Security API - -FORCE CLEARING/ZEROING OF MEMORY - - IMB_DLL_EXPORT void imb_clear_mem(void *mem, const size_t size); - -To assist in clearing sensitive application data such as keys, plaintext -etc. the library provides the imb_clear_mem() API. This API zeros -_‘size’_ bytes of memory pointed to by _‘mem’_ followed by the _sfence_ -instruction to ensure memory is cleared before the function returns. - -Galois Counter Mode (GCM) TAG Size - -The library GCM and GMAC implementation provides flexibility as to tag -size selection. As explained in NIST Special Publication 800-38D section -5.2.1.2 and Appendix C, using tag sizes shorter than 96 bits can be -insecure. Please refer to the aforementioned sections to understand the -details, trade offs and mitigations of using shorter tag sizes. - - - -8. INSTALLATION - - -Linux (64-bit only) - -First compile the library and then install: -> make -> sudo make install - -To uninstall the library run: -> sudo make uninstall - -If you want to change install location then define PREFIX: -> sudo make install PREFIX= - -If there is no need to run ldconfig at install stage please use -NOLDCONFIG=y option: -> sudo make install NOLDCONFIG=y - -If library was compiled as an archive (not a default option) then -install it using SHARED=n option: -> sudo make install SHARED=n - - -Windows (x64 only) - -First compile the library and then install from a command prompt in -administrator mode: -> nmake /f win_x64.mak -> nmake /f win_x64.mak install - -To uninstall the library run: -> nmake /f win_x64.mak uninstall - -If you want to change install location then define PREFIX (default -C:Files): -> nmake /f win_x64.mak install PREFIX= - -If library was compiled as a static library (not a default option) then -install it using SHARED=n option: -> nmake /f win_x64.mak install SHARED=n - - -FreeBSD (64-bit only) - -First compile the library and then install: -> gmake -> sudo gmake install - -To uninstall the library run: -> sudo gmake uninstall - -If you want to change install location then define PREFIX: -> sudo gmake install PREFIX= - -If there is no need to run ldconfig at install stage please use -NOLDCONFIG=y option: -> sudo gmake install NOLDCONFIG=y - -If library was compiled as an archive (not a default option) then -install it using SHARED=n option: -> sudo gmake install SHARED=n - - - -9. BACKWARDS COMPATIBILITY - - -In version 0.54, some symbols have been renamed to avoid too generic -names (such as cipher modes or hash algorithms). - -When building an application and linking it against the IPSec Multi -Buffer library, by default the old symbols (up to v0.53) are exported, -to maintain backwards compatibility. - -Applications should move to the new API as soon as possible, as the old -symbols are marked as deprecated and will be removed in a future -release. - -For applications which face symbol conflicts due to these old generic -names, they should be compiled with the flag -DNO_COMPAT_IMB_API_053, -which will not export the old symbols. - - - -10. DISCLAIMER (ZUC, KASUMI, SNOW3G) - - -Please note that cryptographic material, such as ciphering algorithms, -may be subject to national regulations. What is more, use of some -algorithms in real networks and production equipment can be subject to -agreement or licensing by the GSMA and/or the ETSI. - -For more details please see: -- GSMA https://www.gsma.com/security/security-algorithms/ -- ETSI -https://www.etsi.org/security-algorithms-and-codes/cellular-algorithm-licences - - - -11. LEGAL DISCLAIMER - - -THIS SOFTWARE IS PROVIDED BY INTEL“AS IS”. NO LICENSE, EXPRESS OR -IMPLIED, BY ESTOPPEL OR OTHERWISE, TO ANY INTELLECTUAL PROPERTY RIGHTS -ARE GRANTED THROUGH USE. EXCEPT AS PROVIDED IN INTEL’S TERMS AND -CONDITIONS OF SALE, INTEL ASSUMES NO LIABILITY WHATSOEVER AND INTEL -DISCLAIMS ANY EXPRESS OR IMPLIED WARRANTY, RELATING TO SALE AND/OR -USE OF INTEL PRODUCTS INCLUDING LIABILITY OR WARRANTIES RELATING TO -FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR INFRINGEMENT -OF ANY PATENT, COPYRIGHT OR OTHER INTELLECTUAL PROPERTY RIGHT. - - - -12. FIPS COMPLIANCE - - -The library does not fulfill technical requirements to achieve -Cryptographic Module (CMVP) certification as a standalone component. It -is fit for Cryptographic Algorithm validation and certification (CAVP) -and it can be part of CMVP as one of the components. - -CAVP - -ACVP test application located in test directory is to support CAVP -process. It implements validation of the following algorithms: -- AES-GCM -- AES-GMAC -- AES-CCM -- AES-CBC -- AES-CTR -- AES-CMAC -- SHA1 (SHA-1) -- SHA224 (SHA2-224) -- SHA256 (SHA2-256) -- SHA384 (SHA2-384) -- SHA512 (SHA2-512) -- HMAC-SHA1 (HMAC-SHA-1) -- HMAC-SHA224 (HMAC-SHA2-224) -- HMAC-SHA256 (HMAC-SHA2-256) -- HMAC-SHA384 (HMAC-SHA2-384) -- HMAC-SHA512 (HMAC-SHA2-512) - -CAVP Algorithm Parameters - -NOTE: all sizes in bits - - +--------------------------------------------------------------------------------------------+ - | Algorithm | Standard | Parameters | - |---------------------+-----------+----------------------------------------------------------| - | AES-GCM | SP800-38D | Key size: 128, 192, 256 | - | | | Direction: encrypt and decrypt | - | | | ivLen: [min = 8, max = 1024, increment 8] | - | | | tagLen: 32, 64, 96, 104, 112, 120, 128 | - | | | payloadLen: [min = 0, max = 65536, increment = 8] | - | | | aadLen: [min = 0, max = 65536, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | AES-CBC | SP800-38A | Key size: 128, 192, 256 | - | | | Direction: encrypt and decrypt | - |---------------------+-----------+----------------------------------------------------------| - | AES-CTR | SP800-38A | Key size: 128, 192, 256 | - | | | Direction: encrypt and decrypt | - | | | payloadLen: [min = 8, max = 128, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | SHA1 (SHA-1) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | SHA224 (SHA2-224) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | SHA256 (SHA2-256) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | SHA384 (SHA2-384) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | SHA512 (SHA2-512) | FIPS180-4 | messageLength: [min = 0, max = 65528, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | AES-CCM | SP800-38C | Key size: 128, 256 | - | | | Direction: encrypt and decrypt | - | | | ivLen: [min = 56, max = 104, increment 8] | - | | | tagLen: 32, 48, 64, 80, 96, 112, 128 | - | | | payloadLen: [min = 0, max = 256, increment = 8] | - | | | aadLen: [min = 0, max = 368, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | AES-GMAC | SP800-38B | Key size: 128, 192, 256 | - | | | ivLen: [min = 8, max = 1024, increment 8] | - | | | tagLen: 32, 64, 96, 104, 112, 120, 128 | - | | | aadLen: [min = 0, max = 65536, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | AES-CMAC | SP800-38B | Key size: 128, 256 | - | | | msgLen: [min = 8, max = 65528, increment 8] | - | | | macLen: [min = 8, max = 128, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | HMAC-SHA1 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | - | (HMAC-SHA-1) | | macLen: [min = 32, max = 160, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | HMAC-SHA224 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | - | (HMAC-SHA2-224) | | macLen: [min = 32, max = 224, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | HMAC-SHA256 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | - | (HMAC-SHA2-256) | | macLen: [min = 32, max = 256, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | HMAC-SHA384 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | - | (HMAC-SHA2-384) | | macLen: [min = 32, max = 384, increment = 8] | - |---------------------+-----------+----------------------------------------------------------| - | HMAC-SHA512 | FIPS198-1 | keyLen: [min = 8, max = 524288, increment = 8] | - | (HMAC-SHA2-512) | | macLen: [min = 32, max = 512, increment = 8] | - +--------------------------------------------------------------------------------------------+ - -Self-Test - -In order to support CMVP, the library implements Self-Test functionality -that is available with all compilation options. The test is always -performed as part of library initialization (power-up). There is no -conditional self-test functionality as none of such conditions occur -(i.e. pair-wise consistency test, software/firmware load test, manual -key entry test, continuous random number generator test, and bypass -test). The self-test consists of Cryptographic algorithm test (known -answer test) on following algorithms: -- AES-GCM -- AES-GMAC -- AES-CCM -- AES-CBC -- AES-CTR -- AES-CMAC -- SHA1 -- SHA224 -- SHA256 -- SHA384 -- SHA512 -- HMAC-SHA1 -- HMAC-SHA224 -- HMAC-SHA256 -- HMAC-SHA384 -- HMAC-SHA512 - -Example detection of library self-test completion & error in the -application: - - IMB_ARCH arch; - IMB_MGR *p_mgr = alloc_mb_mgr(0); - - init_mb_mgr_auto(p_mgr, &arch); /* or init_mb_mgr_sse/avx/avx2/avx512 */ - - /* - * check for self-test presence and successful - * - requires library version v1.3 or newer - */ - if (p_mgr->features & IMB_FEATURE_SELF_TEST) { - /* self-test feature present */ - if (p_mgr->features & IMB_FEATURE_SELF_TEST_PASS) { - printf("SELF-TEST: PASS\n"); - } else { - printf("SELF-TEST: FAIL\n"); - } - } else { - printf("SELF-TEST: N/A (requires library >= v1.3)\n"); - } - - /* check for initialization self-test error */ - if (imb_get_errno(p_mgr) == IMB_ERR_SELFTEST) { - /* self-test error */ - exit(EXIT_FAILURE); - } - - - -13.DLL INJECTION ATTACK - - -Problem - -The Windows OS has an insecure predefined search order and set of -defaults when trying to locate a resource. If the resource location is -not specified by the software, an attacker need only place a malicious -version in one of the locations Windows will search, and it will be -loaded instead. Although this weakness can occur with any resource, it -is especially common with DLL files. - -Solutions - -Applications using intel-ipsec-mb DLL library may need to apply one of -the solutions to prevent from DLL injection attack. - -Two solutions are available: - Using a Fully Qualified Path is the most -secure way to load a DLL -- Signature verification of the DLL - -Resources and Solution Details - -- Security remarks section of LoadLibraryEx documentation by - Microsoft: - https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-loadlibraryexa#security-remarks -- Microsoft Dynamic Link Library Security article: - https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-security -- Hijack Execution Flow: DLL Search Order Hijacking: - https://attack.mitre.org/techniques/T1574/001 -- Hijack Execution Flow: DLL Side-Loading: - https://attack.mitre.org/techniques/T1574/002 -- GitLab