diff --git a/.clang-format b/.clang-format index 194dba31c6dfcee38afee6516173d212a4936690..c1e7bf5b65ef2363a2e4d1874f7593b9fcc8e80e 100644 --- a/.clang-format +++ b/.clang-format @@ -39,7 +39,6 @@ AllowAllParametersOfDeclarationOnNextLine: false BinPackParameters: true BinPackArguments: true ReflowComments: true -SpaceAfterCStyleCast: false ColumnLimit: 100 Cpp11BracedListStyle: false MaxEmptyLinesToKeep: 1 diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml new file mode 100644 index 0000000000000000000000000000000000000000..af7be0e765e6e9d4361eaf3451e74119501f0d2c --- /dev/null +++ b/.github/workflows/style.yml @@ -0,0 +1,34 @@ +name: Style Check + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + +jobs: + # Style check with clang-format 18 + style-check: + runs-on: ubuntu-latest + + steps: + - name: Install packages + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo apt install -y nasm clang-format-18 + + - name: Checkout repo + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + repository: 'intel/intel-ipsec-mb' + + - name: Configure CMake + run: cmake -B ${{github.workspace}}/build -DCLANG_FORMAT_BIN=clang-format-18 + + - name: Run style check + run: cmake --build ${{github.workspace}}/build --target style diff --git a/cmake/clang-format.cmake b/cmake/clang-format.cmake index ebdd9215c9b73a201e94a4dd7179ebc84db22541..d7578a465b0d0c4a42ad7d542a68f01eacffe40a 100644 --- a/cmake/clang-format.cmake +++ b/cmake/clang-format.cmake @@ -32,7 +32,7 @@ find_program(CLANG_FORMAT NAMES ${CLANG_FORMAT_BIN}) # set up target if clang-format available if(CLANG_FORMAT) - set(CLANG_FORMAT_REQUIRED "13.0.1") + set(CLANG_FORMAT_REQUIRED "18.1.0") execute_process( COMMAND ${CLANG_FORMAT} --version diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e28100959d34497d4bfafcfd59d553c094eb6d21..42f65ca8bbe2c75cb7dc0c15b5430516a067cb1d 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -75,7 +75,7 @@ if(NASM_VERSION) if(NASM_VERSION_SMX_NI VERSION_GREATER ${CMAKE_MATCH_1}) message( NOTICE - "Minimum required NASM version for SM3/SM4/SHA512-NI: ${NASM_VERSION_SMX_NI}. SM3/SHA/SHA512-NI code not compiled - update NASM." + "Minimum required NASM version for SM3/SM4/SHA512-NI: ${NASM_VERSION_SMX_NI}. SM3/SM4/SHA512-NI code not compiled - update NASM." ) else() # SM3/SM4/SHA512-NI supported by NASM @@ -276,6 +276,10 @@ endif() if(AVX_IFMA) list(APPEND LIB_DEFINES AVX_IFMA) endif() +# enable SMX-NI support +if(SMX_NI) + list(APPEND LIB_DEFINES SMX_NI) +endif() ######################################## # add OS specific options diff --git a/lib/avx2_t4/sm4_ni_avx2.asm b/lib/avx2_t4/sm4_ni_avx2.asm index 7cc550bdafdc9766c957ae90d5c7bc372d63e905..ea9b26d61d611e7950cd7b5d806cd4bb5b9f1a37 100644 --- a/lib/avx2_t4/sm4_ni_avx2.asm +++ b/lib/avx2_t4/sm4_ni_avx2.asm @@ -27,6 +27,7 @@ %include "include/os.inc" %include "include/clear_regs.inc" +%include "include/aes_common.inc" %include "include/cet.inc" %include "include/error.inc" @@ -57,16 +58,83 @@ dd 0x30373E45, 0x4C535A61, 0x686F767D, 0x848B9299, dd 0xA0A7AEB5, 0xBCC3CAD1, 0xD8DFE6ED, 0xF4FB0209, dd 0x10171E25, 0x2C333A41, 0x484F565D, 0x646B7279 +align 32 in_shufb: db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c +db 0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04 +db 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c +align 32 out_shufb: db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 +db 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 +db 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 +%define APPEND(x, y) x %+ y mksection .text +; +; Shuffle up to 8 YMMs +; +%macro SHUFFLE_BLOCKS 10 +%define %%NUM_BLOCKS %1 +%define %%YDATA0 %2 +%define %%YDATA1 %3 +%define %%YDATA2 %4 +%define %%YDATA3 %5 +%define %%YDATA4 %6 +%define %%YDATA5 %7 +%define %%YDATA6 %8 +%define %%YDATA7 %9 +%define %%YSHUF %10 + +%assign %%REMAIN_BLOCK (%%NUM_BLOCKS % 2) + +%assign j 0 +%rep %%NUM_BLOCKS / 2 + vpshufb APPEND(YDATA, j), %%YSHUF +%assign j (j+1) +%endrep +%if (%%REMAIN_BLOCK == 1) + vpshufb APPEND(YDATA, j), %%YSHUF +%endif +%endmacro + +; +; Perform 8 SM4 rounds on YMMs +; +%macro SM4_ROUNDS 10 +%define %%NUM_BLOCKS %1 +%define %%YDATA0 %2 +%define %%YDATA1 %3 +%define %%YDATA2 %4 +%define %%YDATA3 %5 +%define %%YDATA4 %6 +%define %%YDATA5 %7 +%define %%YDATA6 %8 +%define %%YDATA7 %9 +%define %%YKEY %10 + +%assign %%REMAIN_BLOCK (%%NUM_BLOCKS % 2) + +%assign %%I 0 +%rep 8 ; Number of SM4 rounds + vbroadcasti128 %%YKEY, [KEY_EXP + 16*%%I] +%assign %%J 0 +%rep %%NUM_BLOCKS/2 + vsm4rnds4 APPEND(%%YDATA, %%J), APPEND(%%YDATA, %%J), %%YKEY +%assign %%J (%%J+1) +%endrep +%if (%%REMAIN_BLOCK == 1) + vsm4rnds4 APPEND(%%YDATA, %%J), APPEND(%%YDATA, %%J), %%YKEY +%endif + +%assign %%I (%%I + 1) +%endrep +%endmacro + align 32 MKGLOBAL(sm4_ecb_ni_avx2,function,internal) sm4_ecb_ni_avx2: @@ -77,29 +145,122 @@ sm4_ecb_ni_avx2: %define KEY_EXP arg4 %define IDX r10 -%define XDATA0 xmm0 +%define TMP r11 + +%define YDATA0 ymm0 +%define YDATA1 ymm1 +%define YDATA2 ymm2 +%define YDATA3 ymm3 +%define YDATA4 ymm4 +%define YDATA5 ymm5 +%define YDATA6 ymm6 +%define YDATA7 ymm7 +%define YKEY ymm15 + +%define YSHUFB_IN ymm13 +%define YSHUFB_OUT ymm14 + +%define NBLOCKS_MAIN 8*2 - xor IDX, IDX -main_loop: or SIZE, SIZE jz done - vmovdqu XDATA0, [IN + IDX] - vpshufb XDATA0, XDATA0, [rel in_shufb] + vmovdqa YSHUFB_IN, [rel in_shufb] + vmovdqa YSHUFB_OUT, [rel out_shufb] + xor IDX, IDX + mov TMP, SIZE + and TMP, 255 ; number of initial bytes (0 to 15 SM4 blocks) + jz main_loop -%assign i 0 -%rep 8 - vsm4rnds4 XDATA0, XDATA0, [KEY_EXP + 16*i] -%assign i (i + 1) -%endrep - vpshufb XDATA0, [rel out_shufb] + ; branch to different code block based on remainder + cmp TMP, 8*16 + je initial_num_blocks_is_8 + jb initial_num_blocks_is_7_1 + cmp TMP, 12*16 + je initial_num_blocks_is_12 + jb initial_num_blocks_is_11_9 + ;; 15, 14 or 13 + cmp TMP, 14*16 + ja initial_num_blocks_is_15 + je initial_num_blocks_is_14 + jmp initial_num_blocks_is_13 +initial_num_blocks_is_11_9: + ;; 11, 10 or 9 + cmp TMP, 10*16 + ja initial_num_blocks_is_11 + je initial_num_blocks_is_10 + jmp initial_num_blocks_is_9 +initial_num_blocks_is_7_1: + cmp TMP, 4*16 + je initial_num_blocks_is_4 + jb initial_num_blocks_is_3_1 + ;; 7, 6 or 5 + cmp TMP, 6*16 + ja initial_num_blocks_is_7 + je initial_num_blocks_is_6 + jmp initial_num_blocks_is_5 +initial_num_blocks_is_3_1: + ;; 3, 2 or 1 + cmp TMP, 2*16 + ja initial_num_blocks_is_3 + je initial_num_blocks_is_2 + ;; fall through for `jmp initial_num_blocks_is_1` + +%assign initial_num_blocks 1 +%rep 15 + +initial_num_blocks_is_ %+ initial_num_blocks : +%assign remaining_block (initial_num_blocks %% 2) + + ; load initial blocks + YMM_LOAD_BLOCKS_AVX2_0_16 initial_num_blocks, IN, 0, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + + ; shuffle initial blocks initial blocks + SHUFFLE_BLOCKS initial_num_blocks, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YSHUFB_IN - vmovdqu [OUT + IDX], XDATA0 + SM4_ROUNDS initial_num_blocks, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YKEY - add IDX, 16 - sub SIZE, 16 + SHUFFLE_BLOCKS initial_num_blocks, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YSHUFB_OUT + + ; store initial blocks + YMM_STORE_BLOCKS_AVX2_0_16 initial_num_blocks, OUT, 0, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + + add IDX, initial_num_blocks*16 + cmp IDX, SIZE + je done + +%assign initial_num_blocks (initial_num_blocks + 1) jmp main_loop +%endrep + +align 32 +main_loop: + YMM_LOAD_BLOCKS_AVX2_0_16 NBLOCKS_MAIN, IN, IDX, YDATA0,\ + YDATA1, YDATA2, YDATA3, YDATA4, YDATA5,\ + YDATA6, YDATA7 + + SHUFFLE_BLOCKS NBLOCKS_MAIN, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YSHUFB_IN + + SM4_ROUNDS NBLOCKS_MAIN, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YKEY + + SHUFFLE_BLOCKS NBLOCKS_MAIN, YDATA0, YDATA1, YDATA2, YDATA3, \ + YDATA4, YDATA5, YDATA6, YDATA7, YSHUFB_OUT + + ; store initial blocks + YMM_STORE_BLOCKS_AVX2_0_16 NBLOCKS_MAIN, OUT, IDX, YDATA0, YDATA1,\ + YDATA2, YDATA3, YDATA4, YDATA5, YDATA6, YDATA7 + add IDX, 16*NBLOCKS_MAIN + cmp IDX, SIZE + jne main_loop done: %ifdef SAFE_DATA diff --git a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm index 3fdd2444a8bcad00aa290b1b1937883cadceb4de..1d307482e8e70f02347a439a1fe278b004aea159 100644 --- a/lib/avx512_t2/crc32_by16_vclmul_avx512.asm +++ b/lib/avx512_t2/crc32_by16_vclmul_avx512.asm @@ -441,9 +441,9 @@ pshufb_shf_table: align 16 pshufb_shift_table: ;; use these values to shift data for the pshufb instruction - db 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - db 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF - db 0xFF, 0xFF + db 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C + db 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + db 0xFF, 0xFF, 0xFF align 16 mask1: dq 0x8080808080808080, 0x8080808080808080 diff --git a/lib/include/kasumi_internal.h b/lib/include/kasumi_internal.h index 4e76bc114a2559874088d8c090b0cb8304e8a115..d88551ef573b8505fb98c694787daa1fe68ca103 100644 --- a/lib/include/kasumi_internal.h +++ b/lib/include/kasumi_internal.h @@ -166,7 +166,7 @@ typedef union SafeBuffer { (data) = datal ^ datah; \ (data) ^= (key2); \ datal = LOOKUP16_SSE(sso_kasumi_S7e, (data) >> 9, 256); \ - datah = LOOKUP16_SSE(sso_kasumi_S9e, (data) &0x1FF, 512); \ + datah = LOOKUP16_SSE(sso_kasumi_S9e, (data) & 0x1FF, 512); \ (data) = datal ^ datah; \ (data) ^= (key3); \ } while (0) @@ -1410,8 +1410,8 @@ kasumi_f8_n_buffer(const kasumi_key_sched_t *pKeySchedule, const uint64_t IV[], dataLen[inner_idx] = tempLen; } } /* for inner packet idx (inner bubble-sort) */ - } /* for outer packet idx (outer bubble-sort) */ - } /* if sortNeeded */ + } /* for outer packet idx (outer bubble-sort) */ + } /* if sortNeeded */ packet_idx = dataCount; while (packet_idx--) diff --git a/lib/include/mb_mgr_burst.h b/lib/include/mb_mgr_burst.h index 694f44b9a2a2e664445cce07aacfe0204d05298a..da761536fa5bfeb13de689b05a8fd2ae5bc2220b 100644 --- a/lib/include/mb_mgr_burst.h +++ b/lib/include/mb_mgr_burst.h @@ -257,6 +257,122 @@ submit_aes_ctr_burst(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, return n_jobs; } + +__forceinline uint32_t +submit_aes_ecb_enc_burst(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, const int run_check) +{ + if (run_check) { + uint32_t i; + + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, IMB_CIPHER_ECB, IMB_AUTH_NULL, + IMB_DIR_ENCRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == IMB_KEY_128_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_ENC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else if (key_size == IMB_KEY_192_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_ENC_192(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else /* assume 256-bit key */ { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_ENC_256(job->src + job->cipher_start_src_offset_in_bytes, + job->enc_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } + + return n_jobs; +} + +__forceinline uint32_t +submit_aes_ecb_dec_burst(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, + const IMB_KEY_SIZE_BYTES key_size, const int run_check) +{ + if (run_check) { + uint32_t i; + + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, IMB_CIPHER_ECB, IMB_AUTH_NULL, + IMB_DIR_DECRYPT, key_size)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (key_size == IMB_KEY_128_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_DEC_128(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else if (key_size == IMB_KEY_192_BYTES) { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_DEC_192(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } else /* assume 256-bit key */ { + uint32_t i; + + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + AES_ECB_DEC_256(job->src + job->cipher_start_src_offset_in_bytes, + job->dec_keys, job->dst, + job->msg_len_to_cipher_in_bytes & (~15)); + job->status = IMB_STATUS_COMPLETED; + } + } + + return n_jobs; +} #endif /* __aarch64__ */ __forceinline uint32_t @@ -282,6 +398,12 @@ submit_cipher_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jo return submit_aes_cbc_burst_dec(state, jobs, n_jobs, key_size, run_check); case IMB_CIPHER_CNTR: return submit_aes_ctr_burst(state, jobs, n_jobs, key_size, run_check); + + case IMB_CIPHER_ECB: + if (dir == IMB_DIR_ENCRYPT) + return submit_aes_ecb_enc_burst(state, jobs, n_jobs, key_size, run_check); + else + return submit_aes_ecb_dec_burst(state, jobs, n_jobs, key_size, run_check); #endif /* __aarch64__ */ default: break; @@ -434,6 +556,131 @@ submit_burst_hmac_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, co return completed_jobs; } + +__forceinline uint32_t +submit_burst_sha_x(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs, const int run_check, + const IMB_HASH_ALG hash_alg) +{ + uint32_t i, completed_jobs = 0; + + if (run_check) { + /* validate jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + /* validate job */ + if (is_job_invalid(state, job, IMB_CIPHER_NULL, hash_alg, IMB_DIR_ENCRYPT, + job->key_len_in_bytes)) { + job->status = IMB_STATUS_INVALID_ARGS; + return 0; + } + } + } + + if (hash_alg == IMB_AUTH_SHA_1) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA1(state->sha_1_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA1(state->sha_1_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_224) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA224(state->sha_224_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA224(state->sha_224_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_256) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA256(state->sha_256_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA256(state->sha_256_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_384) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA384(state->sha_384_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA384(state->sha_384_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } else if (hash_alg == IMB_AUTH_SHA_512) { + /* submit all jobs */ + for (i = 0; i < n_jobs; i++) { + IMB_JOB *job = &jobs[i]; + + job = SUBMIT_JOB_SHA512(state->sha_512_ooo, job); + if (job != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + /* flush any outstanding jobs */ + if (completed_jobs != n_jobs) { + IMB_JOB *job = NULL; + + while ((job = FLUSH_JOB_SHA512(state->sha_512_ooo, job)) != NULL) { + job->status = IMB_STATUS_COMPLETED; + completed_jobs++; + } + } + } + + return completed_jobs; +} #endif /* __aarch64__ */ __forceinline uint32_t @@ -466,6 +713,16 @@ submit_hash_burst_and_check(IMB_MGR *state, IMB_JOB *jobs, const uint32_t n_jobs case IMB_AUTH_HMAC_SHA_512: return submit_burst_hmac_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_HMAC_SHA_512); + case IMB_AUTH_SHA_1: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_1); + case IMB_AUTH_SHA_224: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_224); + case IMB_AUTH_SHA_256: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_256); + case IMB_AUTH_SHA_384: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_384); + case IMB_AUTH_SHA_512: + return submit_burst_sha_x(state, jobs, n_jobs, run_check, IMB_AUTH_SHA_512); #endif /* __aarch64__ */ default: break; diff --git a/lib/include/mb_mgr_burst_async.h b/lib/include/mb_mgr_burst_async.h index 7aaae16b51d016011e88e4bd7efd2202464f9383..be535a37680b06ed36156a28ca2bfffdd00bb3eb 100644 --- a/lib/include/mb_mgr_burst_async.h +++ b/lib/include/mb_mgr_burst_async.h @@ -150,12 +150,11 @@ submit_burst_and_check(IMB_MGR *state, const uint32_t n_jobs, IMB_JOB **jobs, co } /* validate job->suite_id */ - void *t[4]; + uint32_t t[2]; set_cipher_suite_id(jobs[i], t); - if (jobs[i]->suite_id[0] != t[0] || jobs[i]->suite_id[1] != t[1] || - jobs[i]->suite_id[2] != t[2] || jobs[i]->suite_id[3] != t[3]) { + if (jobs[i]->suite_id[0] != t[0] || jobs[i]->suite_id[1] != t[1]) { imb_set_errno(state, IMB_ERR_BURST_SUITE_ID); goto return_invalid_job; } diff --git a/lib/include/mb_mgr_job_api.h b/lib/include/mb_mgr_job_api.h index bacb4e99be42061ecb5eab7ee51d5fc7adeb27fe..a776f96f2cb93639b90a0c35d5b094a4275d333a 100644 --- a/lib/include/mb_mgr_job_api.h +++ b/lib/include/mb_mgr_job_api.h @@ -4067,21 +4067,46 @@ FLUSH_JOB(IMB_MGR *state) /* ========================================================================= */ __forceinline void -set_cipher_suite_id(IMB_JOB *job, void **id) +set_cipher_suite_id(IMB_JOB *job, uint32_t id[2]) { const unsigned c_idx = calc_cipher_tab_index(job); const unsigned h_idx = (unsigned) job->hash_alg; - id[0] = (void *) tab_submit_cipher[c_idx]; - id[1] = (void *) tab_submit_hash[h_idx]; - id[2] = (void *) tab_flush_cipher[c_idx]; - id[3] = (void *) tab_flush_hash[h_idx]; + id[0] = c_idx; + id[1] = h_idx; } -#define CALL_SUBMIT_CIPHER(s, j) ((submit_flush_fn_t) (j)->suite_id[0])(s, j) -#define CALL_FLUSH_CIPHER(s, j) ((submit_flush_fn_t) (j)->suite_id[2])(s, j) -#define CALL_SUBMIT_HASH(s, j) ((submit_flush_fn_t) (j)->suite_id[1])(s, j) -#define CALL_FLUSH_HASH(s, j) ((submit_flush_fn_t) (j)->suite_id[3])(s, j) +__forceinline IMB_JOB * +CALL_SUBMIT_CIPHER(IMB_MGR *state, IMB_JOB *job) +{ + const unsigned c_idx = job->suite_id[0]; + + return tab_submit_cipher[c_idx](state, job); +} + +__forceinline IMB_JOB * +CALL_FLUSH_CIPHER(IMB_MGR *state, IMB_JOB *job) +{ + const unsigned c_idx = job->suite_id[0]; + + return tab_flush_cipher[c_idx](state, job); +} + +__forceinline IMB_JOB * +CALL_SUBMIT_HASH(IMB_MGR *state, IMB_JOB *job) +{ + const unsigned h_idx = job->suite_id[1]; + + return tab_submit_hash[h_idx](state, job); +} + +__forceinline IMB_JOB * +CALL_FLUSH_HASH(IMB_MGR *state, IMB_JOB *job) +{ + const unsigned h_idx = job->suite_id[1]; + + return tab_flush_hash[h_idx](state, job); +} IMB_DLL_EXPORT void SET_SUITE_ID_FN(IMB_MGR *state, IMB_JOB *job) diff --git a/lib/include/snow3g_common.h b/lib/include/snow3g_common.h index 8dbfd17e0aa712ad7c4ed70dd607edd724893aa0..8c2e7b2726364ae90f9879d7ad36b368190965c5 100644 --- a/lib/include/snow3g_common.h +++ b/lib/include/snow3g_common.h @@ -3168,8 +3168,8 @@ SNOW3G_F8_N_BUFFER(const snow3g_key_schedule_t *pCtx, const void *const IV[], lensBuf[inner_index] = tempLen; } } /* for inner packet index (inner bubble-sort) */ - } /* for outer packet index (outer bubble-sort) */ - } /* if sortNeeded */ + } /* for outer packet index (outer bubble-sort) */ + } /* if sortNeeded */ packet_index = 0; /* process 8 buffers at-a-time */ @@ -3334,8 +3334,8 @@ SNOW3G_F8_N_BUFFER_MULTIKEY(const snow3g_key_schedule_t *const pCtx[], const voi pCtxBuf[inner_index] = tempCtx; } } /* for inner packet index (inner bubble-sort) */ - } /* for outer packet index (outer bubble-sort) */ - } /* if sortNeeded */ + } /* for outer packet index (outer bubble-sort) */ + } /* if sortNeeded */ packet_index = 0; /* process 8 buffers at-a-time */ diff --git a/lib/ipsec-mb.h b/lib/ipsec-mb.h index 94c6c96f64e552703214a6009948e92b21e1dfa0..37fc407ed46626923a7d1b402879c25e83678b5b 100644 --- a/lib/ipsec-mb.h +++ b/lib/ipsec-mb.h @@ -48,7 +48,7 @@ typedef struct { #if defined __linux__ || defined __FreeBSD__ /**< Linux/FreeBSD */ #define DECLARE_ALIGNED(decl, alignval) decl __attribute__((aligned(alignval))) -#define __forceinline static inline __attribute__((always_inline)) +#define __forceinline static inline __attribute__((always_inline)) #if __GNUC__ >= 4 #define IMB_DLL_EXPORT __attribute__((visibility("default"))) @@ -70,7 +70,7 @@ typedef struct { #else /* MSVS */ #define DECLARE_ALIGNED(decl, alignval) __declspec(align(alignval)) decl -#define __forceinline static __forceinline +#define __forceinline static __forceinline #endif /* __MINGW__ */ @@ -430,7 +430,7 @@ typedef struct IMB_JOB { /**< Length of message to hash (in bytes) */ uint64_t msg_len_to_hash_in_bits; /**< Length of message to hash (in bits) */ - }; /**< Length of message to hash */ + }; /**< Length of message to hash */ const uint8_t *iv; /**< Initialization Vector (IV) */ uint64_t iv_len_in_bytes; /**< IV length in bytes */ uint8_t *auth_tag_output; /**< Authentication tag output */ @@ -552,8 +552,8 @@ typedef struct IMB_JOB { } CBCS; /**< CBCS specific fields */ } cipher_fields; /**< Cipher algorithm-specific fields */ - void *suite_id[4]; /**< see imb_set_session() */ - uint32_t session_id; /**< see imb_set_session() */ + uint32_t suite_id[2]; /**< see imb_set_session() */ + uint32_t session_id; /**< see imb_set_session() */ } IMB_JOB; /* KASUMI */ @@ -650,7 +650,7 @@ struct chacha20_poly1305_context_data { #ifdef __WIN32 __declspec(align(64)) #endif /* WIN32 */ - struct gcm_key_data { +struct gcm_key_data { uint8_t expanded_keys[IMB_GCM_ENC_KEY_LEN * IMB_GCM_KEY_SETS]; union { /**< Storage for precomputed hash keys */ diff --git a/lib/sse_t1/sm3_base_update_sse.asm b/lib/sse_t1/sm3_base_update_sse.asm index 9a32991a011c6b511d772b3f5e5529ec35e3d989..22118c2bd17debbd26812cc7e6c6ff625778e500 100644 --- a/lib/sse_t1/sm3_base_update_sse.asm +++ b/lib/sse_t1/sm3_base_update_sse.asm @@ -421,6 +421,11 @@ sm3_base_compress_16_63: movdqu [rsp + _W + 4*4], xmm0 movdqu [rsp + _W + 8*4], xmm0 movdqu [rsp + _W + 12*4], xmm0 + + movdqu [rsp + _W + 16*4], xmm0 + movdqu [rsp + _W + 20*4], xmm0 + movdqu [rsp + _W + 24*4], xmm0 + movdqu [rsp + _W + 28*4], xmm0 %endif FUNC_END sm3_base_update_end: diff --git a/lib/sse_t2/sha1_ni_x2_sse.asm b/lib/sse_t2/sha1_ni_x2_sse.asm index f69ddee4f3b9249c37b3333dbee54f9fb4a84f67..bedc77dcdaa4d8e724b39ec62fb93ace179feb4e 100644 --- a/lib/sse_t2/sha1_ni_x2_sse.asm +++ b/lib/sse_t2/sha1_ni_x2_sse.asm @@ -151,19 +151,6 @@ mksection .text mov rdi, [rsp + GP_OFFSET] mov rsi, [rsp + GP_OFFSET + 8] -%ifdef SAFE_DATA - pxor xmm5, xmm5 - movdqa [rsp + 0*16], xmm5 - movdqa [rsp + 1*16], xmm5 - movdqa [rsp + 2*16], xmm5 - movdqa [rsp + 3*16], xmm5 - movdqa [rsp + 4*16], xmm5 - movdqa [rsp + 5*16], xmm5 - movdqa [rsp + 6*16], xmm5 - movdqa [rsp + 7*16], xmm5 - movdqa [rsp + 8*16], xmm5 - movdqa [rsp + 9*16], xmm5 -%endif mov rbx, [rsp + GP_OFFSET + 2*8] mov rbp, [rsp + GP_OFFSET + 3*8] %endif diff --git a/lib/sse_t2/sha256_ni_one_block_sse.asm b/lib/sse_t2/sha256_ni_one_block_sse.asm index 9d1001604dc3b54ad0ff0a7238b0baeddc3cf541..7c5e7ae83dcd4bbaf87d582a6028674080f1ba86 100644 --- a/lib/sse_t2/sha256_ni_one_block_sse.asm +++ b/lib/sse_t2/sha256_ni_one_block_sse.asm @@ -32,7 +32,7 @@ struc frame .ABEF_SAVE reso 1 .CDGH_SAVE reso 1 -.XMM_SAVE reso 3 +.XMM_SAVE reso 4 .align resq 1 endstruc @@ -82,9 +82,10 @@ sha256_ni_block_sse: sub rsp, frame_size %ifndef LINUX - movdqa [rsp + frame.XMM_SAVE], xmm6 - movdqa [rsp + frame.XMM_SAVE + 16], xmm14 - movdqa [rsp + frame.XMM_SAVE + 16*2], xmm15 + movdqa [rsp + frame.XMM_SAVE + 0*16], xmm6 + movdqa [rsp + frame.XMM_SAVE + 1*16], xmm7 + movdqa [rsp + frame.XMM_SAVE + 2*16], xmm14 + movdqa [rsp + frame.XMM_SAVE + 3*16], xmm15 %endif ;; load initial digest @@ -313,9 +314,10 @@ sha256_ni_block_sse: %endif %ifndef LINUX - movdqa xmm6, [rsp + frame.XMM_SAVE] - movdqa xmm14, [rsp + frame.XMM_SAVE + 16] - movdqa xmm15, [rsp + frame.XMM_SAVE + 16*2] + movdqa xmm6, [rsp + frame.XMM_SAVE + 0*16] + movdqa xmm7, [rsp + frame.XMM_SAVE + 1*16] + movdqa xmm14, [rsp + frame.XMM_SAVE + 2*16] + movdqa xmm15, [rsp + frame.XMM_SAVE + 3*16] %endif add rsp, frame_size ret diff --git a/lib/sse_t2/sha256_ni_x1_sse.asm b/lib/sse_t2/sha256_ni_x1_sse.asm index 46309a1568692f202227fc1f57510bffdcaa46bb..f7a1e355584019a730b0898460f1670e5a5beece 100644 --- a/lib/sse_t2/sha256_ni_x1_sse.asm +++ b/lib/sse_t2/sha256_ni_x1_sse.asm @@ -146,21 +146,6 @@ mksection .text movdqa xmm13, [rsp + 10*16] movdqa xmm14, [rsp + 11*16] movdqa xmm15, [rsp + 12*16] - -%ifdef SAFE_DATA - pxor xmm5, xmm5 - movdqa [rsp + 3*16], xmm5 - movdqa [rsp + 4*16], xmm5 - movdqa [rsp + 5*16], xmm5 - movdqa [rsp + 6*16], xmm5 - movdqa [rsp + 7*16], xmm5 - movdqa [rsp + 8*16], xmm5 - movdqa [rsp + 9*16], xmm5 - movdqa [rsp + 10*16], xmm5 - movdqa [rsp + 11*16], xmm5 - movdqa [rsp + 12*16], xmm5 - -%endif %endif ; LINUX mov rsp, [rsp + 5*8] ;; rsp pointer %endmacro diff --git a/lib/sse_t2/sha256_ni_x2_sse.asm b/lib/sse_t2/sha256_ni_x2_sse.asm index 0c5cdacd4ac259c594ab286496cbd71b0a8ca886..ae60f4af75615319aa4680cd541f485f9e7a7c69 100644 --- a/lib/sse_t2/sha256_ni_x2_sse.asm +++ b/lib/sse_t2/sha256_ni_x2_sse.asm @@ -159,20 +159,6 @@ mksection .text movdqa xmm13, [rsp + 10*16] movdqa xmm14, [rsp + 11*16] movdqa xmm15, [rsp + 12*16] - -%ifdef SAFE_DATA - pxor xmm5, xmm5 - movdqa xmm5, [rsp + 3*16] - movdqa xmm5, [rsp + 4*16] - movdqa xmm5, [rsp + 5*16] - movdqa xmm5, [rsp + 6*16] - movdqa xmm5, [rsp + 7*16] - movdqa xmm5, [rsp + 8*16] - movdqa xmm5, [rsp + 9*16] - movdqa xmm5, [rsp + 10*16] - movdqa xmm5, [rsp + 11*16] - movdqa xmm5, [rsp + 12*16] -%endif %endif ; LINUX mov rsp, [rsp + 5*8] ;; rsp pointer %endmacro diff --git a/lib/x86_64/alloc.c b/lib/x86_64/alloc.c index 9f7644b7b40e1518b06561d38466bfac798a563c..78991f7ccdf2f52ada3564aa9db5d90f135482bc 100644 --- a/lib/x86_64/alloc.c +++ b/lib/x86_64/alloc.c @@ -45,11 +45,8 @@ #define ALIGN(x, y) ((x + (y - 1)) & (~(y - 1))) #define OOO_INFO(imb_mgr_ooo_ptr_name__, ooo_mgr_type__) \ - { \ - offsetof(IMB_MGR, imb_mgr_ooo_ptr_name__), \ - ALIGN(sizeof(ooo_mgr_type__), ALIGNMENT), \ - offsetof(ooo_mgr_type__, road_block) \ - } + { offsetof(IMB_MGR, imb_mgr_ooo_ptr_name__), ALIGN(sizeof(ooo_mgr_type__), ALIGNMENT), \ + offsetof(ooo_mgr_type__, road_block) } const struct { size_t ooo_ptr_offset; diff --git a/lib/x86_64/self_test.c b/lib/x86_64/self_test.c index 7bd667820a0f3c7a06b04c90ebf591b92924027a..58d83ee79a5c9480fdd96a42ac866b356f9a7441 100644 --- a/lib/x86_64/self_test.c +++ b/lib/x86_64/self_test.c @@ -325,10 +325,7 @@ static const uint8_t tdes_ede_cbc_cipher_text[] = { }; #define ADD_CIPHER_VECTOR(_cmode, _key, _iv, _plain, _cipher, _descr) \ - { \ - _cmode, _key, sizeof(_key), _iv, sizeof(_iv), _plain, sizeof(_plain), _cipher, \ - _descr \ - } + { _cmode, _key, sizeof(_key), _iv, sizeof(_iv), _plain, sizeof(_plain), _cipher, _descr } static const struct self_test_cipher_vector cipher_vectors[] = { ADD_CIPHER_VECTOR(IMB_CIPHER_CBC, aes_cbc_128_key, aes_cbc_128_iv, aes_cbc_128_plain_text, @@ -575,9 +572,7 @@ const uint8_t sha512_digest[] = { 0x20, 0x4a, 0x8f, 0xc6, 0xdd, 0xa8, 0x2f, 0x0a 0x03, 0x54, 0xec, 0x63, 0x12, 0x38, 0xca, 0x34, 0x45 }; #define ADD_SHA_VECTOR(_hmode, _msg, _digest, _descr) \ - { \ - _hmode, NULL, 0, _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, _descr \ - } + { _hmode, NULL, 0, _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, _descr } /* * Test vector from https://csrc.nist.gov/csrc/media/publications/fips/198/ @@ -685,8 +680,8 @@ static const uint8_t hmac_sha512_digest[] = { 0xfc, 0x25, 0xe2, 0x40, 0x65, 0x8c #define ADD_HMAC_SHA_VECTOR(_hmode, _key, _msg, _digest, _descr) \ { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, \ - _descr \ + _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), \ + NULL, 0, _descr \ } /* @@ -712,8 +707,8 @@ static const uint8_t aes_cmac_256_tag[] = { 0x15, 0x67, 0x27, 0xDC, 0x08, 0x78, #define ADD_CMAC_VECTOR(_hmode, _key, _msg, _digest, _descr) \ { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), NULL, 0, \ - _descr \ + _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _digest, sizeof(_digest), \ + NULL, 0, _descr \ } /* @@ -768,10 +763,8 @@ static const uint8_t aes_gmac_256_tag[] = { 0x77, 0x46, 0x0D, 0x6F, 0xB1, 0x87, 0x46, 0xAD, 0xCD, 0xFB, 0xB7, 0xF9, 0x13, 0xA1 }; #define ADD_GMAC_VECTOR(_hmode, _key, _iv, _msg, _tag, _descr) \ - { \ - _hmode, _key, sizeof(_key), _msg, sizeof(_msg), _tag, sizeof(_tag), _iv, \ - sizeof(_iv), _descr \ - } + { _hmode, _key, sizeof(_key), _msg, sizeof(_msg), \ + _tag, sizeof(_tag), _iv, sizeof(_iv), _descr } static const struct self_test_hash_vector hash_vectors[] = { ADD_SHA_VECTOR(IMB_AUTH_SHA_1, sha_message, sha1_digest, "SHA1"), @@ -1095,10 +1088,9 @@ static const uint8_t aes_gcm_256_tag[] = { 0x76, 0xfc, 0x6e, 0xce, 0x0f, 0x4e, 0 0xcd, 0xdf, 0x88, 0x53, 0xbb, 0x2d, 0x55, 0x1b }; #define ADD_GCM_VECTOR(_key, _iv, _aad, _plain, _cipher, _tag, _descr) \ - { \ - IMB_AUTH_AES_GMAC, IMB_CIPHER_GCM, _key, sizeof(_key), _iv, sizeof(_iv), _aad, \ - sizeof(_aad), _plain, sizeof(_plain), _cipher, _tag, sizeof(_tag), _descr \ - } + { IMB_AUTH_AES_GMAC, IMB_CIPHER_GCM, _key, sizeof(_key), _iv, \ + sizeof(_iv), _aad, sizeof(_aad), _plain, sizeof(_plain), \ + _cipher, _tag, sizeof(_tag), _descr } static const struct self_test_aead_gcm_vector aead_gcm_vectors[] = { ADD_GCM_VECTOR(aes_gcm_128_key, aes_gcm_128_iv, aes_gcm_128_aad, aes_gcm_128_plain_text, @@ -1357,11 +1349,9 @@ static const uint8_t aes_ccm_256_cipher_text[] = { 0x21, 0x61, 0x63, 0xDE, 0xCF, static const uint8_t aes_ccm_256_tag[] = { 0xCA, 0x8A, 0xFA, 0xA2, 0x3F, 0x22, 0x3E, 0x64 }; #define ADD_CCM_VECTOR(_key, _nonce, _aad, _plain, _cipher, _tag, _descr) \ - { \ - IMB_AUTH_AES_CCM, IMB_CIPHER_CCM, _key, sizeof(_key), _nonce, sizeof(_nonce), \ - _aad, sizeof(_aad), _plain, sizeof(_plain), _cipher, _tag, sizeof(_tag), \ - _descr \ - } + { IMB_AUTH_AES_CCM, IMB_CIPHER_CCM, _key, sizeof(_key), _nonce, \ + sizeof(_nonce), _aad, sizeof(_aad), _plain, sizeof(_plain), \ + _cipher, _tag, sizeof(_tag), _descr } static const struct self_test_aead_ccm_vector aead_ccm_vectors[] = { ADD_CCM_VECTOR(aes_ccm_128_key, aes_ccm_128_nonce, aes_ccm_128_aad, aes_ccm_128_plain_text, diff --git a/perf/ipsec_perf.c b/perf/ipsec_perf.c index 61a64cd3b73aa419aafb88ceadd7bb41a7a8a2ba..71bd057215c98316db270194e9ed283728f75463 100644 --- a/perf/ipsec_perf.c +++ b/perf/ipsec_perf.c @@ -51,8 +51,8 @@ #define strdup _strdup #undef __forceinline #define __forceinline static __forceinline -#define __func__ __FUNCTION__ -#define strcasecmp _stricmp +#define __func__ __FUNCTION__ +#define strcasecmp _stricmp #else #include #ifndef __aarch64__ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5e3ba2e9eab52a129281e8fb61b6ba543999cfdc..74a39466014257f02b6422585213f32c699d121b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,6 +39,11 @@ if(NOT "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") add_subdirectory(wycheproof-app) endif() +# build imb-mp application +if(NOT "${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64") + add_subdirectory(mp-app) +endif() + if(NOT WINDOWS) # build imb-acvp if libacvp available if(NOT ACVP_LOC) diff --git a/test/fuzz-app/job_api_fuzz_test.c b/test/fuzz-app/job_api_fuzz_test.c index 141b536837e79ccdfb5bf11fbf8e339955ea2f50..054f3a6820a032807f8d39ca399b55f97eabb97c 100644 --- a/test/fuzz-app/job_api_fuzz_test.c +++ b/test/fuzz-app/job_api_fuzz_test.c @@ -34,9 +34,8 @@ #include #include -#define BUFF_SIZE (32 * 1024 * 1024) -#define MAX_BURST_JOBS 32 -#define MAX_SGL_SEGS 32 +#define BUFF_SIZE (32 * 1024 * 1024) +#define MAX_SGL_SEGS 32 int LLVMFuzzerTestOneInput(const uint8_t *, size_t); @@ -492,7 +491,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) if (dataSize < sizeof(IMB_JOB)) return 0; - if (num_jobs > MAX_BURST_JOBS || num_jobs == 0 || key_len == 0) + if (num_jobs > IMB_MAX_BURST_SIZE || num_jobs == 0 || key_len == 0) return 0; if (cipher_dir != NULL) { @@ -579,7 +578,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_JOB(p_mgr); } } else if (burst) { - IMB_JOB *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = { NULL }; while (IMB_GET_NEXT_BURST(p_mgr, num_jobs, jobs) < (uint32_t) num_jobs) IMB_FLUSH_BURST(p_mgr, num_jobs, jobs); @@ -614,7 +613,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_BURST(p_mgr, num_jobs, jobs); } else if (cipher_burst) { - IMB_JOB jobs[MAX_BURST_JOBS] = { 0 }; + IMB_JOB jobs[IMB_MAX_BURST_SIZE] = { 0 }; for (i = 0; i < num_jobs; i++) { job = &jobs[i]; @@ -641,7 +640,7 @@ LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize) IMB_SUBMIT_CIPHER_BURST(p_mgr, jobs, num_jobs, cipher, dir, key_len); } else if (hash_burst) { - IMB_JOB jobs[MAX_BURST_JOBS] = { 0 }; + IMB_JOB jobs[IMB_MAX_BURST_SIZE] = { 0 }; for (i = 0; i < num_jobs; i++) { job = &jobs[i]; diff --git a/test/kat-app/aes_cbc_test.c b/test/kat-app/aes_cbc_test.c index b4a892f7dd9cd219a170984fce32f9349bab826b..9e87ea8d24a6cd8c721318cdbed2e4cea50212f0 100644 --- a/test/kat-app/aes_cbc_test.c +++ b/test/kat-app/aes_cbc_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int cbc_test(struct IMB_MGR *mb_mgr); @@ -185,7 +183,7 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_keys, cons const int dir, const int order, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -291,7 +289,7 @@ test_aes_many_cipher_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_key const int dir, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, jobs[MAX_BURST_JOBS]; + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE]; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -545,7 +543,7 @@ test_cbc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, int cbc_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx128; diff --git a/test/kat-app/aes_cfb_test.c b/test/kat-app/aes_cfb_test.c index 765c000e44c11db2712d19f5142bf6d105648bb4..53b860c84bd5363f43100ec95f6497a982aacdf0 100644 --- a/test/kat-app/aes_cfb_test.c +++ b/test/kat-app/aes_cfb_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int cfb_test(struct IMB_MGR *mb_mgr); diff --git a/test/kat-app/aes_test.c b/test/kat-app/aes_test.c index 69476ef765771d5ec2091711d5b084b9d1c19b68..de1fa0a5d587733b2e64591b494dca8428d98f16 100644 --- a/test/kat-app/aes_test.c +++ b/test/kat-app/aes_test.c @@ -35,8 +35,6 @@ #include "utils.h" -#define MAX_BURST_JOBS 64 - int aes_test(struct IMB_MGR *mb_mgr); @@ -838,11 +836,17 @@ static const uint8_t DOCRC17_CT[] = { #define DOCRC17_FRAME_LEN DIM(DOCRC17_PT) #define MK_DOCRC_VEC(_n) \ - { \ - _n##_FRAME_LEN, _n##_KEY, _n##_KEY_LEN, _n##_IV, _n##_PT, _n##_CT, \ - _n##_HASH_OFFSET, _n##_HASH_LENGTH, _n##_CIPHER_OFFSET, \ - _n##_CIPHER_LENGTH, _n##_CRC \ - } + { _n##_FRAME_LEN, \ + _n##_KEY, \ + _n##_KEY_LEN, \ + _n##_IV, \ + _n##_PT, \ + _n##_CT, \ + _n##_HASH_OFFSET, \ + _n##_HASH_LENGTH, \ + _n##_CIPHER_OFFSET, \ + _n##_CIPHER_LENGTH, \ + _n##_CRC } struct docsis_crc_vector { uint64_t frame_len; @@ -1023,7 +1027,7 @@ test_aes_many_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_keys, cons const int dir, const int order, const IMB_CIPHER_MODE cipher, const int in_place, const int key_len, const int num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **targets = malloc(num_jobs * sizeof(void *)); int i, completed_jobs, jobs_rx = 0, ret = -1; @@ -1454,7 +1458,7 @@ test_docrc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx128, int aes_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx128; diff --git a/test/kat-app/api_test.c b/test/kat-app/api_test.c index 058790b60ba0435c49ff566885f0dad61e721727..6d1c69a0662085323d6f83f91ab7b4544450a5cc 100644 --- a/test/kat-app/api_test.c +++ b/test/kat-app/api_test.c @@ -38,8 +38,6 @@ #define __func__ __FUNCTION__ #endif -#define MAX_BURST_JOBS 32 - int api_test(struct IMB_MGR *mb_mgr); @@ -658,8 +656,8 @@ static int is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const int test_num, int expected_errnum) { - IMB_JOB *jobs[MAX_BURST_JOBS] = { NULL }; - uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + IMB_JOB *jobs[IMB_MAX_BURST_SIZE] = { NULL }; + uint32_t i, completed_jobs, n_jobs = IMB_MAX_BURST_SIZE; int err; while (IMB_GET_NEXT_BURST(mb_mgr, n_jobs, jobs) < n_jobs) @@ -707,8 +705,8 @@ is_submit_burst_invalid(struct IMB_MGR *mb_mgr, const struct IMB_JOB *job, const static int test_burst_api(struct IMB_MGR *mb_mgr) { - struct IMB_JOB *job = NULL, *jobs[MAX_BURST_JOBS] = { NULL }; - uint32_t i, completed_jobs, n_jobs = MAX_BURST_JOBS; + struct IMB_JOB *job = NULL, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; + uint32_t i, completed_jobs, n_jobs = IMB_MAX_BURST_SIZE; struct IMB_JOB **null_jobs = NULL; int err; diff --git a/test/kat-app/ecb_test.c b/test/kat-app/ecb_test.c index 59e9712c09eaac92e56ef656874bf55792ba554d..d13ac9bc3c894d60423e89fe3addb394cb84d0f3 100644 --- a/test/kat-app/ecb_test.c +++ b/test/kat-app/ecb_test.c @@ -148,6 +148,86 @@ end: return ret; } +static int +test_ecb_burst(struct IMB_MGR *mb_mgr, void *enc_keys, void *dec_keys, const uint8_t *in_text, + const uint8_t *out_text, unsigned text_len, int dir, IMB_CIPHER_MODE cipher, + const int in_place, const int key_len, const int num_jobs) +{ + IMB_JOB burst_jobs[IMB_MAX_BURST_SIZE]; + uint8_t padding[16]; + uint32_t n_jobs = num_jobs; + uint8_t **targets = malloc(num_jobs * sizeof(void *)); + int i, jobs_rx = 0, job_idx = 0, ret = -1; + const uint32_t burst_sz = (n_jobs > IMB_MAX_BURST_SIZE) ? IMB_MAX_BURST_SIZE : n_jobs; + + if (targets == NULL) + return ret; + + memset(targets, 0, num_jobs * sizeof(void *)); + memset(padding, -1, sizeof(padding)); + + for (i = 0; i < num_jobs; i++) { + targets[i] = malloc(text_len + (sizeof(padding) * 2)); + if (targets[i] == NULL) + goto end; + memset(targets[i], -1, text_len + (sizeof(padding) * 2)); + if (in_place) { + /* copy input text to the allocated buffer */ + memcpy(targets[i] + sizeof(padding), in_text, text_len); + } + } + + while (n_jobs) { + const int n = (n_jobs > burst_sz) ? burst_sz : n_jobs; + + for (i = 0; i < n; i++) { + IMB_JOB *job = &burst_jobs[i]; + + if (!in_place) + job->src = in_text; + else + job->src = targets[job_idx] + sizeof(padding); + job->dst = targets[job_idx] + sizeof(padding); + job->enc_keys = enc_keys; + job->dec_keys = dec_keys; + + job->iv_len_in_bytes = 0; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = text_len; + job->user_data = targets[job_idx]; + job->user_data2 = (void *) ((uint64_t) job_idx); + + job_idx++; + } + + jobs_rx = IMB_SUBMIT_CIPHER_BURST(mb_mgr, burst_jobs, n, cipher, dir, key_len); + + if (jobs_rx != n) { + printf("Expected %d jobs, received %d\n", n, jobs_rx); + goto end; + } + + for (i = 0; i < jobs_rx; i++) { + IMB_JOB *job = &burst_jobs[i]; + + if (!ecb_job_ok(job, out_text, job->user_data, padding, sizeof(padding), + text_len)) + goto end; + } + n_jobs -= n; + } + + ret = 0; + +end: + for (i = 0; i < num_jobs; i++) + if (targets[i] != NULL) + free(targets[i]); + if (targets != NULL) + free(targets); + return ret; +} + static void test_ecb_vectors(struct IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher, const int num_jobs, struct test_suite_context *ts128, struct test_suite_context *ts192, @@ -225,6 +305,45 @@ test_ecb_vectors(struct IMB_MGR *mb_mgr, const IMB_CIPHER_MODE cipher, const int } else { test_suite_update(ctx, 1, 0); } + + // test burst API + if (test_ecb_burst(mb_mgr, enc_keys, dec_keys, (const void *) v->msg, + (const void *) v->ct, (unsigned) v->msgSize / 8, IMB_DIR_ENCRYPT, + cipher, 0, (unsigned) v->keySize / 8, num_jobs)) { + printf("error #%zu burst encrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ecb_burst(mb_mgr, enc_keys, dec_keys, (const void *) v->ct, + (const void *) v->msg, (unsigned) v->msgSize / 8, + IMB_DIR_DECRYPT, cipher, 0, (unsigned) v->keySize / 8, + num_jobs)) { + printf("error #%zu burst decrypt\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ecb_burst(mb_mgr, enc_keys, dec_keys, (const void *) v->msg, + (const void *) v->ct, (unsigned) v->msgSize / 8, IMB_DIR_ENCRYPT, + cipher, 1, (unsigned) v->keySize / 8, num_jobs)) { + printf("error #%zu burst encrypt in-place\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } + + if (test_ecb_burst(mb_mgr, enc_keys, dec_keys, (const void *) v->ct, + (const void *) v->msg, (unsigned) v->msgSize / 8, + IMB_DIR_DECRYPT, cipher, 1, (unsigned) v->keySize / 8, + num_jobs)) { + printf("error #%zu burst decrypt in-place\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } } if (!quiet_mode) printf("\n"); @@ -234,7 +353,7 @@ int ecb_test(struct IMB_MGR *mb_mgr) { struct test_suite_context ts128, ts192, ts256; - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17 }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, 32, 64, 128, 256, 284 }; unsigned i; int errors = 0; diff --git a/test/kat-app/gcm_ctr_vectors_test.h b/test/kat-app/gcm_ctr_vectors_test.h index 5c4558781e6cc0e63841c2bdb062d32cfcaff6c7..2c6735dfe45f4215db5ff27f886cd09811798f8e 100644 --- a/test/kat-app/gcm_ctr_vectors_test.h +++ b/test/kat-app/gcm_ctr_vectors_test.h @@ -51,21 +51,14 @@ struct gcm_ctr_vector { }; #define vector(N) \ - { \ - K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, P##N, sizeof(P##N), \ - C##N, T##N, sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, \ + P##N, sizeof(P##N), C##N, T##N, sizeof(T##N) } #define extra_vector(N) \ - { \ - K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, P##N, P##N##_len, \ - C##N, T##N, sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), IV##N, sizeof(IV##N), A##N, A##N##_len, \ + P##N, P##N##_len, C##N, T##N, sizeof(T##N) } #define ghash_vector(N) \ - { \ - K##N, (KBITS(K##N)), NULL, 0, NULL, 0, P##N, sizeof(P##N), NULL, T##N, \ - sizeof(T##N) \ - } + { K##N, (KBITS(K##N)), NULL, 0, NULL, 0, P##N, sizeof(P##N), NULL, T##N, sizeof(T##N) } struct MB_MGR; extern int diff --git a/test/kat-app/hmac_sha1_test.c b/test/kat-app/hmac_sha1_test.c index f9ded9782cb0acb3a5a1ca7fe31a6b9b278844e4..4fea42cbb701873e5ac6ab53c9e013b429b75c39 100644 --- a/test/kat-app/hmac_sha1_test.c +++ b/test/kat-app/hmac_sha1_test.c @@ -38,8 +38,6 @@ int hmac_sha1_test(struct IMB_MGR *mb_mgr); -#define MAX_BURST_JOBS 32 - extern const struct mac_test hmac_sha1_test_kat_json[]; static int hmac_sha1_job_ok(const struct mac_test *vec, const struct IMB_JOB *job, const uint8_t *auth, @@ -178,7 +176,7 @@ end2: static int test_hmac_sha1_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -288,7 +286,7 @@ static int test_hmac_sha1_hash_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, jobs[MAX_BURST_JOBS] = { 0 }; + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE] = { 0 }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -442,7 +440,7 @@ hmac_sha1_test(struct IMB_MGR *mb_mgr) uint32_t num_jobs; test_suite_start(&ts, "HMAC-SHA1"); - for (num_jobs = 1; num_jobs <= MAX_BURST_JOBS; num_jobs++) + for (num_jobs = 1; num_jobs <= IMB_MAX_BURST_SIZE; num_jobs++) test_hmac_sha1_std_vectors(mb_mgr, num_jobs, &ts); errors = test_suite_end(&ts); diff --git a/test/kat-app/hmac_sha256_sha512_test.c b/test/kat-app/hmac_sha256_sha512_test.c index 91fcf2e51a77c4a4c234ffa94c45e19a23f321fe..79f051d817931ee7f41822670a21a1849438e84a 100644 --- a/test/kat-app/hmac_sha256_sha512_test.c +++ b/test/kat-app/hmac_sha256_sha512_test.c @@ -198,14 +198,6 @@ test_hmac_shax(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_ job = IMB_SUBMIT_JOB(mb_mgr); if (job) { jobs_rx++; - /* - * SHANI HMAC-SHA implementation can return a completed - * job after 2nd submission - */ - if (num_jobs < 2) { - printf("%d Unexpected return from submit_job\n", __LINE__); - goto end; - } if (!hmac_shax_job_ok(vec, job, sha_type, job->user_data, padding, sizeof(padding))) goto end; diff --git a/test/kat-app/hmac_sm3_test.c b/test/kat-app/hmac_sm3_test.c index 459c4392f126da9e2cc2ffe5f43ef4f91cde41ea..d341f9327b74d7258f4044f1843038a9e5cd44ff 100644 --- a/test/kat-app/hmac_sm3_test.c +++ b/test/kat-app/hmac_sm3_test.c @@ -38,8 +38,6 @@ int hmac_sm3_test(struct IMB_MGR *mb_mgr); -#define MAX_BURST_JOBS 32 - extern const struct mac_test hmac_sm3_test_kat_json[]; static int hmac_sm3_job_ok(const struct mac_test *vec, const struct IMB_JOB *job, const uint8_t *auth, @@ -169,7 +167,7 @@ end: static int test_hmac_sm3_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const uint32_t num_jobs) { - struct IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + struct IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; uint8_t padding[16]; uint8_t **auths = malloc(num_jobs * sizeof(void *)); uint32_t i = 0, jobs_rx = 0; @@ -320,7 +318,7 @@ hmac_sm3_test(struct IMB_MGR *mb_mgr) uint32_t num_jobs; test_suite_start(&ts, "SM3"); - for (num_jobs = 1; num_jobs <= MAX_BURST_JOBS; num_jobs++) + for (num_jobs = 1; num_jobs <= IMB_MAX_BURST_SIZE; num_jobs++) test_hmac_sm3_std_vectors(mb_mgr, num_jobs, &ts); errors = test_suite_end(&ts); diff --git a/test/kat-app/pon_test.c b/test/kat-app/pon_test.c index dbe1b7ab3a4ee4bb08ae62092f80df0e0ea69d44..1d34f720bb0b139f2ee787d1d83efdc9b7895989 100644 --- a/test/kat-app/pon_test.c +++ b/test/kat-app/pon_test.c @@ -337,16 +337,12 @@ static const uint8_t OUT13_PON[] = { #define LENCIPH13_PON (LENBIP13_PON - OFFSET13_PON) #define ponvector(tname) \ - { \ - KEY##tname, IV##tname, IN##tname, OUT##tname, BIPOUT##tname, LENBIP##tname, \ - LENCIPH##tname, OFFSET##tname \ - } + { KEY##tname, IV##tname, IN##tname, OUT##tname, \ + BIPOUT##tname, LENBIP##tname, LENCIPH##tname, OFFSET##tname } #define pon_no_ctr_vector(tname) \ - { \ - NULL, NULL, IN##tname, OUT##tname, BIPOUT##tname, LENBIP##tname, LENCIPH##tname, \ - OFFSET##tname \ - } + { NULL, NULL, IN##tname, OUT##tname, \ + BIPOUT##tname, LENBIP##tname, LENCIPH##tname, OFFSET##tname } static const struct pon_test_vector { const uint8_t *key; diff --git a/test/kat-app/sha_test.c b/test/kat-app/sha_test.c index 5a63655c986a81134c798aa5dd808d8ed598559c..cb9ca3a6aaa4deae75e70debc16b71931d91f057 100644 --- a/test/kat-app/sha_test.c +++ b/test/kat-app/sha_test.c @@ -172,6 +172,118 @@ end2: return ret; } +static int +test_sha_hash_burst(struct IMB_MGR *mb_mgr, const struct mac_test *vec, const int num_jobs, + const int sha_type) +{ + struct IMB_JOB *job, jobs[IMB_MAX_BURST_SIZE] = { 0 }; + uint8_t padding[16]; + uint8_t **auths = malloc(num_jobs * sizeof(void *)); + int i = 0, jobs_rx = 0, ret = -1; + int completed_jobs = 0; + IMB_HASH_ALG hash_alg; + + if (auths == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end2; + } + + memset(padding, -1, sizeof(padding)); + memset(auths, 0, num_jobs * sizeof(void *)); + + switch (sha_type) { + case 1: + hash_alg = IMB_AUTH_SHA_1; + break; + case 224: + hash_alg = IMB_AUTH_SHA_224; + break; + case 256: + hash_alg = IMB_AUTH_SHA_256; + break; + case 384: + hash_alg = IMB_AUTH_SHA_384; + break; + case 512: + default: + hash_alg = IMB_AUTH_SHA_512; + break; + } + + for (i = 0; i < num_jobs; i++) { + const size_t alloc_len = vec->tagSize / 8 + (sizeof(padding) * 2); + + auths[i] = malloc(alloc_len); + if (auths[i] == NULL) { + fprintf(stderr, "Can't allocate buffer memory\n"); + goto end; + } + memset(auths[i], -1, alloc_len); + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + job->enc_keys = NULL; + job->dec_keys = NULL; + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_HASH_CIPHER; + job->auth_tag_output = auths[i] + sizeof(padding); + job->auth_tag_output_len_in_bytes = vec->tagSize / 8; + job->src = (const void *) vec->msg; + job->msg_len_to_hash_in_bytes = vec->msgSize / 8; + job->cipher_mode = IMB_CIPHER_NULL; + job->hash_alg = hash_alg; + + job->user_data = auths[i]; + } + + completed_jobs = IMB_SUBMIT_HASH_BURST(mb_mgr, jobs, num_jobs, hash_alg); + if (completed_jobs != num_jobs) { + int err = imb_get_errno(mb_mgr); + + if (err != 0) { + printf("submit_burst error %d : '%s'\n", err, imb_get_strerror(err)); + goto end; + } else { + printf("submit_burst error: not enough " + "jobs returned!\n"); + goto end; + } + } + + for (i = 0; i < num_jobs; i++) { + job = &jobs[i]; + + if (job->status != IMB_STATUS_COMPLETED) { + printf("job %u status not complete!\n", i + 1); + goto end; + } + + if (!sha_job_ok(vec, job, job->user_data, padding, sizeof(padding))) + goto end; + jobs_rx++; + } + + if (jobs_rx != num_jobs) { + printf("Expected %u jobs, received %u\n", num_jobs, jobs_rx); + goto end; + } + ret = 0; + +end: + for (i = 0; i < num_jobs; i++) { + if (auths[i] != NULL) + free(auths[i]); + } + +end2: + if (auths != NULL) + free(auths); + + return ret; +} + static void test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, struct test_suite_context *sha224_ctx, struct test_suite_context *sha256_ctx, @@ -226,6 +338,12 @@ test_sha_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *sha1_ctx, } else { test_suite_update(ctx, 1, 0); } + if (test_sha_hash_burst(mb_mgr, v, num_jobs, sha_type)) { + printf("error #%zu\n", v->tcId); + test_suite_update(ctx, 0, 1); + } else { + test_suite_update(ctx, 1, 0); + } } } diff --git a/test/kat-app/sm4_cbc_test.c b/test/kat-app/sm4_cbc_test.c index 81d8172e10e1990b2cf19afff49486646e3d62bb..f7820516f3719821d6fe646f5f1329d06c1e0cba 100644 --- a/test/kat-app/sm4_cbc_test.c +++ b/test/kat-app/sm4_cbc_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int sm4_cbc_test(struct IMB_MGR *mb_mgr); @@ -252,7 +250,7 @@ test_sm4_cbc_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, int sm4_cbc_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx; diff --git a/test/kat-app/sm4_ecb_test.c b/test/kat-app/sm4_ecb_test.c index ebbda744e0c00b2a0a06f74043071ea104e1e00f..99c1da0fed60337d7056fa7c8ae595638dbd8134 100644 --- a/test/kat-app/sm4_ecb_test.c +++ b/test/kat-app/sm4_ecb_test.c @@ -36,8 +36,6 @@ #include "utils.h" #include "cipher_test.h" -#define MAX_BURST_JOBS 64 - int sm4_ecb_test(struct IMB_MGR *mb_mgr); @@ -250,7 +248,7 @@ test_sm4_ecb_vectors(struct IMB_MGR *mb_mgr, struct test_suite_context *ctx, int sm4_ecb_test(struct IMB_MGR *mb_mgr) { - const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, MAX_BURST_JOBS }; + const int num_jobs_tab[] = { 1, 3, 4, 5, 7, 8, 9, 15, 16, 17, IMB_MAX_BURST_SIZE }; unsigned i; int errors = 0; struct test_suite_context ctx; diff --git a/test/kat-app/zuc_eea3_test.c b/test/kat-app/zuc_eea3_test.c index 844e9f81d321c5ebc6dbf43ea7860e4fc182f045..75e7aa5ef433d6462c9b492d79c80d707a72ada3 100644 --- a/test/kat-app/zuc_eea3_test.c +++ b/test/kat-app/zuc_eea3_test.c @@ -53,8 +53,6 @@ #define MAX_BUFFER_LENGTH_IN_BITS 5670 /* biggest test is EIA test 5 */ #define MAX_BUFFER_LENGTH_IN_BYTES ((MAX_BUFFER_LENGTH_IN_BITS) + 7) / 8 -#define MAX_BURST_JOBS 32 - enum api_type { TEST_DIRECT_API, TEST_SINGLE_JOB_API, TEST_BURST_JOB_API }; int @@ -343,7 +341,7 @@ submit_burst_eea3_jobs(struct IMB_MGR *mb_mgr, uint8_t **const keys, uint8_t **c const unsigned int num_jobs, const unsigned int key_len, const unsigned int *iv_lens) { - IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; unsigned int i; unsigned int jobs_rx = 0; uint32_t completed_jobs = 0; diff --git a/test/kat-app/zuc_eia3_test.c b/test/kat-app/zuc_eia3_test.c index b0774290bfe8a0fd7278a973245004eb250c8a1e..e9e4439634cdffcdc75d7fd4a2c0a2dcbb26bbdd 100644 --- a/test/kat-app/zuc_eia3_test.c +++ b/test/kat-app/zuc_eia3_test.c @@ -53,8 +53,6 @@ #define MAX_BUFFER_LENGTH_IN_BITS 5670 /* biggest test is EIA test 5 */ #define MAX_BUFFER_LENGTH_IN_BYTES ((MAX_BUFFER_LENGTH_IN_BITS) + 7) / 8 -#define MAX_BURST_JOBS 32 - enum api_type { TEST_DIRECT_API, TEST_SINGLE_JOB_API, TEST_BURST_JOB_API }; int @@ -326,7 +324,7 @@ submit_burst_eia3_jobs(struct IMB_MGR *mb_mgr, uint8_t **const keys, uint8_t **c const unsigned int num_jobs, const unsigned int key_sz, const size_t *tag_lens, const size_t *iv_lens) { - IMB_JOB *job, *jobs[MAX_BURST_JOBS] = { NULL }; + IMB_JOB *job, *jobs[IMB_MAX_BURST_SIZE] = { NULL }; unsigned int i; unsigned int jobs_rx = 0; uint32_t completed_jobs = 0; diff --git a/test/mp-app/CMakeLists.txt b/test/mp-app/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..839abc7185a737af9e964ded24d58be6292e98c8 --- /dev/null +++ b/test/mp-app/CMakeLists.txt @@ -0,0 +1,101 @@ +# Copyright (c) 2024, Intel Corporation +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of Intel Corporation nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Ensure building entire project +if(NOT FULL_PROJECT_BUILD) + message(FATAL_ERROR "Please run CMake from project root directory") +endif() + +######################################## +# set app and library names +######################################## +set(MP_APP_PRI imb-mp-primary) +set(MP_APP_SEC imb-mp-secondary) +set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../lib) +set(TEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../include) + +######################################## +# set imb-mp-primary source files +######################################## +set(MP_APP_PRI_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/imb-mp-primary.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_shared_mem.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_imb.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_alloc.c +) + +######################################## +# set imb-mp-secondary source files +######################################## +set(MP_APP_SEC_SRC_FILES_C + ${CMAKE_CURRENT_SOURCE_DIR}/imb-mp-secondary.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_shared_mem.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_imb.c + ${CMAKE_CURRENT_SOURCE_DIR}/mp_alloc.c +) + +######################################## +# set C compiler and NASM options +######################################## +# add OS specific options +if(WINDOWS) + if(CMAKE_GENERATOR MATCHES "MinGW Makefiles") + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows-mingw.cmake) + else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/windows.cmake) + endif() +else() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/unix.cmake) +endif() + +######################################## +# add targets +######################################## +include_directories(${LIB_DIR} ${TEST_INCLUDE_DIR}) +link_directories(${LIB_DIR}) + +add_executable(${MP_APP_PRI} ${MP_APP_PRI_SRC_FILES_C}) +target_link_libraries(${MP_APP_PRI} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${MP_APP_PRI} PRIVATE ${APP_DEFINES}) + +add_executable(${MP_APP_SEC} ${MP_APP_SEC_SRC_FILES_C}) +target_link_libraries(${MP_APP_SEC} PRIVATE ${IPSEC_MB_LIB}) +target_compile_definitions(${MP_APP_SEC} PRIVATE ${APP_DEFINES}) + + +######################################## +# add tests +######################################## + +# set working directory for tests +if(IMB_BIN_DIR) + set(TEST_APP_BIN_DIR "${IMB_BIN_DIR}") +else() + set(TEST_APP_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +add_test(NAME MULTIPROCESS + COMMAND ${MP_APP_PRI} ${TEST_APP_BIN_DIR}/${MP_APP_SEC} + WORKING_DIRECTORY ${TEST_APP_BIN_DIR}) diff --git a/test/mp-app/README.md b/test/mp-app/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5cc8f3b8edea933908d7de0ab2bb155555a201ac --- /dev/null +++ b/test/mp-app/README.md @@ -0,0 +1,61 @@ +# Multi-Process Test + +This test is designed to exercise fail-over scenario. In this scenario one process, called primary, is actively processing crypto requests. +Another process, called secondary, is passive but it can take over crypto operations at any point of time if the primary process fails. + +## Contents +1. Overview +2. Running +3. Multi-Buffer Manager (MB MGR) Notes +4. OS Implementation Notes + +## 1. Overview + +The test uses two applications, one is used to represent the primary process and another one the secondary one. +Operation flow of two processes looks as described in the table below. + +| PRIMARY PROCESS | SECONDARY PROCESS | +| :-------- | -----------: | +| [START] | | +| INITIALIZE INFO SHARED MEMORY | | +| INITIALIZE DATA SHARED MEMORY | | +| INITIALIZE MB MGR | | +| INITIALIZE MEMORY FOR 15 CRYPTO OPERATIONS | | +| PRODUCE EXPECTED CRYPTO RESULTS (out-of-place) | | +| SEND 15 CRYPTO REQUESTS (in-place) AND COLLECT RESPONSES IF ANY | | +| UPDATE INFO SHARED MEMORY WITH CRYPTO REQUEST/RESPONSE DETAILS | | +| START SECONDARY PROCESS --> | --> [START] | +| | OPEN INFO SHARED MEMORY | +| | OPEN DATA SHARED MEMORY | +| | INITIALIZE MB MGR IN SHARED MEMORY | +| | COMPLETE OUTSTANDING CRYPTO OPERATIONS | +| | CHECK NUMBER OF PROCESSED BUFFERS IS EQUAL NUMBER OF REQUESTS | +| CHECK CRYPTO RESPONSE BUFFERS MATCH EXPECTED OUTPUT <-- | <-- [END] | +| FREE ALLOCATED RESOURCES | | +| [END] | | + +## 2. Running + +Primary process requires path name of the secondary process application as an argument. +Example: `> ./imb-mp-primary ${PWD}/imb-mp-secondary` + +## 3. Multi-Buffer Manager (MB MGR) Notes + +See `mp_imb.c` file and `init_imb()` function as reference for setting up multi-buffer manager in the primary process and fail-over initialization in the secondary process. +Note that for fail-over scenario, all crypto operations need to be done on data structures allocated in shared memory. Virtual addresses of the buffers and associated data structures need to be identical in the primary and secondary processes. + +## 4. OS Implementation notes + +### Linux + +No issues encountered. Virtual address given automatically to the primary process shared memory at `mmap()` is good to map in the secondary process. + +### FreeBSD + +Using arbitrary virtual address to allocate common virtual addresses for the primary and secondary processes. Address growth from the bottom to top. +`MAP_PREFAULT_READ` was found to be required on FreeBSD system when calling `mmap()`. Otherwise the secondary process was crashing when accessing open shared memory. + +### Windows + +Shared memory handle cannot be closed by the primary process before executing `system()` otherwise secondary process cannot map named shared memory. +Using maximum application address from `GetSystemInfo()` to allocate common virtual addresses for the primary and secondary processes. Stack based approach applied here and moving from the top address down. diff --git a/test/mp-app/imb-mp-primary.c b/test/mp-app/imb-mp-primary.c new file mode 100644 index 0000000000000000000000000000000000000000..d44913a3b91c2e91e3628e2576604ce904e97d8e --- /dev/null +++ b/test/mp-app/imb-mp-primary.c @@ -0,0 +1,276 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include + +#include "mp_alloc.h" +#include "mp_shared_mem.h" +#include "mp_imb.h" +#include "mp_info_context.h" + +#if defined(__MINGW32__) + +static int +mp_primary(const char *name2) +{ + (void) name2; + printf("Multi-Process test not executed.\n"); + return 0; +} + +#else + +#if defined(__linux__) || defined(__FreeBSD__) +#include +#endif + +/* + * ============================================================================= + * ============================================================================= + * Primary processes + */ + +/* + * Process data + * - primary process allocates and initializes them + * - secondary process only picks them up + * All shared memory allocations will have the same virtual address + * in primary and secondary processes. + */ + +static int +alloc_crypto_op_data(struct info_context *ctx, struct allocator *app_alloc, const int is_pri) +{ + if (!is_pri) + return 0; + + /* + * Primary process does all memory allocations in shared memory and + * stores pointers in data section that secondary process will inherit + */ + size_t i; + + ctx->exp_enc_key = mp_alloc(app_alloc, 11 * 16, 16); + ctx->exp_dec_key = mp_alloc(app_alloc, 11 * 16, 16); + ctx->aes_key = mp_alloc(app_alloc, 16, 0); + ctx->iv = mp_alloc(app_alloc, 16, 0); + + for (i = 0; i < IMB_DIM(ctx->buffer_table_in_out); i++) { + ctx->buffer_table_in_out[i] = mp_alloc(app_alloc, buffer_size, 4); + if (ctx->buffer_table_in_out[i] == NULL) + break; + memset(ctx->buffer_table_in_out[i], (int) ~i, buffer_size); + + ctx->buffer_table_ref[i] = mp_alloc(app_alloc, buffer_size, 4); + if (ctx->buffer_table_ref[i] == NULL) + break; + memset(ctx->buffer_table_ref[i], 0, buffer_size); + } + + if (ctx->exp_enc_key == NULL || ctx->exp_dec_key == NULL || ctx->aes_key == NULL || + ctx->iv == NULL || i < IMB_DIM(ctx->buffer_table_in_out)) + return -1; + + return 0; +} + +static int +prepare_reference_output(struct info_context *ctx, const int is_pri) +{ + if (!is_pri) + return 0; + + /* Create key schedule and set IV */ + memset(ctx->aes_key, 0xaa, 16); + IMB_AES_KEYEXP_128(ctx->mb_mgr, ctx->aes_key, ctx->exp_enc_key, ctx->exp_dec_key); + + memset(ctx->iv, 0x55, 16); + + /* + * Use allocated manager to get reference answers + */ + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + + if (submit_aes_cbc_enc_jobs(ctx->mb_mgr, ctx->buffer_table_in_out, ctx->buffer_table_ref, + IMB_DIM(ctx->buffer_table_in_out), &ctx->jobs_received, + &ctx->jobs_sent, ctx->exp_enc_key, ctx->iv, buffer_size) != 0) + return -1; + + if (flush_aes_cbc_enc_jobs(ctx->mb_mgr, &ctx->jobs_received) != 0) + return -1; + + if (ctx->jobs_sent != IMB_DIM(ctx->buffer_table_in_out)) + return -1; + + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + return 0; +} + +static int +mp_primary(const char *name2) +{ + const int is_pri = 1; + struct shared_memory app_shm, info_shm; + struct info_context *ctx = NULL; + struct allocator app_alloc; + + fprintf(stdout, "PRIMARY: init start %p\n", (void *) imb_get_errno); + + if (shm_create(&info_shm, is_pri, SHM_INFO_NAME, SHM_INFO_SIZE, NULL) != 0) + return -1; + + /* cast info shared memory onto info context structure */ + ctx = (struct info_context *) info_shm.ptr; + memset(ctx, 0, sizeof(*ctx)); + + if (shm_create(&app_shm, is_pri, SHM_DATA_NAME, SHM_DATA_SIZE, NULL) != 0) { + (void) shm_destroy(&info_shm, is_pri); + return -1; + } + + /* secondary process needs to mmap app/data shared memory at this address */ + ctx->app_mmap = app_shm.ptr; + + /* init allocator on app/data shared memory */ + mp_init(&app_alloc, app_shm.ptr, app_shm.size); + + /* init IMB */ + ctx->mb_mgr = init_imb(NULL, &app_alloc, is_pri); + if (ctx->mb_mgr == NULL) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "PRIMARY: init complete\n"); + + /* allocate data for crypto operations */ + if (alloc_crypto_op_data(ctx, &app_alloc, is_pri) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + /* generate reference output data */ + if (prepare_reference_output(ctx, is_pri) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + /* send jobs in primary process */ + ctx->jobs_sent = 0; + ctx->jobs_received = 0; + if (submit_aes_cbc_enc_jobs(ctx->mb_mgr, ctx->buffer_table_in_out, ctx->buffer_table_in_out, + IMB_DIM(ctx->buffer_table_in_out), &ctx->jobs_received, + &ctx->jobs_sent, ctx->exp_enc_key, ctx->iv, buffer_size) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "PRIMARY: sent %u AES-128-CBC encrypt jobs\n", ctx->jobs_sent); + fprintf(stdout, "PRIMARY: received %u AES-128-CBC encrypt jobs\n", ctx->jobs_received); + + if (ctx->jobs_sent != IMB_DIM(ctx->buffer_table_in_out)) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + /* + * - spawn secondary process now + * - let the secondary perform the flush operation + * - wait for the secondary process to complete and check the results + */ + fprintf(stdout, "PRIMARY: starting SECONDARY process now\n"); + + const int status = system(name2); + +#ifdef _WIN32 + const int err = (status != EXIT_SUCCESS); +#endif + +#if defined(__linux__) || defined(__FreeBSD__) + const int err = (!WIFEXITED(status)) || (WEXITSTATUS(status) != EXIT_SUCCESS); +#endif + + if (err != 0) { + fprintf(stderr, "PRIMARY: SECONDARY process failed\n"); + fprintf(stdout, "MULTI-PROCESS TEST: FAILED\n"); + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "PRIMARY: SECONDARY has finished\n"); + + /* + * Child process exited normally - let's check the answers + */ + unsigned mismatch = 0; + + for (size_t i = 0; i < IMB_DIM(ctx->buffer_table_in_out); i++) + if (memcmp(ctx->buffer_table_in_out[i], ctx->buffer_table_ref[i], buffer_size) != 0) + mismatch++; + + fprintf(stdout, "MULTI-PROCESS TEST: %s\n", mismatch ? "FAILED " : "PASSED"); + + fprintf(stdout, "PRIMARY: finished\n"); + + /* clean up and exit */ + if (shm_destroy(&info_shm, is_pri) != 0) { + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + if (shm_destroy(&app_shm, is_pri) != 0) + return -1; + + return 0; +} +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +int +main(int argc, char **argv) +{ + int ret = -1; + + if (argc > 1) + ret = mp_primary(argv[1]); + else + fprintf(stderr, + "ERROR: argument required! Command syntax: %s \n", + argv[0]); + + return (ret == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/mp-app/imb-mp-secondary.c b/test/mp-app/imb-mp-secondary.c new file mode 100644 index 0000000000000000000000000000000000000000..7df2ff8657ce0b5c5bf4ec64dc2a8d50fba1d13e --- /dev/null +++ b/test/mp-app/imb-mp-secondary.c @@ -0,0 +1,139 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include + +#include + +#include "mp_shared_mem.h" +#include "mp_imb.h" +#include "mp_info_context.h" + +#if defined(__MINGW32__) + +static int +mp_secondary(void) +{ + printf("Multi-Process test not executed.\n"); + return 0; +} + +#else + +/* + * ============================================================================= + * ============================================================================= + * Secondary processes + */ + +static int +mp_secondary(void) +{ + const int is_pri = 0; + struct shared_memory app_shm, info_shm; + struct info_context *ctx = NULL; + + fprintf(stdout, "SECONDARY: init start %p\n", (void *) imb_get_errno); + + if (shm_create(&info_shm, is_pri, SHM_INFO_NAME, SHM_INFO_SIZE, NULL) != 0) + return -1; + + /* cast info shared memory onto info context structure */ + ctx = (struct info_context *) info_shm.ptr; + + /* check if any jobs were sent */ + if (ctx->jobs_sent == 0) { + (void) shm_destroy(&info_shm, is_pri); + return -1; + } + + if (shm_create(&app_shm, is_pri, SHM_DATA_NAME, SHM_DATA_SIZE, ctx->app_mmap) != 0) { + (void) shm_destroy(&info_shm, is_pri); + return -1; + } + + /* init IMB */ + if (init_imb(ctx->mb_mgr, NULL, is_pri) == NULL) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "SECONDARY: init complete\n"); + + /* flush jobs sent by primary process */ + unsigned jobs_received_now = 0; + + if (flush_aes_cbc_enc_jobs(ctx->mb_mgr, &jobs_received_now) != 0) { + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + ctx->jobs_received += jobs_received_now; + + fprintf(stdout, "SECONDARY: received %u (total %u) AES-128-CBC encrypt jobs\n", + jobs_received_now, ctx->jobs_received); + + if (ctx->jobs_sent != ctx->jobs_received) { + fprintf(stderr, "SECONDARY: expected %u jobs, received %u\n", ctx->jobs_sent, + ctx->jobs_received); + (void) shm_destroy(&info_shm, is_pri); + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + + fprintf(stdout, "SECONDARY: finished\n"); + +#ifdef _WIN32 + _flushall(); +#endif + + /* clean up and exit */ + if (shm_destroy(&info_shm, is_pri) != 0) { + (void) shm_destroy(&app_shm, is_pri); + return -1; + } + if (shm_destroy(&app_shm, is_pri) != 0) + return -1; + + return 0; +} +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +int +main(int argc, char **argv) +{ + (void) argc; + (void) argv; + + const int ret = mp_secondary(); + + return (ret == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/mp-app/mp_alloc.c b/test/mp-app/mp_alloc.c new file mode 100644 index 0000000000000000000000000000000000000000..30394d940b6796ceed5c914ae4aaa5d005825e4e --- /dev/null +++ b/test/mp-app/mp_alloc.c @@ -0,0 +1,58 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "mp_alloc.h" + +void +mp_init(struct allocator *a, void *ptr, const size_t size) +{ + a->ptr = ptr; + a->size = size; + a->offset = 0; +} + +void * +mp_alloc(struct allocator *a, const size_t length, const size_t alignment) +{ + if (a->ptr == NULL) + return NULL; + + if ((a->offset + length) > a->size) + return NULL; + + if (alignment > 1) { + const size_t align_mask = alignment - 1; + + a->offset = (a->offset + align_mask) & (~align_mask); + } + + void *ptr = ((char *) a->ptr + a->offset); + + a->offset += length; + + return ptr; +} diff --git a/test/mp-app/mp_alloc.h b/test/mp-app/mp_alloc.h new file mode 100644 index 0000000000000000000000000000000000000000..a1fc7c067ca5afd5d0f84053f9e31ee3417eb2ee --- /dev/null +++ b/test/mp-app/mp_alloc.h @@ -0,0 +1,68 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_ALLOC_H +#define MP_ALLOC_H + +#include + +/* + * ============================================================================= + * ============================================================================= + * Basic shared memory allocator + */ + +struct allocator { + void *ptr; + size_t offset; + size_t size; +}; + +/** + * @brief Simple memory allocator initialization + * + * @param a pointer to allocator instance structure + * @param ptr pointer to memory chunk base pointer + * @param size memory chunk size in bytes + */ +void +mp_init(struct allocator *a, void *ptr, const size_t size); + +/** + * @brief Simple memory allocator from the shared memory pool + * + * @param a pointer to allocator instance structure + * @param length data size to allocate in bytes + * @param alignment 0 or any power of 2 to align memory allocation to + * + * @return Pointer to allocated memory + * @retval NULL allocation error + */ +void * +mp_alloc(struct allocator *a, const size_t length, const size_t alignment); + +#endif /* MP_ALLOC_H */ diff --git a/test/mp-app/mp_imb.c b/test/mp-app/mp_imb.c new file mode 100644 index 0000000000000000000000000000000000000000..ea1d90fa3c1f4a548e4825d3275d0df2079b783f --- /dev/null +++ b/test/mp-app/mp_imb.c @@ -0,0 +1,145 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include +#include +#include "mp_alloc.h" +#include "mp_imb.h" + +/* + * ============================================================================= + * ============================================================================= + * IMB submit, flush and init functions + */ + +int +flush_aes_cbc_enc_jobs(IMB_MGR *p_mgr, unsigned *jobs_received) +{ + if (p_mgr == NULL || jobs_received == NULL) + return -2; + + while (IMB_FLUSH_JOB(p_mgr) != NULL) { + const int err = imb_get_errno(p_mgr); + + if (err != 0) { + fprintf(stderr, "!Flush error: %s!\n", imb_get_strerror(err)); + return -1; + } else { + *jobs_received = *jobs_received + 1; + } + } + + return 0; +} + +int +submit_aes_cbc_enc_jobs(IMB_MGR *p_mgr, void **in, void **out, const size_t n, + unsigned *jobs_received, unsigned *jobs_sent, void *exp_enc_key, void *iv, + const size_t msg_size) +{ + if (p_mgr == NULL || in == NULL || out == NULL || n == 0 || jobs_received == NULL || + jobs_sent == NULL || exp_enc_key == NULL || iv == NULL) + return -2; + + for (size_t i = 0; i < n; i++) { + IMB_JOB *job = IMB_GET_NEXT_JOB(p_mgr); + + memset(job, 0, sizeof(*job)); + + job->cipher_direction = IMB_DIR_ENCRYPT; + job->chain_order = IMB_ORDER_CIPHER_HASH; + job->src = in[i]; + job->dst = out[i]; + job->cipher_mode = IMB_CIPHER_CBC; + job->enc_keys = exp_enc_key; + job->dec_keys = NULL; + job->key_len_in_bytes = IMB_KEY_128_BYTES; + + job->iv = iv; + job->iv_len_in_bytes = 16; + job->cipher_start_src_offset_in_bytes = 0; + job->msg_len_to_cipher_in_bytes = msg_size; + job->hash_alg = IMB_AUTH_NULL; + + IMB_JOB *job_ret = IMB_SUBMIT_JOB(p_mgr); + + if (imb_get_errno(p_mgr) == 0) + *jobs_sent = *jobs_sent + 1; + else + return -1; + + if (job_ret != NULL) + *jobs_received = *jobs_received + 1; + } + + return 0; +} + +IMB_MGR * +init_imb(IMB_MGR *in_mb_mgr, struct allocator *app_alloc, const int is_pri) +{ + if (is_pri) { + /* + * Primary process does all memory allocations in shared memory and + * stores pointers in data section that secondary process will inherit + */ + void *p_mgr = mp_alloc(app_alloc, imb_get_mb_mgr_size(), 64); + + if (p_mgr == NULL) + return NULL; + + /* + * Set up multi-buffer manager in the shared memory + * - imb_set_pointers_mb_mgr() call with reset parameter is required + * Normally, alloc_mb_mgr() clears memory and sets selected feature flags. + * - it is followed with init_mb_mgr_auto() call + */ + + IMB_MGR *mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 1); + + if (mb_mgr == NULL) + return NULL; + + init_mb_mgr_auto(mb_mgr, NULL); + + if (imb_get_errno(mb_mgr) != 0) + return NULL; + + return mb_mgr; + + } else { + /* + * Secondary process picks allocations done by primary process and + * resets functions pointers in the manager + */ + void *p_mgr = (void *) in_mb_mgr; + IMB_MGR *mb_mgr = imb_set_pointers_mb_mgr(p_mgr, 0, 0); + + return mb_mgr; + } +} diff --git a/test/mp-app/mp_imb.h b/test/mp-app/mp_imb.h new file mode 100644 index 0000000000000000000000000000000000000000..344cbd3202b55a1311938f8d4426f8d38d499255 --- /dev/null +++ b/test/mp-app/mp_imb.h @@ -0,0 +1,51 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_IMB_H +#define MP_IMB_H + +#include +#include "mp_alloc.h" + +/* + * ============================================================================= + * ============================================================================= + * IMB submit, flush and init API + */ + +int +flush_aes_cbc_enc_jobs(IMB_MGR *p_mgr, unsigned *jobs_received); + +int +submit_aes_cbc_enc_jobs(IMB_MGR *p_mgr, void **in, void **out, const size_t n, + unsigned *jobs_received, unsigned *jobs_sent, void *exp_enc_key, void *iv, + const size_t msg_size); + +IMB_MGR * +init_imb(IMB_MGR *in_mb_mgr, struct allocator *app_alloc, const int is_pri); + +#endif /* MP_IMB_H */ diff --git a/test/mp-app/mp_info_context.h b/test/mp-app/mp_info_context.h new file mode 100644 index 0000000000000000000000000000000000000000..3adda9df0e965762d21eb54d5cd1d5dd42f766bd --- /dev/null +++ b/test/mp-app/mp_info_context.h @@ -0,0 +1,50 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef INFO_CONTEXT_H +#define INFO_CONTEXT_H + +#include + +const size_t buffer_size = 17 * 16; + +struct info_context { + unsigned jobs_sent; + unsigned jobs_received; + + void *app_mmap; + + IMB_MGR *mb_mgr; + void *exp_enc_key; + void *exp_dec_key; + void *aes_key; + void *iv; + void *buffer_table_in_out[15]; + void *buffer_table_ref[15]; +}; + +#endif /* INFO_CONTEXT_H */ diff --git a/test/mp-app/mp_shared_mem.c b/test/mp-app/mp_shared_mem.c new file mode 100644 index 0000000000000000000000000000000000000000..b996335c7cd68f17b2273454d08d66bedc8cc441 --- /dev/null +++ b/test/mp-app/mp_shared_mem.c @@ -0,0 +1,306 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "mp_shared_mem.h" + +#if defined(__MINGW32__) + +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + (void) is_pri; + sm->name = NULL; + sm->size = 0; + sm->ptr = NULL; + return 0; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr) +{ + (void) is_pri; + (void) mmap_ptr; + sm->name = name; + sm->size = size; + sm->ptr = NULL; + return 0; +} + +#else + +#if defined(__linux__) || defined(__FreeBSD__) +#include +#include +#include +#include +#endif + +#ifdef _WIN32 +/* + * Disable C5105 to workaround warning coming from winbase.h file + * "Windows Kits\10\include\10.0.19041.0\um\winbase.h(9531): warning + * C5105: macro expansion producing 'defined' has undefined behavior" + */ +#pragma warning(disable : 5105) + +#include +#include +#include +#endif + +/* + * ============================================================================= + * ============================================================================= + * Linux & FreeBSD: Shared memory create and destroy + */ + +#if defined(__FreeBSD__) || defined(__linux__) +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + int ret = 0; + + if (!is_pri) + if (munmap(sm->ptr, sm->size) != 0) + ret = -1; + sm->ptr = NULL; + + if (is_pri) + if (shm_unlink(sm->name) != 0) + ret = -1; + + sm->name = NULL; + sm->size = 0; + return ret; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr) +{ + int fd = -1; + + sm->name = name; + sm->size = size; + sm->ptr = MAP_FAILED; + + /* create the shared memory object */ + if (is_pri) + fd = shm_open(sm->name, O_CREAT | O_RDWR, 0666); + else + fd = shm_open(sm->name, O_RDWR, 0666); + + if (fd == -1) + return -1; + + /* configure the size of the shared memory object */ + if (is_pri) { + if (ftruncate(fd, sm->size) != 0) { + (void) shm_destroy(sm, is_pri); + close(fd); + return -1; + } + } + + /* + * memory map the shared memory object + * - secondary process maps shared memory into the same region as the primary process + */ + if (is_pri) { +#ifdef __FreeBSD__ + static char *base = (char *) 0x900000000; /* arbitrary VA to start mapping from */ + const size_t page_sz = (size_t) (getpagesize() - 1); + + sm->ptr = mmap((void *) base, sm->size, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PREFAULT_READ | MAP_SHARED, fd, 0); + base += ((sm->size + page_sz) & (~page_sz)); +#else + sm->ptr = mmap(0, sm->size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); +#endif + } else { + if (mmap_ptr == NULL) { +#ifdef __FreeBSD__ + const int flags = MAP_PREFAULT_READ | MAP_SHARED; +#else + const int flags = MAP_SHARED; +#endif + + sm->ptr = mmap(NULL, sm->size, PROT_READ | PROT_WRITE, flags, fd, 0); + } else { +#ifdef __FreeBSD__ + const int flags = MAP_PREFAULT_READ | MAP_SHARED | MAP_FIXED; +#else + const int flags = MAP_SHARED | MAP_FIXED; +#endif + + sm->ptr = mmap(mmap_ptr, sm->size, PROT_READ | PROT_WRITE, flags, fd, 0); + if (mmap_ptr != sm->ptr) { + printf("mmap() %p != mmap_ptr %p\n", sm->ptr, mmap_ptr); + (void) shm_destroy(sm, is_pri); + return -1; + } + } + } + + close(fd); + + if (sm->ptr == MAP_FAILED) { + fprintf(stderr, "!mmap() of %s shared memory error\n", sm->name); + (void) shm_destroy(sm, is_pri); + return -1; + } + + return 0; +} +#endif /* __linux__ || __FreeBSD__ */ + +/* + * ============================================================================= + * ============================================================================= + * Windows: Shared memory create and destroy + */ + +#ifdef _WIN32 +static void +printLastError(const DWORD error) +{ + LPSTR message = NULL; + + FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR) &message, 0, + NULL); + + if (message) { + fprintf(stderr, "ERROR: %s\n", message); + LocalFree(message); + } +} + +int +shm_destroy(struct shared_memory *sm, const int is_pri) +{ + int ret = 0; + + if (sm->ptr != NULL) + UnmapViewOfFile(sm->ptr); + + if (sm->fd != INVALID_HANDLE_VALUE) + CloseHandle(sm->fd); + + sm->ptr = NULL; + sm->name = NULL; + sm->size = 0; + sm->fd = INVALID_HANDLE_VALUE; + return ret; +} + +int +shm_create(struct shared_memory *sm, const int is_pri, const TCHAR *name, const size_t size, + void *mmap_ptr) +{ + static char *base = NULL; + static SYSTEM_INFO si; + + if (base == NULL) { + GetSystemInfo(&si); + + const size_t allocMask = si.dwAllocationGranularity - 1; + const uintptr_t new_base = (uintptr_t) si.lpMaximumApplicationAddress; + + /* align base mapping address to allocation granularity */ + base = (char *) (new_base & (~allocMask)); + } + + HANDLE fd = INVALID_HANDLE_VALUE; + + sm->name = name; + sm->size = size; + sm->ptr = NULL; + sm->fd = INVALID_HANDLE_VALUE; + + if (is_pri) { + fd = CreateFileMappingA((HANDLE) INVALID_HANDLE_VALUE, /* Use the page file */ + NULL, PAGE_READWRITE, 0, (DWORD) sm->size, sm->name); + } else { + fd = OpenFileMappingA(FILE_MAP_ALL_ACCESS, FALSE, sm->name); + } + + if (fd == INVALID_HANDLE_VALUE) { + fprintf(stderr, "ERROR: %s failed to create shared memory object (%d)\n", sm->name, + GetLastError()); + printLastError(GetLastError()); + return -1; + } + + /* Map the shared memory object into the address space of the current process */ + if (is_pri) { + const size_t allocMask = si.dwAllocationGranularity - 1; + const size_t allocAlignedSize = (size + allocMask) & (~allocMask); + + /* adjust mapping address to allocation size (aligned to allocation granularity) */ + base = base - allocAlignedSize; + + sm->ptr = (void *) MapViewOfFileEx(fd, FILE_MAP_ALL_ACCESS, /* Read/write access */ + 0, 0, /* offset = 0 */ + 0 /* map all object */, (LPVOID) base); + + if (sm->ptr != (void *) base) { + fprintf(stderr, "!mmap(%p) = %p %s shared memory error\n", base, sm->ptr, + sm->name); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + } else { + sm->ptr = (void *) MapViewOfFileEx(fd, FILE_MAP_ALL_ACCESS, /* Read/write access */ + 0, 0, /* Offset = 0 */ + 0 /* map all object */, (LPVOID) mmap_ptr); + if ((mmap_ptr != NULL) && (sm->ptr != mmap_ptr)) { + fprintf(stderr, "!mmap(%p) = %p %s shared memory error\n", mmap_ptr, + sm->ptr, sm->name); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + } + + if (sm->ptr == NULL) { + fprintf(stderr, "ERROR: %s failed to map view of shared memory (%d)\n", sm->name, + GetLastError()); + printLastError(GetLastError()); + (void) shm_destroy(sm, is_pri); + return -1; + } + + sm->fd = fd; + return 0; +} +#endif + +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ diff --git a/test/mp-app/mp_shared_mem.h b/test/mp-app/mp_shared_mem.h new file mode 100644 index 0000000000000000000000000000000000000000..0201b0e2e1856d5e3216d6e1320028a207958e20 --- /dev/null +++ b/test/mp-app/mp_shared_mem.h @@ -0,0 +1,105 @@ +/***************************************************************************** + Copyright (c) 2024, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef MP_SHARED_MEM_H +#define MP_SHARED_MEM_H + +#include + +/* + * ============================================================================= + * ============================================================================= + * Shared memory definitions + */ + +#if defined(__MINGW32__) + +struct shared_memory { + size_t size; + const char *name; + void *ptr; +}; + +#else + +#ifdef _WIN32 +/* + * Disable C5105 to workaround warning coming from winbase.h file + * "Windows Kits\10\include\10.0.19041.0\um\winbase.h(9531): warning + * C5105: macro expansion producing 'defined' has undefined behavior" + */ +#pragma warning(disable : 5105) + +#include +#include +#endif + +struct shared_memory { + size_t size; +#ifdef _WIN32 + const TCHAR *name; + HANDLE fd; +#else + const char *name; +#endif + void *ptr; +}; + +#define SHM_DATA_SIZE (2ULL * 1024ULL * 1024ULL) +#define SHM_INFO_SIZE (4ULL * 1024ULL) + +#ifdef _WIN32 +#define SHM_DATA_NAME TEXT("Local\\MpAppShmData") +#define SHM_INFO_NAME TEXT("Local\\MpAppShmInfo") +#endif + +#ifdef __linux__ +#define SHM_DATA_NAME "mp-app-shm-data" +#define SHM_INFO_NAME "mp-app-shm-info" +#endif + +#ifdef __FreeBSD__ +#define SHM_DATA_NAME "/tmp/mp-app-shm-data" +#define SHM_INFO_NAME "/tmp/mp-app-shm-info" +#endif + +#endif /* _WIN32 || __linux__ || __FreeBSD__ */ + +/* + * ============================================================================= + * ============================================================================= + * Shared memory API + */ + +int +shm_destroy(struct shared_memory *sm, const int is_pri); + +int +shm_create(struct shared_memory *sm, const int is_pri, const char *name, const size_t size, + void *mmap_ptr); + +#endif /* MP_SHARED_MEM_H */ diff --git a/test/xvalid-app/ipsec_xvalid.c b/test/xvalid-app/ipsec_xvalid.c index f03c62cd00667ca1afe21700d5fe4369949e5415..1561c8617b6dabd0ebb5606776918a3a7be35716 100644 --- a/test/xvalid-app/ipsec_xvalid.c +++ b/test/xvalid-app/ipsec_xvalid.c @@ -45,9 +45,9 @@ #ifdef _WIN32 #include -#define strdup _strdup -#define BSWAP64 _byteswap_uint64 -#define __func__ __FUNCTION__ +#define strdup _strdup +#define BSWAP64 _byteswap_uint64 +#define __func__ __FUNCTION__ #define strcasecmp _stricmp #else #ifndef __aarch64__